From 983e5e12cb54ba2d7f4c4dbe3a5ca1cb7db7c961 Mon Sep 17 00:00:00 2001 From: Leon Chen Date: Mon, 22 Aug 2016 05:40:52 -0400 Subject: [PATCH] implement dense layer with tests --- .gitignore | 5 ++- index.html | 3 +- package.json | 5 ++- src/activations.js | 68 +++++++++++++++++----------------- src/engine/topology.js | 73 +++++++++++++++++++++++++++++++++++++ src/index.js | 4 +- src/layers/core.js | 69 +++++++++++++++++++++++++++++++++++ src/layers/index.js | 3 ++ src/tensor.js | 43 ++++++++++++++++++++++ test/activations.js | 14 +++---- test/layers/core.js | 83 ++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 323 insertions(+), 47 deletions(-) create mode 100644 src/engine/topology.js create mode 100644 src/layers/core.js create mode 100644 src/layers/index.js create mode 100644 test/layers/core.js diff --git a/.gitignore b/.gitignore index 920209c..46a9ae6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ node_modules/ npm-debug.log -# jupyter notebooks -notebooks/ +# jupyter +notebooks/**/.ipynb_checkpoints/ +notebooks/_scratchpad.ipynb diff --git a/index.html b/index.html index ffafd62..91b1b70 100644 --- a/index.html +++ b/index.html @@ -18,8 +18,9 @@ + diff --git a/package.json b/package.json index 6f791df..e0f75df 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "webgl", "gpu" ], - "author": "Leon Chen", + "author": "Leon Chen ", "license": "MIT", "bugs": { "url": "https://github.com/transcranial/keras-js/issues" @@ -33,7 +33,8 @@ "dependencies": { "cwise": "^1.0.9", "ndarray": "^1.0.18", - "ndarray-ops": "^1.2.2" + "ndarray-ops": "^1.2.2", + "ndarray-squeeze": "^1.0.2" }, "devDependencies": { "babel-core": "^6.13.2", diff --git a/src/activations.js b/src/activations.js index fbefe35..ccc0d7b 100644 --- a/src/activations.js +++ b/src/activations.js @@ -3,10 +3,10 @@ import ops from 'ndarray-ops' import cwise from 'cwise' /** - * Softmax activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Softmax activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function softmax (x) { if (x.tensor.shape.length === 1) { ops.expeq(x.tensor) @@ -32,10 +32,10 @@ const _softplus = cwise({ }) /** - * Softplus activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Softplus activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function softplus (x) { _softplus(x.tensor) return this @@ -49,22 +49,22 @@ const _softsign = cwise({ }) /** - * Softsign activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Softsign activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function softsign (x) { _softsign(x.tensor) return this } /** - * ReLU activation function. In-place operation. - * @param {Tensor} x - * @param {Number} alpha - * @param {Number} maxValue - * @returns {Tensor} `this` - */ +* ReLU activation function. In-place operation. +* @param {Tensor} x +* @param {Number} alpha +* @param {Number} maxValue +* @returns {Tensor} `this` +*/ export function relu (x, opts = {}) { const { alpha = 0, maxValue = null } = opts let neg @@ -91,10 +91,10 @@ const _tanh = cwise({ }) /** - * Tanh activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Tanh activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function tanh (x) { _tanh(x.tensor) return this @@ -108,10 +108,10 @@ const _sigmoid = cwise({ }) /** - * Sigmoid activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Sigmoid activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function sigmoid (x) { _sigmoid(x.tensor) return this @@ -132,20 +132,20 @@ const _hardSigmoid = cwise({ }) /** - * Hard-sigmoid activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Hard-sigmoid activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function hardSigmoid (x) { _hardSigmoid(x.tensor) return this } /** - * Linear activation function. In-place operation. - * @param {Tensor} x - * @returns {Tensor} `this` - */ +* Linear activation function. In-place operation. +* @param {Tensor} x +* @returns {Tensor} `this` +*/ export function linear (x) { return this } diff --git a/src/engine/topology.js b/src/engine/topology.js new file mode 100644 index 0000000..19ad8a0 --- /dev/null +++ b/src/engine/topology.js @@ -0,0 +1,73 @@ +import ndarray from 'ndarray' +import squeeze from 'ndarray-squeeze' + +export class Layer { + constructor (attrs = {}) { + this.name = attrs.name + } + + inboundNodes = [] + outboundNodes = [] + + params = [] + weights = {} + weblasWeights = {} + + /** + * Method for setting layer weights + * We store the weights as both Tensor instances, + * as well as weblas pipeline tensors if possible (which are in GPU memory) + * see https://github.com/waylonflinn/weblas/wiki/Pipeline + * + * @param {Tensor[]} weightsArr + */ + setWeights = weightsArr => { + this.params.forEach((p, i) => { + this.weights[p] = weightsArr[i] + }) + + // create weblas pipeline tensor weights + this.createWeblasWeights() + } + + /** + * Create weblas pipeline tensor weights + * 2-D only + */ + createWeblasWeights = () => { + this.params.forEach((p, i) => { + if (this.weights[p].tensor.shape.length === 1) { + const shape = [1, this.weights[p].tensor.shape[0]] + this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data) + } if (this.weights[p].tensor.shape.length === 2) { + const shape = this.weights[p].tensor.shape + this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data) + } + }) + } + + /** + * Sync weblas pipeline tensor weights + */ + syncWeblasWeights = () => { + this.params.forEach((p, i) => { + if (this.weblasWeights[p]) { + const shape = this.weblasWeights[p].shape + const arr = this.weblasWeights[p].transfer(true) + this.weights[p].tensor = squeeze(ndarray(arr, shape)) + } + }) + } + + /** + * Delete weblas pipeline tensor weights + */ + deleteWeblasWeights = () => { + this.params.forEach((p, i) => { + if (this.weblasWeights[p]) { + this.weblasWeights[p].delete() + delete this.weblasWeights[p] + } + }) + } +} diff --git a/src/index.js b/src/index.js index ceb8bd7..053b8ad 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,9 @@ import Tensor from './tensor' import * as activations from './activations' +import * as layers from './layers' export { Tensor, - activations + activations, + layers } diff --git a/src/layers/core.js b/src/layers/core.js new file mode 100644 index 0000000..0cad1ee --- /dev/null +++ b/src/layers/core.js @@ -0,0 +1,69 @@ +import * as activations from '../activations' +import { Layer } from '../engine/topology' + +export class Dense extends Layer { + constructor (outputDim, attrs = {}) { + super(attrs) + const { + activation = 'linear', + inputDim = null, + bias = true + } = attrs + + this.activation = activations[activation] + this.outputDim = outputDim + this.inputDim = inputDim + this.bias = bias + + /** + * Layer weights specification + */ + this.params = this.bias ? ['W', 'b'] : ['W'] + + /** + * Input shape specification + */ + if (this.inputDim) { + this.inputShape = [this.inputDim] + } + } + + /** + * Method for layer computational logic + * + * weblas notes: + * sgemm(M, N, K, alpha, A, B, beta, C), where A, B, C are Float32Array + * - alpha * A * B + beta * C + * - A has shape M x N + * - B has shape N x K + * - C has shape M x K + * pipeline.sgemm(alpha, A, B, beta, C), where A, B, C are weblas.pipeline.Tensor here + * - alpha * A * B^T + beta * C + * + * @param {Tensor} x + * @returns {Tensor} `this` + */ + call = x => { + if (!x.weblasTensor) { + x.createWeblasTensor() + } + + const bias = this.bias + ? this.weblasWeights.b + : new weblas.pipeline.Tensor([1, this.outputDim], new Float32Array(this.outputDim)) + + x.weblasTensor = weblas.pipeline.sgemm( + 1.0, + x.weblasTensor, + this.weblasWeights.W.transpose(true), + 1.0, + bias + ) + + // activation function in CPU memory + x.transferWeblasTensor() + this.activation(x) + + return this + } +} diff --git a/src/layers/index.js b/src/layers/index.js new file mode 100644 index 0000000..041b8f8 --- /dev/null +++ b/src/layers/index.js @@ -0,0 +1,3 @@ +import { Dense } from './core' + +export { Dense } diff --git a/src/tensor.js b/src/tensor.js index d009fc6..d2ed21e 100644 --- a/src/tensor.js +++ b/src/tensor.js @@ -1,4 +1,5 @@ import ndarray from 'ndarray' +import squeeze from 'ndarray-squeeze' export default class Tensor { constructor (data, shape, options = {}) { @@ -17,4 +18,46 @@ export default class Tensor { this.tensor = ndarray(new TypedArray([]), []) } } + + /** + * Reference to weblas pipeline tensor in GPU memory, if available + * see https://github.com/waylonflinn/weblas/wiki/Pipeline + */ + weblasTensor = null + + /** + * Create weblas pipeline tensor + * 2-D only + */ + createWeblasTensor = () => { + if (this.tensor.shape.length === 1) { + const shape = [1, this.tensor.shape[0]] + this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data) + } else if (this.tensor.shape.length === 2) { + const shape = this.tensor.shape + this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data) + } + } + + /** + * Transfers weblas pipeline tensor from GPU memory + */ + transferWeblasTensor = () => { + if (this.weblasTensor) { + const shape = this.weblasTensor.shape + const arr = this.weblasTensor.transfer(true) + this.tensor = squeeze(ndarray(arr, shape)) + } + } + + /** + * Delete weblas pipeline tensor + */ + deleteWeblasTensor = () => { + if (this.weblasTensor) { + this.weblasTensor.delete() + this.weblasTensor = null + } + } + } diff --git a/test/activations.js b/test/activations.js index 3ab577d..419c8a9 100644 --- a/test/activations.js +++ b/test/activations.js @@ -1,13 +1,13 @@ /* eslint-env browser, mocha */ -const assert = chai.assert -const activations = KerasJS.activations - -const styles = testUtils.styles -const approxEquals = testUtils.approxEquals -const logTime = testUtils.logTime - describe('activations', function () { + const assert = chai.assert + const styles = testUtils.styles + const approxEquals = testUtils.approxEquals + const logTime = testUtils.logTime + + const activations = KerasJS.activations + /********************************************************* * softmax *********************************************************/ diff --git a/test/layers/core.js b/test/layers/core.js new file mode 100644 index 0000000..d7fe536 --- /dev/null +++ b/test/layers/core.js @@ -0,0 +1,83 @@ +/* eslint-env browser, mocha */ + +describe('Layers: Core', function () { + const assert = chai.assert + const styles = testUtils.styles + const approxEquals = testUtils.approxEquals + const logTime = testUtils.logTime + + const layers = KerasJS.layers + + /********************************************************* + * Dense + *********************************************************/ + + describe('Dense', function () { + it('should produce expected values', function () { + console.log('\n%Layers: Core', styles.h1) + console.log('\n%cDense', styles.h2) + console.log('\n%ctest 1', styles.h3) + let testLayer = new layers.Dense(2) + testLayer.setWeights([ + new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]), + new KerasJS.Tensor([0.5, 0.7], [2]) + ]) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([7.3, -0.21]) + const shapeExpected = [2] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should produce expected values, with sigmoid activation function', function () { + console.log('\n%ctest 2 (with sigmoid activation)', styles.h3) + let testLayer = new layers.Dense(2, { activation: 'sigmoid' }) + testLayer.setWeights([ + new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]), + new KerasJS.Tensor([0.5, 0.7], [2]) + ]) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.999325, 0.447692]) + const shapeExpected = [2] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should produce expected values, with softplus activation function and no bias', function () { + console.log('\n%ctest 3 (with softplus activation and no bias)', styles.h3) + let testLayer = new layers.Dense(2, { activation: 'softplus', bias: false }) + testLayer.setWeights([ + new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]) + ]) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([6.801113, 0.338274]) + const shapeExpected = [2] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) +})