implement dense layer with tests

.gitignore

@@ -2,5 +2,6 @@
 node_modules/
 npm-debug.log
 
-# jupyter notebooks
-notebooks/
+# jupyter
+notebooks/**/.ipynb_checkpoints/
+notebooks/_scratchpad.ipynb

index.html

@@ -18,8 +18,9 @@
   <script>mocha.setup('bdd')</script>
   <script src="/test/test-utils.js"></script>
   <script src="/test/activations.js"></script>
+  <script src="/test/layers/core.js"></script>
   <script>
-    mocha.checkLeaks();
+    // mocha.checkLeaks();
     mocha.globals(['jQuery']);
     mocha.run();
   </script>

package.json

@@ -24,7 +24,7 @@
     "webgl",
     "gpu"
   ],
-  "author": "Leon Chen",
+  "author": "Leon Chen <leon@md.ai>",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/transcranial/keras-js/issues"
@@ -33,7 +33,8 @@
   "dependencies": {
     "cwise": "^1.0.9",
     "ndarray": "^1.0.18",
-    "ndarray-ops": "^1.2.2"
+    "ndarray-ops": "^1.2.2",
+    "ndarray-squeeze": "^1.0.2"
   },
   "devDependencies": {
     "babel-core": "^6.13.2",

src/activations.js

@@ -3,10 +3,10 @@ import ops from 'ndarray-ops'
 import cwise from 'cwise'
 
 /**
- * Softmax activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softmax activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function softmax (x) {
   if (x.tensor.shape.length === 1) {
     ops.expeq(x.tensor)
@@ -32,10 +32,10 @@ const _softplus = cwise({
 })
 
 /**
- * Softplus activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softplus activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function softplus (x) {
   _softplus(x.tensor)
   return this
@@ -49,22 +49,22 @@ const _softsign = cwise({
 })
 
 /**
- * Softsign activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softsign activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function softsign (x) {
   _softsign(x.tensor)
   return this
 }
 
 /**
- * ReLU activation function. In-place operation.
- * @param {Tensor} x
- * @param {Number} alpha
- * @param {Number} maxValue
- * @returns {Tensor} `this`
- */
+* ReLU activation function. In-place operation.
+* @param {Tensor} x
+* @param {Number} alpha
+* @param {Number} maxValue
+* @returns {Tensor} `this`
+*/
 export function relu (x, opts = {}) {
   const { alpha = 0, maxValue = null } = opts
   let neg
@@ -91,10 +91,10 @@ const _tanh = cwise({
 })
 
 /**
- * Tanh activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Tanh activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function tanh (x) {
   _tanh(x.tensor)
   return this
@@ -108,10 +108,10 @@ const _sigmoid = cwise({
 })
 
 /**
- * Sigmoid activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Sigmoid activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function sigmoid (x) {
   _sigmoid(x.tensor)
   return this
@@ -132,20 +132,20 @@ const _hardSigmoid = cwise({
 })
 
 /**
- * Hard-sigmoid activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Hard-sigmoid activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function hardSigmoid (x) {
   _hardSigmoid(x.tensor)
   return this
 }
 
 /**
- * Linear activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Linear activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
 export function linear (x) {
   return this
 }

src/engine/topology.js

@@ -0,0 +1,73 @@
+import ndarray from 'ndarray'
+import squeeze from 'ndarray-squeeze'
+
+export class Layer {
+  constructor (attrs = {}) {
+    this.name = attrs.name
+  }
+
+  inboundNodes = []
+  outboundNodes = []
+
+  params = []
+  weights = {}
+  weblasWeights = {}
+
+  /**
+  * Method for setting layer weights
+  * We store the weights as both Tensor instances,
+  * as well as weblas pipeline tensors if possible (which are in GPU memory)
+  * see https://github.com/waylonflinn/weblas/wiki/Pipeline
+  *
+  * @param {Tensor[]} weightsArr
+  */
+  setWeights = weightsArr => {
+    this.params.forEach((p, i) => {
+      this.weights[p] = weightsArr[i]
+    })
+
+    // create weblas pipeline tensor weights
+    this.createWeblasWeights()
+  }
+
+  /**
+  * Create weblas pipeline tensor weights
+  * 2-D only
+  */
+  createWeblasWeights = () => {
+    this.params.forEach((p, i) => {
+      if (this.weights[p].tensor.shape.length === 1) {
+        const shape = [1, this.weights[p].tensor.shape[0]]
+        this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data)
+      } if (this.weights[p].tensor.shape.length === 2) {
+        const shape = this.weights[p].tensor.shape
+        this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data)
+      }
+    })
+  }
+
+  /**
+  * Sync weblas pipeline tensor weights
+  */
+  syncWeblasWeights = () => {
+    this.params.forEach((p, i) => {
+      if (this.weblasWeights[p]) {
+        const shape = this.weblasWeights[p].shape
+        const arr = this.weblasWeights[p].transfer(true)
+        this.weights[p].tensor = squeeze(ndarray(arr, shape))
+      }
+    })
+  }
+
+  /**
+  * Delete weblas pipeline tensor weights
+  */
+  deleteWeblasWeights = () => {
+    this.params.forEach((p, i) => {
+      if (this.weblasWeights[p]) {
+        this.weblasWeights[p].delete()
+        delete this.weblasWeights[p]
+      }
+    })
+  }
+}

src/index.js

@@ -1,7 +1,9 @@
 import Tensor from './tensor'
 import * as activations from './activations'
+import * as layers from './layers'
 
 export {
   Tensor,
-  activations
+  activations,
+  layers
 }

src/layers/core.js

@@ -0,0 +1,69 @@
+import * as activations from '../activations'
+import { Layer } from '../engine/topology'
+
+export class Dense extends Layer {
+  constructor (outputDim, attrs = {}) {
+    super(attrs)
+    const {
+      activation = 'linear',
+      inputDim = null,
+      bias = true
+    } = attrs
+
+    this.activation = activations[activation]
+    this.outputDim = outputDim
+    this.inputDim = inputDim
+    this.bias = bias
+
+    /**
+    * Layer weights specification
+    */
+    this.params = this.bias ? ['W', 'b'] : ['W']
+
+    /**
+    * Input shape specification
+    */
+    if (this.inputDim) {
+      this.inputShape = [this.inputDim]
+    }
+  }
+
+  /**
+  * Method for layer computational logic
+  *
+  * weblas notes:
+  * sgemm(M, N, K, alpha, A, B, beta, C), where A, B, C are Float32Array
+  * - alpha * A * B + beta * C
+  * - A has shape M x N
+  * - B has shape N x K
+  * - C has shape M x K
+  * pipeline.sgemm(alpha, A, B, beta, C), where A, B, C are weblas.pipeline.Tensor here
+  * - alpha * A * B^T + beta * C
+  *
+  * @param {Tensor} x
+  * @returns {Tensor} `this`
+  */
+  call = x => {
+    if (!x.weblasTensor) {
+      x.createWeblasTensor()
+    }
+
+    const bias = this.bias
+      ? this.weblasWeights.b
+      : new weblas.pipeline.Tensor([1, this.outputDim], new Float32Array(this.outputDim))
+
+    x.weblasTensor = weblas.pipeline.sgemm(
+      1.0,
+      x.weblasTensor,
+      this.weblasWeights.W.transpose(true),
+      1.0,
+      bias
+    )
+
+    // activation function in CPU memory
+    x.transferWeblasTensor()
+    this.activation(x)
+
+    return this
+  }
+}

src/layers/index.js

@@ -0,0 +1,3 @@
+import { Dense } from './core'
+
+export { Dense }

src/tensor.js

@@ -1,4 +1,5 @@
 import ndarray from 'ndarray'
+import squeeze from 'ndarray-squeeze'
 
 export default class Tensor {
   constructor (data, shape, options = {}) {
@@ -17,4 +18,46 @@ export default class Tensor {
       this.tensor = ndarray(new TypedArray([]), [])
     }
   }
+
+  /**
+  * Reference to weblas pipeline tensor in GPU memory, if available
+  * see https://github.com/waylonflinn/weblas/wiki/Pipeline
+  */
+  weblasTensor = null
+
+  /**
+  * Create weblas pipeline tensor
+  * 2-D only
+  */
+  createWeblasTensor = () => {
+    if (this.tensor.shape.length === 1) {
+      const shape = [1, this.tensor.shape[0]]
+      this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data)
+    } else if (this.tensor.shape.length === 2) {
+      const shape = this.tensor.shape
+      this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data)
+    }
+  }
+
+  /**
+  * Transfers weblas pipeline tensor from GPU memory
+  */
+  transferWeblasTensor = () => {
+    if (this.weblasTensor) {
+      const shape = this.weblasTensor.shape
+      const arr = this.weblasTensor.transfer(true)
+      this.tensor = squeeze(ndarray(arr, shape))
+    }
+  }
+
+  /**
+  * Delete weblas pipeline tensor
+  */
+  deleteWeblasTensor = () => {
+    if (this.weblasTensor) {
+      this.weblasTensor.delete()
+      this.weblasTensor = null
+    }
+  }
+
 }

test/activations.js

@@ -1,13 +1,13 @@
 /* eslint-env browser, mocha */
 
-const assert = chai.assert
-const activations = KerasJS.activations
-
-const styles = testUtils.styles
-const approxEquals = testUtils.approxEquals
-const logTime = testUtils.logTime
-
 describe('activations', function () {
+  const assert = chai.assert
+  const styles = testUtils.styles
+  const approxEquals = testUtils.approxEquals
+  const logTime = testUtils.logTime
+
+  const activations = KerasJS.activations
+
   /*********************************************************
    * softmax
    *********************************************************/

test/layers/core.js

@@ -0,0 +1,83 @@
+/* eslint-env browser, mocha */
+
+describe('Layers: Core', function () {
+  const assert = chai.assert
+  const styles = testUtils.styles
+  const approxEquals = testUtils.approxEquals
+  const logTime = testUtils.logTime
+
+  const layers = KerasJS.layers
+
+  /*********************************************************
+  * Dense
+  *********************************************************/
+
+  describe('Dense', function () {
+    it('should produce expected values', function () {
+      console.log('\n%Layers: Core', styles.h1)
+      console.log('\n%cDense', styles.h2)
+      console.log('\n%ctest 1', styles.h3)
+      let testLayer = new layers.Dense(2)
+      testLayer.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, t)
+      const startTime = performance.now()
+      testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, t)
+      logTime(startTime, endTime)
+      const dataOut = t.tensor.data
+      const shapeOut = t.tensor.shape
+      const dataExpected = new Float32Array([7.3, -0.21])
+      const shapeExpected = [2]
+      assert.deepEqual(shapeOut, shapeExpected)
+      assert.isTrue(approxEquals(dataOut, dataExpected))
+    })
+
+    it('should produce expected values, with sigmoid activation function', function () {
+      console.log('\n%ctest 2 (with sigmoid activation)', styles.h3)
+      let testLayer = new layers.Dense(2, { activation: 'sigmoid' })
+      testLayer.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, t)
+      const startTime = performance.now()
+      testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, t)
+      logTime(startTime, endTime)
+      const dataOut = t.tensor.data
+      const shapeOut = t.tensor.shape
+      const dataExpected = new Float32Array([0.999325, 0.447692])
+      const shapeExpected = [2]
+      assert.deepEqual(shapeOut, shapeExpected)
+      assert.isTrue(approxEquals(dataOut, dataExpected))
+    })
+
+    it('should produce expected values, with softplus activation function and no bias', function () {
+      console.log('\n%ctest 3 (with softplus activation and no bias)', styles.h3)
+      let testLayer = new layers.Dense(2, { activation: 'softplus', bias: false })
+      testLayer.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, t)
+      const startTime = performance.now()
+      testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, t)
+      logTime(startTime, endTime)
+      const dataOut = t.tensor.data
+      const shapeOut = t.tensor.shape
+      const dataExpected = new Float32Array([6.801113, 0.338274])
+      const shapeExpected = [2]
+      assert.deepEqual(shapeOut, shapeExpected)
+      assert.isTrue(approxEquals(dataOut, dataExpected))
+    })
+  })
+})