diff --git a/.gitignore b/.gitignore
index 920209c..46a9ae6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,5 +2,6 @@
node_modules/
npm-debug.log
-# jupyter notebooks
-notebooks/
+# jupyter
+notebooks/**/.ipynb_checkpoints/
+notebooks/_scratchpad.ipynb
diff --git a/index.html b/index.html
index ffafd62..91b1b70 100644
--- a/index.html
+++ b/index.html
@@ -18,8 +18,9 @@
+
diff --git a/package.json b/package.json
index 6f791df..e0f75df 100644
--- a/package.json
+++ b/package.json
@@ -24,7 +24,7 @@
"webgl",
"gpu"
],
- "author": "Leon Chen",
+ "author": "Leon Chen ",
"license": "MIT",
"bugs": {
"url": "https://github.com/transcranial/keras-js/issues"
@@ -33,7 +33,8 @@
"dependencies": {
"cwise": "^1.0.9",
"ndarray": "^1.0.18",
- "ndarray-ops": "^1.2.2"
+ "ndarray-ops": "^1.2.2",
+ "ndarray-squeeze": "^1.0.2"
},
"devDependencies": {
"babel-core": "^6.13.2",
diff --git a/src/activations.js b/src/activations.js
index fbefe35..ccc0d7b 100644
--- a/src/activations.js
+++ b/src/activations.js
@@ -3,10 +3,10 @@ import ops from 'ndarray-ops'
import cwise from 'cwise'
/**
- * Softmax activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softmax activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function softmax (x) {
if (x.tensor.shape.length === 1) {
ops.expeq(x.tensor)
@@ -32,10 +32,10 @@ const _softplus = cwise({
})
/**
- * Softplus activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softplus activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function softplus (x) {
_softplus(x.tensor)
return this
@@ -49,22 +49,22 @@ const _softsign = cwise({
})
/**
- * Softsign activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Softsign activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function softsign (x) {
_softsign(x.tensor)
return this
}
/**
- * ReLU activation function. In-place operation.
- * @param {Tensor} x
- * @param {Number} alpha
- * @param {Number} maxValue
- * @returns {Tensor} `this`
- */
+* ReLU activation function. In-place operation.
+* @param {Tensor} x
+* @param {Number} alpha
+* @param {Number} maxValue
+* @returns {Tensor} `this`
+*/
export function relu (x, opts = {}) {
const { alpha = 0, maxValue = null } = opts
let neg
@@ -91,10 +91,10 @@ const _tanh = cwise({
})
/**
- * Tanh activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Tanh activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function tanh (x) {
_tanh(x.tensor)
return this
@@ -108,10 +108,10 @@ const _sigmoid = cwise({
})
/**
- * Sigmoid activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Sigmoid activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function sigmoid (x) {
_sigmoid(x.tensor)
return this
@@ -132,20 +132,20 @@ const _hardSigmoid = cwise({
})
/**
- * Hard-sigmoid activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Hard-sigmoid activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function hardSigmoid (x) {
_hardSigmoid(x.tensor)
return this
}
/**
- * Linear activation function. In-place operation.
- * @param {Tensor} x
- * @returns {Tensor} `this`
- */
+* Linear activation function. In-place operation.
+* @param {Tensor} x
+* @returns {Tensor} `this`
+*/
export function linear (x) {
return this
}
diff --git a/src/engine/topology.js b/src/engine/topology.js
new file mode 100644
index 0000000..19ad8a0
--- /dev/null
+++ b/src/engine/topology.js
@@ -0,0 +1,73 @@
+import ndarray from 'ndarray'
+import squeeze from 'ndarray-squeeze'
+
+export class Layer {
+ constructor (attrs = {}) {
+ this.name = attrs.name
+ }
+
+ inboundNodes = []
+ outboundNodes = []
+
+ params = []
+ weights = {}
+ weblasWeights = {}
+
+ /**
+ * Method for setting layer weights
+ * We store the weights as both Tensor instances,
+ * as well as weblas pipeline tensors if possible (which are in GPU memory)
+ * see https://github.com/waylonflinn/weblas/wiki/Pipeline
+ *
+ * @param {Tensor[]} weightsArr
+ */
+ setWeights = weightsArr => {
+ this.params.forEach((p, i) => {
+ this.weights[p] = weightsArr[i]
+ })
+
+ // create weblas pipeline tensor weights
+ this.createWeblasWeights()
+ }
+
+ /**
+ * Create weblas pipeline tensor weights
+ * 2-D only
+ */
+ createWeblasWeights = () => {
+ this.params.forEach((p, i) => {
+ if (this.weights[p].tensor.shape.length === 1) {
+ const shape = [1, this.weights[p].tensor.shape[0]]
+ this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data)
+ } if (this.weights[p].tensor.shape.length === 2) {
+ const shape = this.weights[p].tensor.shape
+ this.weblasWeights[p] = new weblas.pipeline.Tensor(shape, this.weights[p].tensor.data)
+ }
+ })
+ }
+
+ /**
+ * Sync weblas pipeline tensor weights
+ */
+ syncWeblasWeights = () => {
+ this.params.forEach((p, i) => {
+ if (this.weblasWeights[p]) {
+ const shape = this.weblasWeights[p].shape
+ const arr = this.weblasWeights[p].transfer(true)
+ this.weights[p].tensor = squeeze(ndarray(arr, shape))
+ }
+ })
+ }
+
+ /**
+ * Delete weblas pipeline tensor weights
+ */
+ deleteWeblasWeights = () => {
+ this.params.forEach((p, i) => {
+ if (this.weblasWeights[p]) {
+ this.weblasWeights[p].delete()
+ delete this.weblasWeights[p]
+ }
+ })
+ }
+}
diff --git a/src/index.js b/src/index.js
index ceb8bd7..053b8ad 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,7 +1,9 @@
import Tensor from './tensor'
import * as activations from './activations'
+import * as layers from './layers'
export {
Tensor,
- activations
+ activations,
+ layers
}
diff --git a/src/layers/core.js b/src/layers/core.js
new file mode 100644
index 0000000..0cad1ee
--- /dev/null
+++ b/src/layers/core.js
@@ -0,0 +1,69 @@
+import * as activations from '../activations'
+import { Layer } from '../engine/topology'
+
+export class Dense extends Layer {
+ constructor (outputDim, attrs = {}) {
+ super(attrs)
+ const {
+ activation = 'linear',
+ inputDim = null,
+ bias = true
+ } = attrs
+
+ this.activation = activations[activation]
+ this.outputDim = outputDim
+ this.inputDim = inputDim
+ this.bias = bias
+
+ /**
+ * Layer weights specification
+ */
+ this.params = this.bias ? ['W', 'b'] : ['W']
+
+ /**
+ * Input shape specification
+ */
+ if (this.inputDim) {
+ this.inputShape = [this.inputDim]
+ }
+ }
+
+ /**
+ * Method for layer computational logic
+ *
+ * weblas notes:
+ * sgemm(M, N, K, alpha, A, B, beta, C), where A, B, C are Float32Array
+ * - alpha * A * B + beta * C
+ * - A has shape M x N
+ * - B has shape N x K
+ * - C has shape M x K
+ * pipeline.sgemm(alpha, A, B, beta, C), where A, B, C are weblas.pipeline.Tensor here
+ * - alpha * A * B^T + beta * C
+ *
+ * @param {Tensor} x
+ * @returns {Tensor} `this`
+ */
+ call = x => {
+ if (!x.weblasTensor) {
+ x.createWeblasTensor()
+ }
+
+ const bias = this.bias
+ ? this.weblasWeights.b
+ : new weblas.pipeline.Tensor([1, this.outputDim], new Float32Array(this.outputDim))
+
+ x.weblasTensor = weblas.pipeline.sgemm(
+ 1.0,
+ x.weblasTensor,
+ this.weblasWeights.W.transpose(true),
+ 1.0,
+ bias
+ )
+
+ // activation function in CPU memory
+ x.transferWeblasTensor()
+ this.activation(x)
+
+ return this
+ }
+}
diff --git a/src/layers/index.js b/src/layers/index.js
new file mode 100644
index 0000000..041b8f8
--- /dev/null
+++ b/src/layers/index.js
@@ -0,0 +1,3 @@
+import { Dense } from './core'
+
+export { Dense }
diff --git a/src/tensor.js b/src/tensor.js
index d009fc6..d2ed21e 100644
--- a/src/tensor.js
+++ b/src/tensor.js
@@ -1,4 +1,5 @@
import ndarray from 'ndarray'
+import squeeze from 'ndarray-squeeze'
export default class Tensor {
constructor (data, shape, options = {}) {
@@ -17,4 +18,46 @@ export default class Tensor {
this.tensor = ndarray(new TypedArray([]), [])
}
}
+
+ /**
+ * Reference to weblas pipeline tensor in GPU memory, if available
+ * see https://github.com/waylonflinn/weblas/wiki/Pipeline
+ */
+ weblasTensor = null
+
+ /**
+ * Create weblas pipeline tensor
+ * 2-D only
+ */
+ createWeblasTensor = () => {
+ if (this.tensor.shape.length === 1) {
+ const shape = [1, this.tensor.shape[0]]
+ this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data)
+ } else if (this.tensor.shape.length === 2) {
+ const shape = this.tensor.shape
+ this.weblasTensor = new weblas.pipeline.Tensor(shape, this.tensor.data)
+ }
+ }
+
+ /**
+ * Transfers weblas pipeline tensor from GPU memory
+ */
+ transferWeblasTensor = () => {
+ if (this.weblasTensor) {
+ const shape = this.weblasTensor.shape
+ const arr = this.weblasTensor.transfer(true)
+ this.tensor = squeeze(ndarray(arr, shape))
+ }
+ }
+
+ /**
+ * Delete weblas pipeline tensor
+ */
+ deleteWeblasTensor = () => {
+ if (this.weblasTensor) {
+ this.weblasTensor.delete()
+ this.weblasTensor = null
+ }
+ }
+
}
diff --git a/test/activations.js b/test/activations.js
index 3ab577d..419c8a9 100644
--- a/test/activations.js
+++ b/test/activations.js
@@ -1,13 +1,13 @@
/* eslint-env browser, mocha */
-const assert = chai.assert
-const activations = KerasJS.activations
-
-const styles = testUtils.styles
-const approxEquals = testUtils.approxEquals
-const logTime = testUtils.logTime
-
describe('activations', function () {
+ const assert = chai.assert
+ const styles = testUtils.styles
+ const approxEquals = testUtils.approxEquals
+ const logTime = testUtils.logTime
+
+ const activations = KerasJS.activations
+
/*********************************************************
* softmax
*********************************************************/
diff --git a/test/layers/core.js b/test/layers/core.js
new file mode 100644
index 0000000..d7fe536
--- /dev/null
+++ b/test/layers/core.js
@@ -0,0 +1,83 @@
+/* eslint-env browser, mocha */
+
+describe('Layers: Core', function () {
+ const assert = chai.assert
+ const styles = testUtils.styles
+ const approxEquals = testUtils.approxEquals
+ const logTime = testUtils.logTime
+
+ const layers = KerasJS.layers
+
+ /*********************************************************
+ * Dense
+ *********************************************************/
+
+ describe('Dense', function () {
+ it('should produce expected values', function () {
+ console.log('\n%Layers: Core', styles.h1)
+ console.log('\n%cDense', styles.h2)
+ console.log('\n%ctest 1', styles.h3)
+ let testLayer = new layers.Dense(2)
+ testLayer.setWeights([
+ new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+ new KerasJS.Tensor([0.5, 0.7], [2])
+ ])
+ let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+ console.log('%cin', styles.h4, t)
+ const startTime = performance.now()
+ testLayer.call(t)
+ const endTime = performance.now()
+ console.log('%cout', styles.h4, t)
+ logTime(startTime, endTime)
+ const dataOut = t.tensor.data
+ const shapeOut = t.tensor.shape
+ const dataExpected = new Float32Array([7.3, -0.21])
+ const shapeExpected = [2]
+ assert.deepEqual(shapeOut, shapeExpected)
+ assert.isTrue(approxEquals(dataOut, dataExpected))
+ })
+
+ it('should produce expected values, with sigmoid activation function', function () {
+ console.log('\n%ctest 2 (with sigmoid activation)', styles.h3)
+ let testLayer = new layers.Dense(2, { activation: 'sigmoid' })
+ testLayer.setWeights([
+ new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+ new KerasJS.Tensor([0.5, 0.7], [2])
+ ])
+ let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+ console.log('%cin', styles.h4, t)
+ const startTime = performance.now()
+ testLayer.call(t)
+ const endTime = performance.now()
+ console.log('%cout', styles.h4, t)
+ logTime(startTime, endTime)
+ const dataOut = t.tensor.data
+ const shapeOut = t.tensor.shape
+ const dataExpected = new Float32Array([0.999325, 0.447692])
+ const shapeExpected = [2]
+ assert.deepEqual(shapeOut, shapeExpected)
+ assert.isTrue(approxEquals(dataOut, dataExpected))
+ })
+
+ it('should produce expected values, with softplus activation function and no bias', function () {
+ console.log('\n%ctest 3 (with softplus activation and no bias)', styles.h3)
+ let testLayer = new layers.Dense(2, { activation: 'softplus', bias: false })
+ testLayer.setWeights([
+ new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2])
+ ])
+ let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+ console.log('%cin', styles.h4, t)
+ const startTime = performance.now()
+ testLayer.call(t)
+ const endTime = performance.now()
+ console.log('%cout', styles.h4, t)
+ logTime(startTime, endTime)
+ const dataOut = t.tensor.data
+ const shapeOut = t.tensor.shape
+ const dataExpected = new Float32Array([6.801113, 0.338274])
+ const shapeExpected = [2]
+ assert.deepEqual(shapeOut, shapeExpected)
+ assert.isTrue(approxEquals(dataOut, dataExpected))
+ })
+ })
+})