From 7a80001bd79f4e492e9ed57efb8696d4a36ef841 Mon Sep 17 00:00:00 2001 From: Leon Chen Date: Sun, 21 Aug 2016 22:59:23 -0400 Subject: [PATCH] implement activation functions with tests --- package.json | 1 + src/activations.js | 90 ++++++++++- test/activations.js | 356 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 441 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 6ea7535..6f791df 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ }, "homepage": "https://github.com/transcranial/keras-js#readme", "dependencies": { + "cwise": "^1.0.9", "ndarray": "^1.0.18", "ndarray-ops": "^1.2.2" }, diff --git a/src/activations.js b/src/activations.js index 5bb7fb6..fbefe35 100644 --- a/src/activations.js +++ b/src/activations.js @@ -1,5 +1,6 @@ import ndarray from 'ndarray' import ops from 'ndarray-ops' +import cwise from 'cwise' /** * Softmax activation function. In-place operation. @@ -23,12 +24,38 @@ export function softmax (x) { return this } -export function softplus (x) { +const _softplus = cwise({ + args: ['array'], + body: function (_x) { + _x = Math.log(Math.exp(_x) + 1) + } +}) +/** + * Softplus activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ +export function softplus (x) { + _softplus(x.tensor) + return this } -export function softsign (x) { +const _softsign = cwise({ + args: ['array'], + body: function (_x) { + _x /= 1 + Math.abs(_x) + } +}) +/** + * Softsign activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ +export function softsign (x) { + _softsign(x.tensor) + return this } /** @@ -56,18 +83,69 @@ export function relu (x, opts = {}) { return this } +const _tanh = cwise({ + args: ['array'], + body: function (_x) { + _x = Math.tanh(_x) + } +}) + +/** + * Tanh activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ export function tanh (x) { - + _tanh(x.tensor) + return this } +const _sigmoid = cwise({ + args: ['array'], + body: function (_x) { + _x = 1 / (1 + Math.exp(-_x)) + } +}) + +/** + * Sigmoid activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ export function sigmoid (x) { - + _sigmoid(x.tensor) + return this } +// Reference hard sigmoid with slope and shift values from theano, see +// https://github.com/Theano/Theano/blob/master/theano/tensor/nnet/sigm.py +const _hardSigmoid = cwise({ + args: ['array'], + body: function (_x) { + _x = (_x * 0.2) + 0.5 + if (_x <= 0) { + _x = 0 + } else if (_x >= 1) { + _x = 1 + } + } +}) + +/** + * Hard-sigmoid activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ export function hardSigmoid (x) { - + _hardSigmoid(x.tensor) + return this } +/** + * Linear activation function. In-place operation. + * @param {Tensor} x + * @returns {Tensor} `this` + */ export function linear (x) { - return x + return this } diff --git a/test/activations.js b/test/activations.js index bb13f9b..3ab577d 100644 --- a/test/activations.js +++ b/test/activations.js @@ -8,6 +8,10 @@ const approxEquals = testUtils.approxEquals const logTime = testUtils.logTime describe('activations', function () { + /********************************************************* + * softmax + *********************************************************/ + describe('softmax', function () { it('should work for 1D tensor', function () { console.log('\n%cactivations', styles.h1) @@ -46,6 +50,126 @@ describe('activations', function () { }) }) + /********************************************************* + * softplus + *********************************************************/ + + describe('softplus', function () { + it('should work for 1D tensor', function () { + console.log('\n%csoftplus', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softplus(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softplus(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928, 0.67826, 0.854355, 0.693147, 1.171101, 0.554355, 1.313262]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softplus(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.693147, 0.798139, 0.474077, 0.644397, 1.313262, 2.126928, 0.67826, 2.395545, 0.693147, 1.171101, 0.554355, 1.313262]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) + + /********************************************************* + * softsign + *********************************************************/ + + describe('softsign', function () { + it('should work for 1D tensor', function () { + console.log('\n%csoftsign', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softsign(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softsign(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.230769, 0.0, 0.444444, -0.230769, 0.5]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.softsign(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.166667, -0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.69697, 0.0, 0.444444, -0.230769, 0.5]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) + + /********************************************************* + * relu + *********************************************************/ + describe('relu', function () { it('should work for 1D tensor', function () { console.log('\n%crelu', styles.h2) @@ -133,4 +257,236 @@ describe('activations', function () { assert.isTrue(approxEquals(dataOut, dataExpected)) }) }) + + /********************************************************* + * tanh + *********************************************************/ + + describe('tanh', function () { + it('should work for 1D tensor', function () { + console.log('\n%ctanh', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.tanh(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.tanh(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.291313, 0.0, 0.664037, -0.291313, 0.761594]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.tanh(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.0, 0.197375, -0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.980096, 0.0, 0.664037, -0.291313, 0.761594]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) + + /********************************************************* + * sigmoid + *********************************************************/ + + describe('sigmoid', function () { + it('should work for 1D tensor', function () { + console.log('\n%csigmoid', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.sigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.sigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797, 0.492501, 0.574443, 0.5, 0.689974, 0.425557, 0.731059]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.sigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.549834, 0.377541, 0.475021, 0.731059, 0.880797, 0.492501, 0.908877, 0.5, 0.689974, 0.425557, 0.731059]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) + + /********************************************************* + * hardSigmoid + *********************************************************/ + + describe('hardSigmoid', function () { + it('should work for 1D tensor', function () { + console.log('\n%chardSigmoid', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.hardSigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.54, 0.6, 0.48, 0.7, 0.9]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.hardSigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.54, 0.6, 0.48, 0.7, 0.9, 0.494, 0.56, 0.5, 0.66, 0.44, 0.7]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.hardSigmoid(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0.5, 0.54, 0.4, 0.48, 0.7, 0.9, 0.494, 0.96, 0.5, 0.66, 0.44, 0.7]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) + + /********************************************************* + * linear + *********************************************************/ + + describe('linear', function () { + it('should work for 1D tensor', function () { + console.log('\n%clinear', styles.h2) + console.log('\n%c1D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.linear(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0, 0.2, 0.5, -0.1, 1, 2]) + const shapeExpected = [6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 2D tensor', function () { + console.log('\n%c2D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.linear(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]) + const shapeExpected = [2, 6] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + + it('should work for 3D tensor', function () { + console.log('\n%c3D', styles.h3) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3]) + console.log('%cin', styles.h4, t) + const startTime = performance.now() + activations.linear(t) + const endTime = performance.now() + console.log('%cout', styles.h4, t) + logTime(startTime, endTime) + const dataOut = t.tensor.data + const shapeOut = t.tensor.shape + const dataExpected = new Float32Array([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]) + const shapeExpected = [2, 2, 3] + assert.deepEqual(shapeOut, shapeExpected) + assert.isTrue(approxEquals(dataOut, dataExpected)) + }) + }) })