From e235061eb98f80fb476ed5b7c0ef21d3de37c66c Mon Sep 17 00:00:00 2001 From: Leon Chen Date: Wed, 24 Aug 2016 01:07:18 -0400 Subject: [PATCH] implement advanced activation layers, with tests --- package.json | 2 +- src/layers/advanced_activations.js | 184 ++++++++++++++++++++++++++++ test/layers/advanced_activations.js | 143 +++++++++++++++++++++ 3 files changed, 328 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index bc6e1e3..72aaa0c 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "babel-plugin-transform-object-rest-spread": "^6.8.0", "babel-preset-es2015": "^6.13.2", "http-server": "^0.9.0", - "standard": "^8.0.0-beta.5", + "standard": "^8.0.0", "webpack": "^1.13.2" }, "standard": { diff --git a/src/layers/advanced_activations.js b/src/layers/advanced_activations.js index a9cc820..a81178c 100644 --- a/src/layers/advanced_activations.js +++ b/src/layers/advanced_activations.js @@ -1,5 +1,6 @@ import { Layer } from '../engine/topology' import { relu } from '../activations' +import cwise from 'cwise' /** * LeakyReLU advanced activation layer class @@ -24,3 +25,186 @@ export class LeakyReLU extends Layer { return x } } + +/** +* PReLU advanced activation layer class +* reference code: +* ``` +* pos = K.relu(x) +* neg = self.alphas * (x - abs(x)) * 0.5 +* return pos + neg +* ``` +*/ +export class PReLU extends Layer { + /** + * Creates a PReLU activation layer + */ + constructor () { + super({}) + + /** + * Layer weights specification + */ + this.params = ['alphas'] + } + + _compute = cwise({ + args: ['array', 'array'], + body: function (_x, alpha) { + _x = Math.max(_x, 0) + alpha * Math.min(_x, 0) + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call = x => { + this._compute(x.tensor, this.weights.alphas.tensor) + return x + } +} + +/** +* ELU advanced activation layer class +*/ +export class ELU extends Layer { + /** + * Creates a ELU activation layer + * @param {number} alpha - scale for the negative factor + */ + constructor (alpha = 1.0) { + super({}) + this.alpha = alpha + } + + _compute = cwise({ + args: ['array', 'scalar'], + body: function (_x, alpha) { + _x = Math.max(_x, 0) + alpha * (Math.exp(Math.min(_x, 0)) - 1) + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call = x => { + this._compute(x.tensor, this.alpha) + return x + } +} + +/** +* ParametricSoftplus advanced activation layer class +* alpha * log(1 + exp(beta * X)) +*/ +export class ParametricSoftplus extends Layer { + /** + * Creates a ParametricSoftplus activation layer + */ + constructor () { + super({}) + + /** + * Layer weights specification + */ + this.params = ['alphas', 'betas'] + } + + _compute = cwise({ + args: ['array', 'array', 'array'], + body: function (_x, alpha, beta) { + _x = alpha * Math.log(1 + Math.exp(beta * _x)) + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call = x => { + this._compute(x.tensor, this.weights.alphas.tensor, this.weights.betas.tensor) + return x + } +} + +/** +* ThresholdedReLU advanced activation layer class +*/ +export class ThresholdedReLU extends Layer { + /** + * Creates a ThresholdedReLU activation layer + * @param {number} theta - float >= 0. Threshold location of activation. + */ + constructor (theta = 1.0) { + super({}) + this.theta = theta + } + + _compute = cwise({ + args: ['array', 'scalar'], + body: function (_x, theta) { + _x = _x * Number(_x > theta) + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call = x => { + this._compute(x.tensor, this.theta) + return x + } +} + +/** +* SReLU advanced activation layer class +* S-shaped Rectified Linear Unit +*/ +export class SReLU extends Layer { + /** + * Creates a SReLU activation layer + */ + constructor () { + super({}) + + /** + * Layer weights specification + */ + this.params = ['t_left', 'a_left', 't_right', 'a_right'] + } + + // t_right_actual = t_left + abs(t_right) + // Y_left_and_center = t_left + K.relu(x - t_left, a_left, t_right_actual - t_left) + // Y_right = K.relu(x - t_right_actual) * a_right + // return Y_left_and_center + Y_right + _compute = cwise({ + args: ['array', 'array', 'array', 'array', 'array'], + body: function (_x, tL, aL, tR, aR) { + _x = tL + Math.min(Math.max(_x - tL, 0), Math.abs(tR)) + aL * Math.min(_x - tL, 0) + + Math.max(_x - (tL + Math.abs(tR)), 0) * aR + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call = x => { + this._compute( + x.tensor, + this.weights.t_left.tensor, + this.weights.a_left.tensor, + this.weights.t_right.tensor, + this.weights.a_right.tensor + ) + return x + } +} diff --git a/test/layers/advanced_activations.js b/test/layers/advanced_activations.js index dfdf4c6..4b7d6bf 100644 --- a/test/layers/advanced_activations.js +++ b/test/layers/advanced_activations.js @@ -37,4 +37,147 @@ describe('Layers: Advanced Activations', function () { assert.isTrue(approxEquals(t.tensor, dataExpected)) }) }) + + /********************************************************* + * PReLU + *********************************************************/ + + describe('PReLU', function () { + before(function () { + console.log('\n%cPReLU', styles.h2) + }) + + it('should produce expected values', function () { + console.log('\n%cweights: alphas', styles.h3) + let testLayer = new layers.PReLU() + testLayer.setWeights([ + new KerasJS.Tensor([-0.03, -0.02, 0.02, -0.03, -0.03, -0.01], [6]) + ]) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, stringifyCondensed(t.tensor)) + const startTime = performance.now() + t = testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, stringifyCondensed(t.tensor)) + logTime(startTime, endTime) + const dataExpected = new Float32Array([0.0, 0.2, -0.01, 0.003, 1.0, 2.0]) + const shapeExpected = [6] + assert.deepEqual(t.tensor.shape, shapeExpected) + assert.isTrue(approxEquals(t.tensor, dataExpected)) + }) + }) + + /********************************************************* + * ELU + *********************************************************/ + + describe('ELU', function () { + before(function () { + console.log('\n%cELU', styles.h2) + }) + + it('should produce expected values', function () { + console.log('\n%calpha=1.1', styles.h3) + let testLayer = new layers.ELU(1.1) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, stringifyCondensed(t.tensor)) + const startTime = performance.now() + t = testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, stringifyCondensed(t.tensor)) + logTime(startTime, endTime) + const dataExpected = new Float32Array([0.0, 0.2, -0.432816, -0.104679, 1.0, 2.0]) + const shapeExpected = [6] + assert.deepEqual(t.tensor.shape, shapeExpected) + assert.isTrue(approxEquals(t.tensor, dataExpected)) + }) + }) + + /********************************************************* + * ParametricSoftplus + *********************************************************/ + + describe('ParametricSoftplus', function () { + before(function () { + console.log('\n%cParametricSoftplus', styles.h2) + }) + + it('should produce expected values', function () { + console.log('\n%cweights: alphas, betas', styles.h3) + let testLayer = new layers.ParametricSoftplus() + testLayer.setWeights([ + new KerasJS.Tensor([0.13, -0.02, 0.02, -0.03, -0.03, -0.01], [6]), + new KerasJS.Tensor([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0], [6]) + ]) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, stringifyCondensed(t.tensor)) + const startTime = performance.now() + t = testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, stringifyCondensed(t.tensor)) + logTime(startTime, endTime) + const dataExpected = new Float32Array([0.090109, -0.013664, 0.013763, -0.020054, -0.023944, -0.006931]) + const shapeExpected = [6] + assert.deepEqual(t.tensor.shape, shapeExpected) + assert.isTrue(approxEquals(t.tensor, dataExpected)) + }) + }) + + /********************************************************* + * ThresholdedReLU + *********************************************************/ + + describe('ThresholdedReLU', function () { + before(function () { + console.log('\n%cThresholdedReLU', styles.h2) + }) + + it('should produce expected values', function () { + console.log('\n%theta=0.9', styles.h3) + let testLayer = new layers.ThresholdedReLU(0.9) + let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, stringifyCondensed(t.tensor)) + const startTime = performance.now() + t = testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, stringifyCondensed(t.tensor)) + logTime(startTime, endTime) + const dataExpected = new Float32Array([0.0, 0.0, 0.0, 0.0, 1.0, 2.0]) + const shapeExpected = [6] + assert.deepEqual(t.tensor.shape, shapeExpected) + assert.isTrue(approxEquals(t.tensor, dataExpected)) + }) + }) + + /********************************************************* + * SReLU + *********************************************************/ + + describe('SReLU', function () { + before(function () { + console.log('\n%cSReLU', styles.h2) + }) + + it('should produce expected values', function () { + console.log('\n%cweights: t_left, a_left, t_right, a_right', styles.h3) + let testLayer = new layers.SReLU() + testLayer.setWeights([ + new KerasJS.Tensor([0.13, -0.02, 0.02, -0.03, -0.03, -0.01], [6]), + new KerasJS.Tensor([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0], [6]), + new KerasJS.Tensor([-0.9, 0.8, 0.0, -1.0, 0.7, 0.4], [6]), + new KerasJS.Tensor([0.1, 0.2, 0.3, 0.0, 0.5, -0.2], [6]) + ]) + let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6]) + console.log('%cin', styles.h4, stringifyCondensed(t.tensor)) + const startTime = performance.now() + t = testLayer.call(t) + const endTime = performance.now() + console.log('%cout', styles.h4, stringifyCondensed(t.tensor)) + logTime(startTime, endTime) + const dataExpected = new Float32Array([0.1339, 0.2, 0.0096, -0.065, 0.835, 0.068]) + const shapeExpected = [6] + assert.deepEqual(t.tensor.shape, shapeExpected) + assert.isTrue(approxEquals(t.tensor, dataExpected)) + }) + }) })