implement activation functions with tests

This commit is contained in:
Leon Chen
2016-08-21 22:59:23 -04:00
parent d4892e1ba5
commit 7a80001bd7
3 changed files with 441 additions and 6 deletions
+1
View File
@@ -31,6 +31,7 @@
},
"homepage": "https://github.com/transcranial/keras-js#readme",
"dependencies": {
"cwise": "^1.0.9",
"ndarray": "^1.0.18",
"ndarray-ops": "^1.2.2"
},
+84 -6
View File
@@ -1,5 +1,6 @@
import ndarray from 'ndarray'
import ops from 'ndarray-ops'
import cwise from 'cwise'
/**
* Softmax activation function. In-place operation.
@@ -23,12 +24,38 @@ export function softmax (x) {
return this
}
export function softplus (x) {
const _softplus = cwise({
args: ['array'],
body: function (_x) {
_x = Math.log(Math.exp(_x) + 1)
}
})
/**
* Softplus activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function softplus (x) {
_softplus(x.tensor)
return this
}
export function softsign (x) {
const _softsign = cwise({
args: ['array'],
body: function (_x) {
_x /= 1 + Math.abs(_x)
}
})
/**
* Softsign activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function softsign (x) {
_softsign(x.tensor)
return this
}
/**
@@ -56,18 +83,69 @@ export function relu (x, opts = {}) {
return this
}
const _tanh = cwise({
args: ['array'],
body: function (_x) {
_x = Math.tanh(_x)
}
})
/**
* Tanh activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function tanh (x) {
_tanh(x.tensor)
return this
}
const _sigmoid = cwise({
args: ['array'],
body: function (_x) {
_x = 1 / (1 + Math.exp(-_x))
}
})
/**
* Sigmoid activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function sigmoid (x) {
_sigmoid(x.tensor)
return this
}
// Reference hard sigmoid with slope and shift values from theano, see
// https://github.com/Theano/Theano/blob/master/theano/tensor/nnet/sigm.py
const _hardSigmoid = cwise({
args: ['array'],
body: function (_x) {
_x = (_x * 0.2) + 0.5
if (_x <= 0) {
_x = 0
} else if (_x >= 1) {
_x = 1
}
}
})
/**
* Hard-sigmoid activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function hardSigmoid (x) {
_hardSigmoid(x.tensor)
return this
}
/**
* Linear activation function. In-place operation.
* @param {Tensor} x
* @returns {Tensor} `this`
*/
export function linear (x) {
return x
return this
}
+356
View File
@@ -8,6 +8,10 @@ const approxEquals = testUtils.approxEquals
const logTime = testUtils.logTime
describe('activations', function () {
/*********************************************************
* softmax
*********************************************************/
describe('softmax', function () {
it('should work for 1D tensor', function () {
console.log('\n%cactivations', styles.h1)
@@ -46,6 +50,126 @@ describe('activations', function () {
})
})
/*********************************************************
* softplus
*********************************************************/
describe('softplus', function () {
it('should work for 1D tensor', function () {
console.log('\n%csoftplus', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softplus(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softplus(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928, 0.67826, 0.854355, 0.693147, 1.171101, 0.554355, 1.313262])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softplus(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.693147, 0.798139, 0.474077, 0.644397, 1.313262, 2.126928, 0.67826, 2.395545, 0.693147, 1.171101, 0.554355, 1.313262])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* softsign
*********************************************************/
describe('softsign', function () {
it('should work for 1D tensor', function () {
console.log('\n%csoftsign', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softsign(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softsign(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.230769, 0.0, 0.444444, -0.230769, 0.5])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.softsign(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.166667, -0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.69697, 0.0, 0.444444, -0.230769, 0.5])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* relu
*********************************************************/
describe('relu', function () {
it('should work for 1D tensor', function () {
console.log('\n%crelu', styles.h2)
@@ -133,4 +257,236 @@ describe('activations', function () {
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* tanh
*********************************************************/
describe('tanh', function () {
it('should work for 1D tensor', function () {
console.log('\n%ctanh', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.tanh(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.tanh(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.291313, 0.0, 0.664037, -0.291313, 0.761594])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.tanh(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.0, 0.197375, -0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.980096, 0.0, 0.664037, -0.291313, 0.761594])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* sigmoid
*********************************************************/
describe('sigmoid', function () {
it('should work for 1D tensor', function () {
console.log('\n%csigmoid', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.sigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.sigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797, 0.492501, 0.574443, 0.5, 0.689974, 0.425557, 0.731059])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.sigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.549834, 0.377541, 0.475021, 0.731059, 0.880797, 0.492501, 0.908877, 0.5, 0.689974, 0.425557, 0.731059])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* hardSigmoid
*********************************************************/
describe('hardSigmoid', function () {
it('should work for 1D tensor', function () {
console.log('\n%chardSigmoid', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.hardSigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.54, 0.6, 0.48, 0.7, 0.9])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.hardSigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.54, 0.6, 0.48, 0.7, 0.9, 0.494, 0.56, 0.5, 0.66, 0.44, 0.7])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.hardSigmoid(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0.5, 0.54, 0.4, 0.48, 0.7, 0.9, 0.494, 0.96, 0.5, 0.66, 0.44, 0.7])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
/*********************************************************
* linear
*********************************************************/
describe('linear', function () {
it('should work for 1D tensor', function () {
console.log('\n%clinear', styles.h2)
console.log('\n%c1D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.linear(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0, 0.2, 0.5, -0.1, 1, 2])
const shapeExpected = [6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 2D tensor', function () {
console.log('\n%c2D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1], [2, 6])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.linear(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1])
const shapeExpected = [2, 6]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
it('should work for 3D tensor', function () {
console.log('\n%c3D', styles.h3)
let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1], [2, 2, 3])
console.log('%cin', styles.h4, t)
const startTime = performance.now()
activations.linear(t)
const endTime = performance.now()
console.log('%cout', styles.h4, t)
logTime(startTime, endTime)
const dataOut = t.tensor.data
const shapeOut = t.tensor.shape
const dataExpected = new Float32Array([0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1])
const shapeExpected = [2, 2, 3]
assert.deepEqual(shapeOut, shapeExpected)
assert.isTrue(approxEquals(dataOut, dataExpected))
})
})
})