implement advanced activation layers, with tests

2026-09-11 12:20:53 +08:00 · 2016-08-24 01:07:18 -04:00
parent c3eec55d0c
commit e235061eb9
3 changed files with 328 additions and 1 deletions
@@ -51,7 +51,7 @@
    "babel-plugin-transform-object-rest-spread": "^6.8.0",
    "babel-preset-es2015": "^6.13.2",
    "http-server": "^0.9.0",
-    "standard": "^8.0.0-beta.5",
+    "standard": "^8.0.0",
    "webpack": "^1.13.2"
  },
  "standard": {
@@ -1,5 +1,6 @@
 import { Layer } from '../engine/topology'
 import { relu } from '../activations'
+import cwise from 'cwise'

 /**
 * LeakyReLU advanced activation layer class
@@ -24,3 +25,186 @@ export class LeakyReLU extends Layer {
    return x
  }
 }
+
+/**
+* PReLU advanced activation layer class
+* reference code:
+* ```
+* pos = K.relu(x)
+* neg = self.alphas * (x - abs(x)) * 0.5
+* return pos + neg
+* ```
+*/
+export class PReLU extends Layer {
+  /**
+  * Creates a PReLU activation layer
+  */
+  constructor () {
+    super({})
+
+    /**
+    * Layer weights specification
+    */
+    this.params = ['alphas']
+  }
+
+  _compute = cwise({
+    args: ['array', 'array'],
+    body: function (_x, alpha) {
+      _x = Math.max(_x, 0) + alpha * Math.min(_x, 0)
+    }
+  })
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    this._compute(x.tensor, this.weights.alphas.tensor)
+    return x
+  }
+}
+
+/**
+* ELU advanced activation layer class
+*/
+export class ELU extends Layer {
+  /**
+  * Creates a ELU activation layer
+  * @param {number} alpha - scale for the negative factor
+  */
+  constructor (alpha = 1.0) {
+    super({})
+    this.alpha = alpha
+  }
+
+  _compute = cwise({
+    args: ['array', 'scalar'],
+    body: function (_x, alpha) {
+      _x = Math.max(_x, 0) + alpha * (Math.exp(Math.min(_x, 0)) - 1)
+    }
+  })
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    this._compute(x.tensor, this.alpha)
+    return x
+  }
+}
+
+/**
+* ParametricSoftplus advanced activation layer class
+* alpha * log(1 + exp(beta * X))
+*/
+export class ParametricSoftplus extends Layer {
+  /**
+  * Creates a ParametricSoftplus activation layer
+  */
+  constructor () {
+    super({})
+
+    /**
+    * Layer weights specification
+    */
+    this.params = ['alphas', 'betas']
+  }
+
+  _compute = cwise({
+    args: ['array', 'array', 'array'],
+    body: function (_x, alpha, beta) {
+      _x = alpha * Math.log(1 + Math.exp(beta * _x))
+    }
+  })
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    this._compute(x.tensor, this.weights.alphas.tensor, this.weights.betas.tensor)
+    return x
+  }
+}
+
+/**
+* ThresholdedReLU advanced activation layer class
+*/
+export class ThresholdedReLU extends Layer {
+  /**
+  * Creates a ThresholdedReLU activation layer
+  * @param {number} theta - float >= 0. Threshold location of activation.
+  */
+  constructor (theta = 1.0) {
+    super({})
+    this.theta = theta
+  }
+
+  _compute = cwise({
+    args: ['array', 'scalar'],
+    body: function (_x, theta) {
+      _x = _x * Number(_x > theta)
+    }
+  })
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    this._compute(x.tensor, this.theta)
+    return x
+  }
+}
+
+/**
+* SReLU advanced activation layer class
+* S-shaped Rectified Linear Unit
+*/
+export class SReLU extends Layer {
+  /**
+  * Creates a SReLU activation layer
+  */
+  constructor () {
+    super({})
+
+    /**
+    * Layer weights specification
+    */
+    this.params = ['t_left', 'a_left', 't_right', 'a_right']
+  }
+
+  // t_right_actual = t_left + abs(t_right)
+  // Y_left_and_center = t_left + K.relu(x - t_left, a_left, t_right_actual - t_left)
+  // Y_right = K.relu(x - t_right_actual) * a_right
+  // return Y_left_and_center + Y_right
+  _compute = cwise({
+    args: ['array', 'array', 'array', 'array', 'array'],
+    body: function (_x, tL, aL, tR, aR) {
+      _x = tL + Math.min(Math.max(_x - tL, 0), Math.abs(tR)) + aL * Math.min(_x - tL, 0) +
+        Math.max(_x - (tL + Math.abs(tR)), 0) * aR
+    }
+  })
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    this._compute(
+      x.tensor,
+      this.weights.t_left.tensor,
+      this.weights.a_left.tensor,
+      this.weights.t_right.tensor,
+      this.weights.a_right.tensor
+    )
+    return x
+  }
+}
@@ -37,4 +37,147 @@ describe('Layers: Advanced Activations', function () {
      assert.isTrue(approxEquals(t.tensor, dataExpected))
    })
  })
+
+  /*********************************************************
+  * PReLU
+  *********************************************************/
+
+  describe('PReLU', function () {
+    before(function () {
+      console.log('\n%cPReLU', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%cweights: alphas', styles.h3)
+      let testLayer = new layers.PReLU()
+      testLayer.setWeights([
+        new KerasJS.Tensor([-0.03, -0.02, 0.02, -0.03, -0.03, -0.01], [6])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.0, 0.2, -0.01, 0.003, 1.0, 2.0])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+
+  /*********************************************************
+  * ELU
+  *********************************************************/
+
+  describe('ELU', function () {
+    before(function () {
+      console.log('\n%cELU', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%calpha=1.1', styles.h3)
+      let testLayer = new layers.ELU(1.1)
+      let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.0, 0.2, -0.432816, -0.104679, 1.0, 2.0])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+
+  /*********************************************************
+  * ParametricSoftplus
+  *********************************************************/
+
+  describe('ParametricSoftplus', function () {
+    before(function () {
+      console.log('\n%cParametricSoftplus', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%cweights: alphas, betas', styles.h3)
+      let testLayer = new layers.ParametricSoftplus()
+      testLayer.setWeights([
+        new KerasJS.Tensor([0.13, -0.02, 0.02, -0.03, -0.03, -0.01], [6]),
+        new KerasJS.Tensor([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0], [6])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.090109, -0.013664, 0.013763, -0.020054, -0.023944, -0.006931])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+
+  /*********************************************************
+  * ThresholdedReLU
+  *********************************************************/
+
+  describe('ThresholdedReLU', function () {
+    before(function () {
+      console.log('\n%cThresholdedReLU', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%theta=0.9', styles.h3)
+      let testLayer = new layers.ThresholdedReLU(0.9)
+      let t = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.0, 0.0, 0.0, 0.0, 1.0, 2.0])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+
+  /*********************************************************
+  * SReLU
+  *********************************************************/
+
+  describe('SReLU', function () {
+    before(function () {
+      console.log('\n%cSReLU', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%cweights: t_left, a_left, t_right, a_right', styles.h3)
+      let testLayer = new layers.SReLU()
+      testLayer.setWeights([
+        new KerasJS.Tensor([0.13, -0.02, 0.02, -0.03, -0.03, -0.01], [6]),
+        new KerasJS.Tensor([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0], [6]),
+        new KerasJS.Tensor([-0.9, 0.8, 0.0, -1.0, 0.7, 0.4], [6]),
+        new KerasJS.Tensor([0.1, 0.2, 0.3, 0.0, 0.5, -0.2], [6])
+      ])
+      let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.1339, 0.2, 0.0096, -0.065, 0.835, 0.068])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
 })