From c3eec55d0cf3b8ef2697177b9d2d32d5e9eed551 Mon Sep 17 00:00:00 2001
From: Leon Chen <leon@md.ai>
Date: Tue, 23 Aug 2016 01:49:43 -0400
Subject: [PATCH] finish merge layer and start advanced activations layers

---
 index.html                          |   1 +
 package.json                        |   2 +
 src/layers/advanced_activations.js  |  26 +++
 src/layers/core.js                  |  62 +++++-
 src/layers/index.js                 |  18 +-
 src/test-utils.js                   |   2 +-
 test/layers/advanced_activations.js |  40 ++++
 test/layers/core.js                 | 293 ++++++++++++++++++++++++++++
 test/test-utils.js                  |  34 ----
 9 files changed, 424 insertions(+), 54 deletions(-)
 create mode 100644 src/layers/advanced_activations.js
 create mode 100644 test/layers/advanced_activations.js
 delete mode 100644 test/test-utils.js

diff --git a/index.html b/index.html
index 7680862..2eb6e93 100644
--- a/index.html
+++ b/index.html
@@ -19,6 +19,7 @@
   <script src="/test/globals.js"></script>
   <script src="/test/activations.js"></script>
   <script src="/test/layers/core.js"></script>
+  <script src="/test/layers/advanced_activations.js"></script>
   <script>
     // mocha.checkLeaks();
     mocha.globals(['jQuery']);
diff --git a/package.json b/package.json
index 0eab5e2..bc6e1e3 100644
--- a/package.json
+++ b/package.json
@@ -35,6 +35,8 @@
     "lodash": "^4.15.0",
     "ndarray": "^1.0.18",
     "ndarray-blas-level2": "^1.1.0",
+    "ndarray-concat-rows": "^1.0.1",
+    "ndarray-gemm": "^1.0.0",
     "ndarray-ops": "^1.2.2",
     "ndarray-squeeze": "^1.0.2",
     "ndarray-tile": "^1.0.3",
diff --git a/src/layers/advanced_activations.js b/src/layers/advanced_activations.js
new file mode 100644
index 0000000..a9cc820
--- /dev/null
+++ b/src/layers/advanced_activations.js
@@ -0,0 +1,26 @@
+import { Layer } from '../engine/topology'
+import { relu } from '../activations'
+
+/**
+* LeakyReLU advanced activation layer class
+*/
+export class LeakyReLU extends Layer {
+  /**
+  * Creates a LeakyReLU activation layer
+  * @param {number} alpha - negative slope coefficient
+  */
+  constructor (alpha = 0.3) {
+    super({})
+    this.alpha = alpha
+  }
+
+  /**
+  * Method for layer computational logic
+  * @param {Tensor} x
+  * @returns {Tensor} x
+  */
+  call = x => {
+    relu(x, { alpha: this.alpha })
+    return x
+  }
+}
diff --git a/src/layers/core.js b/src/layers/core.js
index d9b4449..e93ac20 100644
--- a/src/layers/core.js
+++ b/src/layers/core.js
@@ -3,10 +3,12 @@ import Tensor from '../tensor'
 import { Layer } from '../engine/topology'
 import ndarray from 'ndarray'
 import { gemv } from 'ndarray-blas-level2'
+import gemm from 'ndarray-gemm'
 import ops from 'ndarray-ops'
 import unpack from 'ndarray-unpack'
 import unsqueeze from 'ndarray-unsqueeze'
 import tile from 'ndarray-tile'
+import concatFirstAxis from 'ndarray-concat-rows'
 import flattenDeep from 'lodash/flattenDeep'
 import isEqual from 'lodash/isEqual'
 import isInteger from 'lodash/isInteger'
@@ -300,7 +302,7 @@ export class Merge extends Layer {
   * @returns {boolean} valid
   */
   _validateInputs = inputs => {
-    const shapes = inputs.map(x => x.tensor.shape)
+    const shapes = inputs.map(x => x.tensor.shape.slice())
     if (['sum', 'mul', 'ave', 'cos', 'max'].indexOf(this.mode) > -1) {
       if (!shapes.every(shape => isEqual(shape, shapes[0]))) {
         throw new Error(`${this.name} [Merge layer] All input shapes must be the same for mode ${this.mode}.`)
@@ -314,7 +316,7 @@ export class Merge extends Layer {
         if (this.dotAxes < 0) {
           this.dotAxes = [shapes[0].length + this.dotAxes, shapes[1].length + this.dotAxes]
         } else {
-          this.dotAxes = [this.dotAxes, this.dotAxes]
+          this.dotAxes = [this.dotAxes - 1, this.dotAxes - 1]
         }
       }
       if (shapes[0][this.dotAxes[0]] !== shapes[1][this.dotAxes[1]]) {
@@ -344,15 +346,15 @@ export class Merge extends Layer {
     let output
     let outputShape
     if (['sum', 'mul', 'ave', 'max'].indexOf(this.mode) > -1) {
-      outputShape = inputs[0].tensor.shape
+      outputShape = inputs[0].tensor.shape.slice()
       output = new Tensor([], outputShape)
     } else if (this.mode === 'concat') {
-      outputShape = inputs[0].tensor.shape
+      outputShape = inputs[0].tensor.shape.slice()
       const _concatAxis = this.concatAxis < 0
         ? outputShape.length + this.concatAxis
-        : this.concatAxis
+        : this.concatAxis - 1
       inputs.slice(1, inputs.length).forEach(x => {
-        const d = x.tensor.shape.slice(_concatAxis)[0]
+        const d = x.tensor.shape.slice()[_concatAxis]
         outputShape[_concatAxis] += d
       })
       output = new Tensor([], outputShape)
@@ -361,7 +363,6 @@ export class Merge extends Layer {
       let shape2 = inputs[1].tensor.shape.slice()
       shape1.splice(this.dotAxes[0], 1)
       shape2.splice(this.dotAxes[1], 1)
-      shape2.splice(0, 1)
       outputShape = shape1.concat(shape2)
       if (outputShape.length === 1) {
         outputShape.push(1)
@@ -384,13 +385,56 @@ export class Merge extends Layer {
       }
       ops.divseq(output.tensor, inputs.length)
     } else if (this.mode === 'max') {
-      ops.assigns(output.tensor, inputs[0].tensor)
+      ops.assign(output.tensor, inputs[0].tensor)
       for (let i = 1; i < inputs.length; i++) {
         ops.maxeq(output.tensor, inputs[i].tensor)
       }
     } else if (this.mode === 'concat') {
-    } else if (this.mode === 'cos') {
+      const _concatAxis = this.concatAxis < 0
+        ? inputs[0].tensor.shape.length + this.concatAxis
+        : this.concatAxis - 1
+      if (_concatAxis === 0) {
+        concatFirstAxis(output.tensor, inputs.map(x => x.tensor))
+      } else {
+        let dimsAxisSwap = [_concatAxis]
+        for (let i = 0; i < inputs[0].tensor.shape.length; i++) {
+          if (i !== _concatAxis) dimsAxisSwap.push(i)
+        }
+        concatFirstAxis(
+          output.tensor.transpose(...dimsAxisSwap),
+          inputs.map(x => x.tensor.transpose(...dimsAxisSwap))
+        )
+      }
     } else if (this.mode === 'dot') {
+      if (inputs[0].tensor.shape.length === 2 && inputs[1].tensor.shape.length === 2) {
+        if (this.dotAxes[0] === 0 && this.dotAxes[1] === 0) {
+          gemm(output.tensor, inputs[0].tensor.transpose(1, 0), inputs[1].tensor)
+        } else if (this.dotAxes[0] === 1 && this.dotAxes[1] === 1) {
+          gemm(output.tensor, inputs[0].tensor, inputs[1].tensor.transpose(1, 0))
+        }
+      } else {
+        throw new Error(`${this.name} [Merge layer] dot mode for 3+ dim tensors not yet implemented.`)
+      }
+    } else if (this.mode === 'cos') {
+      if (inputs[0].tensor.shape.length === 2 && inputs[1].tensor.shape.length === 2) {
+        let a = new Tensor([], output.tensor.shape)
+        let b = new Tensor([], output.tensor.shape)
+        if (this.dotAxes[0] === 0 && this.dotAxes[1] === 0) {
+          gemm(a.tensor, inputs[0].tensor.transpose(1, 0), inputs[0].tensor)
+          gemm(b.tensor, inputs[1].tensor.transpose(1, 0), inputs[1].tensor)
+          gemm(output.tensor, inputs[0].tensor.transpose(1, 0), inputs[1].tensor)
+        } else if (this.dotAxes[0] === 1 && this.dotAxes[1] === 1) {
+          gemm(a.tensor, inputs[0].tensor, inputs[0].tensor.transpose(1, 0))
+          gemm(b.tensor, inputs[1].tensor, inputs[1].tensor.transpose(1, 0))
+          gemm(output.tensor, inputs[0].tensor, inputs[1].tensor.transpose(1, 0))
+        }
+        ops.muleq(a.tensor, b.tensor)
+        ops.sqrteq(a.tensor)
+        ops.diveq(output.tensor, a.tensor)
+        output.tensor = unsqueeze(output.tensor, 0)
+      } else {
+        throw new Error(`${this.name} [Merge layer] cos mode for 3+ dim tensors not yet implemented.`)
+      }
     }
 
     return output
diff --git a/src/layers/index.js b/src/layers/index.js
index eb97e8f..4dc8fd7 100644
--- a/src/layers/index.js
+++ b/src/layers/index.js
@@ -1,4 +1,4 @@
-import {
+export {
   Dense,
   Activation,
   Dropout,
@@ -10,12 +10,10 @@ import {
 } from './core'
 
 export {
-  Dense,
-  Activation,
-  Dropout,
-  Flatten,
-  Reshape,
-  Permute,
-  RepeatVector,
-  Merge
-}
+  LeakyReLU,
+  PReLU,
+  ELU,
+  ParametricSoftplus,
+  ThresholdedReLU,
+  SReLU
+} from './advanced_activations'
diff --git a/src/test-utils.js b/src/test-utils.js
index a9a2449..c93fdab 100644
--- a/src/test-utils.js
+++ b/src/test-utils.js
@@ -8,7 +8,7 @@ import isFinite from 'lodash/isFinite'
 * stride/offset prevents us from comparing the array data
 * element-wise directly.
 */
-export function approxEquals (ndarrayOut, dataExpected, tol = 1e-6) {
+export function approxEquals (ndarrayOut, dataExpected, tol = 1e-5) {
   const a = flattenDeep(unpack(ndarrayOut))
   const b = dataExpected
   if (a.length !== b.length) return false
diff --git a/test/layers/advanced_activations.js b/test/layers/advanced_activations.js
new file mode 100644
index 0000000..dfdf4c6
--- /dev/null
+++ b/test/layers/advanced_activations.js
@@ -0,0 +1,40 @@
+/* eslint-env browser, mocha */
+
+describe('Layers: Advanced Activations', function () {
+  const assert = chai.assert
+  const styles = testGlobals.styles
+  const logTime = testGlobals.logTime
+  const stringifyCondensed = testGlobals.stringifyCondensed
+  const approxEquals = KerasJS.testUtils.approxEquals
+  const layers = KerasJS.layers
+
+  before(function () {
+    console.log('\n%Layers: Advanced Activations', styles.h1)
+  })
+
+  /*********************************************************
+  * LeakyReLU
+  *********************************************************/
+
+  describe('LeakyReLU', function () {
+    before(function () {
+      console.log('\n%cLeakyReLU', styles.h2)
+    })
+
+    it('should produce expected values', function () {
+      console.log('\n%calpha=0.4', styles.h3)
+      let testLayer = new layers.LeakyReLU(0.4)
+      let t = new KerasJS.Tensor([0, 0.2, -0.5, -0.1, 1, 2], [6])
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([0.0, 0.2, -0.2, -0.04, 1.0, 2.0])
+      const shapeExpected = [6]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+})
diff --git a/test/layers/core.js b/test/layers/core.js
index 1ed89a2..087d126 100644
--- a/test/layers/core.js
+++ b/test/layers/core.js
@@ -534,5 +534,298 @@ describe('Layers: Core', function () {
       assert.deepEqual(t2.tensor.shape, shapeExpected)
       assert.isTrue(approxEquals(t2.tensor, dataExpected))
     })
+
+    it('should produce expected values in concat mode (1D)', function () {
+      console.log('\n%cmode: concat (1D)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2 = new layers.Merge({ mode: 'concat', concatAxis: -1 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      tb = testLayer1b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer2.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([7.3, -0.21, -2.45, 4.48])
+      const shapeExpected = [4]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in concat mode (2D, concatAxis=-1)', function () {
+      console.log('\n%cmode: concat (2D, concatAxis=-1)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'concat', concatAxis: -1 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48])
+      const shapeExpected = [3, 4]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in concat mode (2D, concatAxis=-2)', function () {
+      console.log('\n%cmode: concat (2D, concatAxis=-2)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'concat', concatAxis: -2 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([7.3, -0.21, 7.3, -0.21, 7.3, -0.21, -2.45, 4.48, -2.45, 4.48, -2.45, 4.48])
+      const shapeExpected = [6, 2]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in concat mode (2D, concatAxis=1)', function () {
+      console.log('\n%cmode: concat (2D, concatAxis=1)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'concat', concatAxis: 1 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([7.3, -0.21, 7.3, -0.21, 7.3, -0.21, -2.45, 4.48, -2.45, 4.48, -2.45, 4.48])
+      const shapeExpected = [6, 2]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in concat mode (2D, concatAxis=2)', function () {
+      console.log('\n%cmode: concat (2D, concatAxis=2)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'concat', concatAxis: 2 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48])
+      const shapeExpected = [3, 4]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in dot mode (2D x 2D, dotAxes=1)', function () {
+      console.log('\n%cmode: dot (2D x 2D, dotAxes=1)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'dot', dotAxes: 1 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([-53.655003, 98.112007, 1.5435, -2.8224])
+      const shapeExpected = [2, 2]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in dot mode (2D x 2D, dotAxes=2)', function () {
+      console.log('\n%cmode: dot (2D x 2D, dotAxes=2)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'dot', dotAxes: 2 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([-18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258])
+      const shapeExpected = [3, 3]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in cos mode (2D x 2D, dotAxes=1)', function () {
+      console.log('\n%cmode: cos (2D x 2D, dotAxes=1)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'cos', dotAxes: 1 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([-1.0, 7.972744, 0.125427, -1.0])
+      const shapeExpected = [1, 2, 2]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+
+    it('should produce expected values in cos mode (2D x 2D, dotAxes=2)', function () {
+      console.log('\n%cmode: cos (2D x 2D, dotAxes=2)', styles.h3)
+      let testLayer1a = new layers.Dense(2)
+      let testLayer2a = new layers.RepeatVector(3)
+      let testLayer1b = new layers.Dense(2)
+      let testLayer2b = new layers.RepeatVector(3)
+      let testLayer3 = new layers.Merge({ mode: 'cos', dotAxes: 2 })
+      testLayer1a.setWeights([
+        new KerasJS.Tensor([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0], [6, 2]),
+        new KerasJS.Tensor([0.5, 0.7], [2])
+      ])
+      testLayer1b.setWeights([
+        new KerasJS.Tensor([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3], [6, 2]),
+        new KerasJS.Tensor([0.1, -0.2], [2])
+      ])
+      let ta = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      let tb = new KerasJS.Tensor([0, 0.2, 0.5, -0.1, 1, 2], [6])
+      ta = testLayer1a.call(ta)
+      ta = testLayer2a.call(ta)
+      tb = testLayer1b.call(tb)
+      tb = testLayer2b.call(tb)
+      console.log('%cin', styles.h4, stringifyCondensed([ta.tensor, tb.tensor]))
+      const startTime = performance.now()
+      let t = testLayer3.call([ta, tb])
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array([-0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843])
+      const shapeExpected = [1, 3, 3]
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
   })
 })
diff --git a/test/test-utils.js b/test/test-utils.js
deleted file mode 100644
index cea2dc2..0000000
--- a/test/test-utils.js
+++ /dev/null
@@ -1,34 +0,0 @@
-(function () {
-  'use strict'
-
-  const styles = {
-    h1: 'color:#001f3f;font-weight:bold;font-size:160%;',
-    h2: 'color:#0074D9;font-weight:bold;font-size:130%;',
-    h3: 'color:#FF4136;font-weight:bold;font-size:110%;',
-    h4: 'color:#AAAAAA;font-size:100%;',
-    time: 'color:#2ECC40;font-weight:bold;font-size:100%;'
-  }
-
-  function approxEquals (a, b, tol = 1e-6) {
-    if (a.length !== b.length) return false
-    for (let i = 0; i < a.length; i++) {
-      if (
-        a[i] < (b[i] - tol) ||
-        a[i] > (b[i] + tol)
-      ) {
-        return false
-      }
-    }
-    return true
-  }
-
-  function logTime (startTime, endTime) {
-    console.log(`%c>>>> exec: ${Math.round(100 * (endTime - startTime)) / 100} ms`, styles.time)
-  }
-
-  window.testUtils = {
-    styles,
-    approxEquals,
-    logTime
-  }
-})()