From 97732ca362e5820bf6fba7a9754959ba603100a5 Mon Sep 17 00:00:00 2001
From: Leon Chen <leon@md.ai>
Date: Thu, 25 Aug 2016 23:07:31 -0400
Subject: [PATCH] get convolution2d working on gpu

---
 index.html                                    |   4 +-
 .../Convolution2D.ipynb}                      | 139 +++++++-----------
 src/layers/convolutional/Convolution2D.js     |  28 +++-
 test/convolutional/Convolution2D.js           | 120 +++++++++++++++
 test/convolutional/convolutional.js           |  87 -----------
 ...convolutional.js => data_Convolution2D.js} |  20 +++
 6 files changed, 212 insertions(+), 186 deletions(-)
 rename notebooks/{convolutional.ipynb => convolutional/Convolution2D.ipynb} (90%)
 create mode 100644 test/convolutional/Convolution2D.js
 delete mode 100644 test/convolutional/convolutional.js
 rename test/convolutional/{data_convolutional.js => data_Convolution2D.js} (93%)

diff --git a/index.html b/index.html
index 8a30eb0..717099e 100644
--- a/index.html
+++ b/index.html
@@ -27,8 +27,8 @@
   <script src="/test/core/data_core.js"></script>
   <script src="/test/core/core.js"></script>
 
-  <script src="/test/convolutional/data_convolutional.js"></script>
-  <script src="/test/convolutional/convolutional.js"></script>
+  <script src="/test/convolutional/data_Convolution2D.js"></script>
+  <script src="/test/convolutional/Convolution2D.js"></script>
 
   <script>
     // mocha.checkLeaks();
diff --git a/notebooks/convolutional.ipynb b/notebooks/convolutional/Convolution2D.ipynb
similarity index 90%
rename from notebooks/convolutional.ipynb
rename to notebooks/convolutional/Convolution2D.ipynb
index 4410417..f355d95 100644
--- a/notebooks/convolutional.ipynb
+++ b/notebooks/convolutional/Convolution2D.ipynb
@@ -35,13 +35,6 @@
     "    return [round(x * 10**places) / 10**places for x in arr]"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Convolution1D"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -344,7 +337,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "**[convolutional.Convolution2D.5] 4 3x3 filters on 5x5x2 input, activation='relu', border_mode='same', subsample=(2,2), dim_ordering='tf', bias=True**"
+    "**[convolutional.Convolution2D.5] 4 3x3 filters on 4x4x2 input, activation='relu', border_mode='same', subsample=(2,2), dim_ordering='tf', bias=True**"
    ]
   },
   {
@@ -403,91 +396,59 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Deconvolution2D"
+    "**[convolutional.Convolution2D.6] 4 3x3 filters on 6x3x1 input, activation='relu', border_mode='same', subsample=(3,2), dim_ordering='tf', bias=True**"
    ]
   },
   {
-   "cell_type": "markdown",
-   "metadata": {},
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "W shape: (3, 3, 1, 4)\n",
+      "W: [-0.98287, 0.908613, -0.10087, 0.220438, 0.70412, 0.47762, 0.948863, 0.714924, 0.710821, 0.719052, -0.295442, -0.575936, 0.560889, 0.798823, -0.904521, -0.725105, -0.647339, -0.926505, 0.020986, 0.534159, -0.030523, -0.580772, -0.162302, 0.128018, -0.216971, 0.026925, 0.392296, 0.991634, 0.214941, 0.639493, -0.574398, -0.430476, -0.117988, -0.261311, -0.808084, -0.466051]\n",
+      "b shape: (4,)\n",
+      "b: [-0.98287, 0.908613, -0.10087, 0.220438]\n",
+      "\n",
+      "in shape: (6, 3, 1)\n",
+      "in: [0.70412, 0.47762, 0.948863, 0.714924, 0.710821, 0.719052, -0.295442, -0.575936, 0.560889, 0.798823, -0.904521, -0.725105, -0.647339, -0.926505, 0.020986, 0.534159, -0.030523, -0.580772]\n",
+      "out shape: (2, 2, 4)\n",
+      "out: [0.0, 0.474708, 0.960876, 1.317228, 0.0, 2.040575, 0.0, 0.060188, 0.0, 2.127169, 0.77897, 0.632372, 0.0, 0.0, 0.462447, 0.405417]\n"
+     ]
+    }
+   ],
    "source": [
-    "### AtrousConvolution2D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### SeparableConvolution2D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Convolution3D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### UpSampling1D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### UpSampling2D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### UpSampling3D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### ZeroPadding1D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### ZeroPadding2D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### ZeroPadding3D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Cropping1D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Cropping2D"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Cropping3D"
+    "conv = Convolution2D(4, 3, 3, activation='relu', border_mode='same', \n",
+    "                     subsample=(3, 2), dim_ordering='tf', bias=True)\n",
+    "\n",
+    "layer_0 = Input(shape=(6, 3, 1))\n",
+    "layer_1 = conv(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for w in model.get_weights():\n",
+    "    np.random.seed(106)\n",
+    "    weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('W shape:', weights[0].shape)\n",
+    "print('W:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('b shape:', weights[1].shape)\n",
+    "print('b:', format_decimal(weights[1].ravel().tolist()))\n",
+    "\n",
+    "data_in_shape = (6, 3, 1)\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
    ]
   },
   {
diff --git a/src/layers/convolutional/Convolution2D.js b/src/layers/convolutional/Convolution2D.js
index 2f651eb..1df9ad8 100644
--- a/src/layers/convolutional/Convolution2D.js
+++ b/src/layers/convolutional/Convolution2D.js
@@ -125,7 +125,7 @@ export default class Convolution2D extends Layer {
     const nbPatches = outputRows * outputCols
     const patchLen = nbRow * nbCol * inputChannels
 
-    const imColsMat = new Tensor([], [patchLen, nbPatches])
+    const imColsMat = new Tensor([], [nbPatches, patchLen])
 
     let patch = new Tensor([], [patchLen])
     let n = 0
@@ -135,7 +135,7 @@ export default class Convolution2D extends Layer {
           x.tensor.hi(i + nbRow, j + nbCol, inputChannels).lo(i, j, 0)
         ))
         patch.replaceTensorData(patchData)
-        ops.assign(imColsMat.tensor.pick(null, n), patch.tensor)
+        ops.assign(imColsMat.tensor.pick(n, null), patch.tensor)
         n += 1
       }
     }
@@ -153,7 +153,7 @@ export default class Convolution2D extends Layer {
     const [nbFilter, nbRow, nbCol] = this.kernelShape
     const patchLen = nbRow * nbCol * inputChannels
 
-    const wRowsMat = new Tensor([], [nbFilter, patchLen])
+    const wRowsMat = new Tensor([], [patchLen, nbFilter])
 
     let patch = new Tensor([], [patchLen])
     for (let n = 0; n < nbFilter; n++) {
@@ -161,7 +161,7 @@ export default class Convolution2D extends Layer {
         this.weights.W.tensor.pick(null, null, null, n)
       ))
       patch.replaceTensorData(patchData)
-      ops.assign(wRowsMat.tensor.pick(n, null), patch.tensor)
+      ops.assign(wRowsMat.tensor.pick(null, n), patch.tensor)
     }
 
     return wRowsMat
@@ -183,19 +183,31 @@ export default class Convolution2D extends Layer {
     const outputRows = this.outputShape[0]
     const outputCols = this.outputShape[1]
     const nbPatches = outputRows * outputCols
-    const matMul = new Tensor([], [nbFilter, nbPatches])
+    const matMul = new Tensor([], [nbPatches, nbFilter])
     if (this.bias) {
       for (let n = 0; n < nbFilter; n++) {
-        ops.assigns(matMul.tensor.pick(n, null), this.weights.b.tensor.get(n))
+        ops.assigns(matMul.tensor.pick(null, n), this.weights.b.tensor.get(n))
       }
     }
-    gemm(matMul.tensor, wRowsMat.tensor, imColsMat.tensor, 1, 1)
+
+    if (x._useWeblas) {
+      const bias = this.bias
+        ? this.weights.b.tensor.data
+        : new Float32Array(wRowsMat.tensor.shape[1])
+      matMul.tensor.data = weblas.sgemm(
+        imColsMat.tensor.shape[0], wRowsMat.tensor.shape[1], imColsMat.tensor.shape[1], // M, N, K
+        1, imColsMat.tensor.data, wRowsMat.tensor.data, // alpha, A, B
+        1, bias // beta, C
+      )
+    } else {
+      gemm(matMul.tensor, imColsMat.tensor, wRowsMat.tensor, 1, 1)
+    }
 
     let output = new Tensor([], this.outputShape)
     let outputChannel = new Tensor([], [outputRows, outputCols])
     for (let n = 0; n < nbFilter; n++) {
       const outputChannelData = flattenDeep(unpack(
-        matMul.tensor.pick(n, null)
+        matMul.tensor.pick(null, n)
       ))
       outputChannel.replaceTensorData(outputChannelData)
       ops.assign(output.tensor.pick(null, null, n), outputChannel.tensor)
diff --git a/test/convolutional/Convolution2D.js b/test/convolutional/Convolution2D.js
new file mode 100644
index 0000000..c4fef0d
--- /dev/null
+++ b/test/convolutional/Convolution2D.js
@@ -0,0 +1,120 @@
+/* eslint-env browser, mocha */
+
+describe('convolutional layer: Convolution2D', function () {
+  const assert = chai.assert
+  const styles = testGlobals.styles
+  const logTime = testGlobals.logTime
+  const stringifyCondensed = testGlobals.stringifyCondensed
+  const approxEquals = KerasJS.testUtils.approxEquals
+  const layers = KerasJS.layers
+
+  const testParams = [
+    {
+      inputShape: [5, 5, 2],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'linear', borderMode: 'valid', subsample: [1, 1], dimOrdering: 'tf', bias: true }
+    },
+    {
+      inputShape: [5, 5, 2],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'linear', borderMode: 'valid', subsample: [1, 1], dimOrdering: 'tf', bias: false }
+    },
+    {
+      inputShape: [5, 5, 2],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'relu', borderMode: 'valid', subsample: [2, 2], dimOrdering: 'tf', bias: true }
+    },
+    {
+      inputShape: [7, 7, 3],
+      kernelShape: [5, 4, 4],
+      attrs: { activation: 'relu', borderMode: 'valid', subsample: [2, 1], dimOrdering: 'tf', bias: true }
+    },
+    {
+      inputShape: [5, 5, 2],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'relu', borderMode: 'same', subsample: [1, 1], dimOrdering: 'tf', bias: true }
+    },
+    {
+      inputShape: [4, 4, 2],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'relu', borderMode: 'same', subsample: [2, 2], dimOrdering: 'tf', bias: true }
+    },
+    {
+      inputShape: [6, 3, 1],
+      kernelShape: [4, 3, 3],
+      attrs: { activation: 'relu', borderMode: 'same', subsample: [3, 2], dimOrdering: 'tf', bias: true }
+    }
+  ]
+
+  before(function () {
+    console.log('\n%cconvolutional layer: Convolution2D', styles.h1)
+  })
+
+  /*********************************************************
+  * CPU
+  *********************************************************/
+
+  describe('CPU', function () {
+    before(function () {
+      console.log('\n%cCPU', styles.h2)
+    })
+
+    testParams.forEach(({ inputShape, kernelShape, attrs }, i) => {
+      const key = `convolutional.Convolution2D.${i}`
+      const [inputRows, inputCols, inputChannels] = inputShape
+      const [nbFilter, nbRow, nbCol] = kernelShape
+      const title = `[${key}] [CPU] test 1: ${nbFilter} ${nbRow}x${nbCol} filters on ${inputRows}x${inputCols}x${inputChannels} input, activation='${attrs.activation}', border_mode='${attrs.borderMode}', subsample=${attrs.subsample}, dim_ordering='${attrs.dimOrdering}', bias=${attrs.bias}`
+
+      it(title, function () {
+        console.log(`\n%c${title}`, styles.h3)
+        let testLayer = new layers.Convolution2D(nbFilter, nbRow, nbCol, attrs)
+        testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
+        let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
+        console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+        const startTime = performance.now()
+        t = testLayer.call(t)
+        const endTime = performance.now()
+        console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+        logTime(startTime, endTime)
+        const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
+        const shapeExpected = TEST_DATA[key].expected.shape
+        assert.deepEqual(t.tensor.shape, shapeExpected)
+        assert.isTrue(approxEquals(t.tensor, dataExpected))
+      })
+    })
+  })
+
+  /*********************************************************
+  * GPU
+  *********************************************************/
+
+  describe('GPU', function () {
+    before(function () {
+      console.log('\n%cGPU', styles.h2)
+    })
+
+    testParams.forEach(({ inputShape, kernelShape, attrs }, i) => {
+      const key = `convolutional.Convolution2D.${i}`
+      const [inputRows, inputCols, inputChannels] = inputShape
+      const [nbFilter, nbRow, nbCol] = kernelShape
+      const title = `[${key}] [GPU] test 1: ${nbFilter} ${nbRow}x${nbCol} filters on ${inputRows}x${inputCols}x${inputChannels} input, activation='${attrs.activation}', border_mode='${attrs.borderMode}', subsample=${attrs.subsample}, dim_ordering='${attrs.dimOrdering}', bias=${attrs.bias}`
+
+      it(title, function () {
+        console.log(`\n%c${title}`, styles.h3)
+        let testLayer = new layers.Convolution2D(nbFilter, nbRow, nbCol, attrs)
+        testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
+        let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape, { useWeblas: true })
+        console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+        const startTime = performance.now()
+        t = testLayer.call(t)
+        const endTime = performance.now()
+        console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+        logTime(startTime, endTime)
+        const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
+        const shapeExpected = TEST_DATA[key].expected.shape
+        assert.deepEqual(t.tensor.shape, shapeExpected)
+        assert.isTrue(approxEquals(t.tensor, dataExpected))
+      })
+    })
+  })
+})
diff --git a/test/convolutional/convolutional.js b/test/convolutional/convolutional.js
deleted file mode 100644
index 3cf88c9..0000000
--- a/test/convolutional/convolutional.js
+++ /dev/null
@@ -1,87 +0,0 @@
-/* eslint-env browser, mocha */
-
-describe('Layers: Convolutional', function () {
-  const assert = chai.assert
-  const styles = testGlobals.styles
-  const logTime = testGlobals.logTime
-  const stringifyCondensed = testGlobals.stringifyCondensed
-  const approxEquals = KerasJS.testUtils.approxEquals
-  const layers = KerasJS.layers
-
-  before(function () {
-    console.log('\n%cLayers: Convolutional', styles.h1)
-  })
-
-  /*********************************************************
-  * Convolution2D
-  *********************************************************/
-
-  describe('Convolution2D', function () {
-    before(function () {
-      console.log('\n%cConvolution2D', styles.h2)
-    })
-
-    const testParams = [
-      {
-        mode: 'CPU',
-        inputShape: [5, 5, 2],
-        kernelShape: [4, 3, 3],
-        attrs: { activation: 'linear', borderMode: 'valid', subsample: [1, 1], dimOrdering: 'tf', bias: true }
-      },
-      {
-        mode: 'CPU',
-        inputShape: [5, 5, 2],
-        kernelShape: [4, 3, 3],
-        attrs: { activation: 'linear', borderMode: 'valid', subsample: [1, 1], dimOrdering: 'tf', bias: false }
-      },
-      {
-        mode: 'CPU',
-        inputShape: [5, 5, 2],
-        kernelShape: [4, 3, 3],
-        attrs: { activation: 'relu', borderMode: 'valid', subsample: [2, 2], dimOrdering: 'tf', bias: true }
-      },
-      {
-        mode: 'CPU',
-        inputShape: [7, 7, 3],
-        kernelShape: [5, 4, 4],
-        attrs: { activation: 'relu', borderMode: 'valid', subsample: [2, 1], dimOrdering: 'tf', bias: true }
-      },
-      {
-        mode: 'CPU',
-        inputShape: [5, 5, 2],
-        kernelShape: [4, 3, 3],
-        attrs: { activation: 'relu', borderMode: 'same', subsample: [1, 1], dimOrdering: 'tf', bias: true }
-      },
-      {
-        mode: 'CPU',
-        inputShape: [4, 4, 2],
-        kernelShape: [4, 3, 3],
-        attrs: { activation: 'relu', borderMode: 'same', subsample: [2, 2], dimOrdering: 'tf', bias: true }
-      }
-    ]
-
-    testParams.forEach(({ mode, inputShape, kernelShape, attrs }, i) => {
-      const key = `convolutional.Convolution2D.${i}`
-      const [inputRows, inputCols, inputChannels] = inputShape
-      const [nbFilter, nbRow, nbCol] = kernelShape
-      const title = `[${key}] [CPU] test 1: ${nbFilter} ${nbRow}x${nbCol} filters on ${inputRows}x${inputCols}x${inputChannels} input, activation='${attrs.activation}', border_mode='${attrs.borderMode}', subsample=${attrs.subsample}, dim_ordering='${attrs.dimOrdering}', bias=${attrs.bias}`
-
-      it(title, function () {
-        console.log(`\n%c${title}`, styles.h3)
-        let testLayer = new layers.Convolution2D(nbFilter, nbRow, nbCol, attrs)
-        testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
-        let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
-        console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
-        const startTime = performance.now()
-        t = testLayer.call(t)
-        const endTime = performance.now()
-        console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
-        logTime(startTime, endTime)
-        const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
-        const shapeExpected = TEST_DATA[key].expected.shape
-        assert.deepEqual(t.tensor.shape, shapeExpected)
-        assert.isTrue(approxEquals(t.tensor, dataExpected))
-      })
-    })
-  })
-})
diff --git a/test/convolutional/data_convolutional.js b/test/convolutional/data_Convolution2D.js
similarity index 93%
rename from test/convolutional/data_convolutional.js
rename to test/convolutional/data_Convolution2D.js
index 95e1f01..46a0461 100644
--- a/test/convolutional/data_convolutional.js
+++ b/test/convolutional/data_Convolution2D.js
@@ -119,6 +119,26 @@
         data: [0.0, 0.565042, 0.12593, 0.0, 0.0, 0.320489, 0.979856, 0.0, 0.0, 0.0, 0.799525, 0.0, 0.0, 0.0, 0.637126, 0.0],
         shape: [2, 2, 4]
       }
+    },
+    'convolutional.Convolution2D.6': {
+      input: {
+        data: [0.70412, 0.47762, 0.948863, 0.714924, 0.710821, 0.719052, -0.295442, -0.575936, 0.560889, 0.798823, -0.904521, -0.725105, -0.647339, -0.926505, 0.020986, 0.534159, -0.030523, -0.580772],
+        shape: [6, 3, 1]
+      },
+      weights: [
+        {
+          data: [-0.98287, 0.908613, -0.10087, 0.220438, 0.70412, 0.47762, 0.948863, 0.714924, 0.710821, 0.719052, -0.295442, -0.575936, 0.560889, 0.798823, -0.904521, -0.725105, -0.647339, -0.926505, 0.020986, 0.534159, -0.030523, -0.580772, -0.162302, 0.128018, -0.216971, 0.026925, 0.392296, 0.991634, 0.214941, 0.639493, -0.574398, -0.430476, -0.117988, -0.261311, -0.808084, -0.466051],
+          shape: [3, 3, 1, 4]
+        },
+        {
+          data: [-0.98287, 0.908613, -0.10087, 0.220438],
+          shape: [4]
+        }
+      ],
+      expected: {
+        data: [0.0, 0.474708, 0.960876, 1.317228, 0.0, 2.040575, 0.0, 0.060188, 0.0, 2.127169, 0.77897, 0.632372, 0.0, 0.0, 0.462447, 0.405417],
+        shape: [2, 2, 4]
+      }
     }
   }