From 9577495a172ea87a74c1a7fcd52c1b54289faa0b Mon Sep 17 00:00:00 2001
From: Leon Chen <leon@md.ai>
Date: Mon, 5 Sep 2016 22:52:23 -0400
Subject: [PATCH] implement BatchNormalization in all modes, with tests

---
 index.html                                    |   3 +
 .../normalization/BatchNormalization.ipynb    | 559 ++++++++++++++++++
 src/layers/index.js                           |   1 +
 .../normalization/BatchNormalization.js       | 124 ++++
 src/layers/normalization/index.js             |   5 +
 test/normalization/BatchNormalization.js      |  46 ++
 test/normalization/data_BatchNormalization.js | 118 ++++
 7 files changed, 856 insertions(+)
 create mode 100644 notebooks/normalization/BatchNormalization.ipynb
 create mode 100644 src/layers/normalization/BatchNormalization.js
 create mode 100644 src/layers/normalization/index.js
 create mode 100644 test/normalization/BatchNormalization.js
 create mode 100644 test/normalization/data_BatchNormalization.js

diff --git a/index.html b/index.html
index 7398423..20b66a3 100644
--- a/index.html
+++ b/index.html
@@ -94,6 +94,9 @@
   <script src="/test/pooling/data_GlobalAveragePooling2D.js"></script>
   <script src="/test/pooling/GlobalAveragePooling2D.js"></script>
 
+  <script src="/test/normalization/data_BatchNormalization.js"></script>
+  <script src="/test/normalization/BatchNormalization.js"></script>
+
   <script>
     // mocha.checkLeaks();
     mocha.globals(['jQuery']);
diff --git a/notebooks/normalization/BatchNormalization.ipynb b/notebooks/normalization/BatchNormalization.ipynb
new file mode 100644
index 0000000..4d75faf
--- /dev/null
+++ b/notebooks/normalization/BatchNormalization.ipynb
@@ -0,0 +1,559 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from keras.models import Model\n",
+    "from keras.layers import Input\n",
+    "from keras.layers.normalization import BatchNormalization\n",
+    "from keras import backend as K"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def format_decimal(arr, places=6):\n",
+    "    return [round(x * 10**places) / 10**places for x in arr]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### BatchNormalization"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.0] epsilon=1e-05, mode=0, axis=-1**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (3,)\n",
+      "gamma: [0.307179, -0.769986, 0.900566]\n",
+      "beta shape: (3,)\n",
+      "beta: [-0.387536, -0.469873, -0.60788]\n",
+      "running_mean shape: (3,)\n",
+      "running_mean: [-0.742023, -0.077688, -0.167692]\n",
+      "running_std shape: (3,)\n",
+      "running_std: [0.597806, 0.435934, 0.01687]\n",
+      "\n",
+      "in shape: (4, 3)\n",
+      "in: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266]\n",
+      "out shape: (4, 3)\n",
+      "out: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=-1)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1000 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.1] epsilon=1e-02, mode=0, axis=-1**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (3,)\n",
+      "gamma: [-0.211487, -0.648815, -0.854588]\n",
+      "beta shape: (3,)\n",
+      "beta: [0.311362, -0.228519, 0.253024]\n",
+      "running_mean shape: (3,)\n",
+      "running_mean: [-0.946541, 0.585593, -0.49527]\n",
+      "running_std shape: (3,)\n",
+      "running_std: [0.557039, 0.055171, 0.263987]\n",
+      "\n",
+      "in shape: (4, 3)\n",
+      "in: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129]\n",
+      "out shape: (4, 3)\n",
+      "out: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3)\n",
+    "norm = BatchNormalization(epsilon=1e-02, mode=0, axis=-1)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1010 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.2] epsilon=1e-05, mode=0, axis=1**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (4,)\n",
+      "gamma: [0.517906, -0.666537, 0.378665, -0.380062]\n",
+      "beta shape: (4,)\n",
+      "beta: [0.400557, 0.743871, 0.437134, -0.5548]\n",
+      "running_mean shape: (4,)\n",
+      "running_mean: [-0.726393, 0.961405, -0.352651, -0.616831]\n",
+      "running_std shape: (4,)\n",
+      "running_std: [0.91407, 0.630071, 0.508933, 0.052897]\n",
+      "\n",
+      "in shape: (4, 3, 2)\n",
+      "in: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617]\n",
+      "out shape: (4, 3, 2)\n",
+      "out: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3, 2)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=1)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1020 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.3] epsilon=1e-05, mode=0, axis=2**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (3,)\n",
+      "gamma: [-0.089873, -0.210305, -0.554444]\n",
+      "beta shape: (3,)\n",
+      "beta: [-0.997964, -0.487343, -0.350362]\n",
+      "running_mean shape: (3,)\n",
+      "running_mean: [-0.145437, 0.11872, -0.368563]\n",
+      "running_std shape: (3,)\n",
+      "running_std: [0.486933, 0.587355, 0.517041]\n",
+      "\n",
+      "in shape: (4, 3, 2)\n",
+      "in: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809]\n",
+      "out shape: (4, 3, 2)\n",
+      "out: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3, 2)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=2)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1030 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.4] epsilon=1e-05, mode=0, axis=3**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (2,)\n",
+      "gamma: [0.660544, -0.047716]\n",
+      "beta shape: (2,)\n",
+      "beta: [-0.956656, 0.127814]\n",
+      "running_mean shape: (2,)\n",
+      "running_mean: [-0.359373, -0.71013]\n",
+      "running_std shape: (2,)\n",
+      "running_std: [0.551285, 0.957558]\n",
+      "\n",
+      "in shape: (4, 3, 2)\n",
+      "in: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951]\n",
+      "out shape: (4, 3, 2)\n",
+      "out: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3, 2)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=3)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1040 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.5] epsilon=1e-05, mode=1, axis=-1**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (2,)\n",
+      "gamma: [0.833294, -0.732828]\n",
+      "beta shape: (2,)\n",
+      "beta: [0.013718, -0.842599]\n",
+      "running_mean shape: (2,)\n",
+      "running_mean: [0.331031, -0.238168]\n",
+      "running_std shape: (2,)\n",
+      "running_std: [0.644294, 0.114097]\n",
+      "\n",
+      "in shape: (4, 3, 2)\n",
+      "in: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567]\n",
+      "out shape: (4, 3, 2)\n",
+      "out: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3, 2)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=1, axis=-1)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1050 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "**[normalization.BatchNormalization.6] epsilon=1e-05, mode=2, axis=-1**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gamma shape: (2,)\n",
+      "gamma: [0.128705, -0.956067]\n",
+      "beta shape: (2,)\n",
+      "beta: [-0.168081, -0.9895]\n",
+      "running_mean shape: (2,)\n",
+      "running_mean: [0.188775, -0.258915]\n",
+      "running_std shape: (2,)\n",
+      "running_std: [0.318404, 0.231636]\n",
+      "\n",
+      "in shape: (4, 3, 2)\n",
+      "in: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054]\n",
+      "out shape: (4, 3, 2)\n",
+      "out: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298]\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_in_shape = (4, 3, 2)\n",
+    "norm = BatchNormalization(epsilon=1e-05, mode=2, axis=-1)\n",
+    "\n",
+    "layer_0 = Input(shape=data_in_shape)\n",
+    "layer_1 = norm(layer_0)\n",
+    "model = Model(input=layer_0, output=layer_1)\n",
+    "\n",
+    "# set weights to random (use seed for reproducibility)\n",
+    "weights = []\n",
+    "for i, w in enumerate(model.get_weights()):\n",
+    "    np.random.seed(1060 + i)\n",
+    "    if i == 3:\n",
+    "        # std should be positive\n",
+    "        weights.append(np.random.random(w.shape))\n",
+    "    else:\n",
+    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
+    "model.set_weights(weights)\n",
+    "print('gamma shape:', weights[0].shape)\n",
+    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
+    "print('beta shape:', weights[1].shape)\n",
+    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
+    "print('running_mean shape:', weights[2].shape)\n",
+    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
+    "print('running_std shape:', weights[3].shape)\n",
+    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
+    "\n",
+    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
+    "print('')\n",
+    "print('in shape:', data_in_shape)\n",
+    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
+    "result = model.predict(np.array([data_in]))\n",
+    "print('out shape:', result[0].shape)\n",
+    "print('out:', format_decimal(result[0].ravel().tolist()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python [default]",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/src/layers/index.js b/src/layers/index.js
index 39c4838..acbbfc8 100644
--- a/src/layers/index.js
+++ b/src/layers/index.js
@@ -2,3 +2,4 @@ export * from './advanced_activations'
 export * from './core'
 export * from './convolutional'
 export * from './pooling'
+export * from './normalization'
diff --git a/src/layers/normalization/BatchNormalization.js b/src/layers/normalization/BatchNormalization.js
new file mode 100644
index 0000000..c1d472b
--- /dev/null
+++ b/src/layers/normalization/BatchNormalization.js
@@ -0,0 +1,124 @@
+import Layer from '../../engine/Layer'
+import Tensor from '../../Tensor'
+import ops from 'ndarray-ops'
+import unpack from 'ndarray-unpack'
+import flattenDeep from 'lodash/flattenDeep'
+
+/**
+ * BatchNormalization layer class
+ */
+export default class BatchNormalization extends Layer {
+  /**
+   * Creates an BatchNormalization layer
+   */
+  constructor (attrs = {}) {
+    super(attrs)
+    const {
+      epsilon = 1e-5,
+      mode = 0,
+      axis = -1
+    } = attrs
+
+    this.epsilon = epsilon
+    this.mode = mode
+    this.axis = axis
+
+    // Layer weights specification
+    // running mean and std are non_trainable_weights in mode 0
+    this.params = this.mode === 0
+      ? ['gamma', 'beta', 'running_mean', 'running_std']
+      : ['gamma', 'beta']
+  }
+
+  /**
+   * Method for layer computational logic
+   * @param {Tensor} x
+   * @returns {Tensor} x
+   */
+  call (x) {
+    // no batch axis, so axis is less 1 compared to representation in keras
+    this.axis = this.axis < 0 ? x.tensor.shape.length + this.axis : this.axis - 1
+
+    let broadcast = []
+    for (let d = 0; d < x.tensor.shape.length; d++) {
+      if (d === this.axis) broadcast.push(1)
+      else broadcast.push(null)
+    }
+
+    // broadcast weights
+    let _gamma = new Tensor([], x.tensor.shape)
+    let _beta = new Tensor([], x.tensor.shape)
+    for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+      broadcast[this.axis] = i
+      ops.assigns(_gamma.tensor.pick(...broadcast), this.weights.gamma.tensor.get(i))
+      ops.assigns(_beta.tensor.pick(...broadcast), this.weights.beta.tensor.get(i))
+    }
+
+    let _mean = new Tensor([], x.tensor.shape)
+    let _std = new Tensor([], x.tensor.shape)
+
+    if (this.mode === 0) {
+      // feature-wise normalization
+      for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+        broadcast[this.axis] = i
+        ops.assigns(_mean.tensor.pick(...broadcast), this.weights.running_mean.tensor.get(i))
+        ops.assigns(_std.tensor.pick(...broadcast), this.weights.running_std.tensor.get(i) + this.epsilon)
+      }
+      ops.sqrteq(_std.tensor)
+    } else if (this.mode === 1) {
+      // sample-wise normalization
+      let reducedShape = x.tensor.shape.slice()
+      reducedShape.splice(this.axis, 1)
+
+      // mean
+      let sampleMean = new Tensor([], reducedShape)
+      for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+        broadcast[this.axis] = i
+        ops.addeq(sampleMean.tensor, x.tensor.pick(...broadcast))
+      }
+      ops.divseq(sampleMean.tensor, x.tensor.shape[this.axis])
+
+      // stddev
+      let sampleStd = new Tensor([], reducedShape)
+      let stdTemp = new Tensor([], reducedShape)
+      for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+        broadcast[this.axis] = i
+        ops.sub(stdTemp.tensor, x.tensor.pick(...broadcast), sampleMean.tensor)
+        ops.powseq(stdTemp.tensor, 2)
+        ops.addeq(sampleStd.tensor, stdTemp.tensor)
+      }
+      ops.divseq(sampleStd.tensor, x.tensor.shape[this.axis])
+      ops.addseq(sampleStd.tensor, this.epsilon)
+      ops.sqrteq(sampleStd.tensor)
+      ops.addseq(sampleStd.tensor, this.epsilon)
+
+      // broadcast
+      for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+        broadcast[this.axis] = i
+        ops.assign(_mean.tensor.pick(...broadcast), sampleMean.tensor)
+        ops.assign(_std.tensor.pick(...broadcast), sampleStd.tensor)
+      }
+    } else if (this.mode === 2) {
+      // feature-wise normalization, using per-batch statistics
+      // here, batch size always = 1
+      for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
+        broadcast[this.axis] = i
+        let reduction = flattenDeep(unpack(x.tensor.pick(...broadcast)))
+        let axisMean = reduction.reduce((a, b) => a + b, 0) / reduction.length
+        let axisStd = reduction.map(x => (x - axisMean) * (x - axisMean)).reduce((a, b) => a + b, 0) / reduction.length
+        ops.assigns(_mean.tensor.pick(...broadcast), axisMean)
+        ops.assigns(_std.tensor.pick(...broadcast), axisStd + this.epsilon)
+      }
+      ops.sqrteq(_std.tensor)
+    } else {
+      throw new Error(`[normalization.BatchNormalization] Invalid mode ${this.mode}.`)
+    }
+
+    ops.subeq(x.tensor, _mean.tensor)
+    ops.diveq(x.tensor, _std.tensor)
+    ops.muleq(x.tensor, _gamma.tensor)
+    ops.addeq(x.tensor, _beta.tensor)
+
+    return x
+  }
+}
diff --git a/src/layers/normalization/index.js b/src/layers/normalization/index.js
new file mode 100644
index 0000000..5604229
--- /dev/null
+++ b/src/layers/normalization/index.js
@@ -0,0 +1,5 @@
+import BatchNormalization from './BatchNormalization'
+
+export {
+  BatchNormalization
+}
diff --git a/test/normalization/BatchNormalization.js b/test/normalization/BatchNormalization.js
new file mode 100644
index 0000000..f20d058
--- /dev/null
+++ b/test/normalization/BatchNormalization.js
@@ -0,0 +1,46 @@
+/* eslint-env browser, mocha */
+
+describe('normalization layer: BatchNormalization', function () {
+  const assert = chai.assert
+  const styles = testGlobals.styles
+  const logTime = testGlobals.logTime
+  const stringifyCondensed = testGlobals.stringifyCondensed
+  const approxEquals = KerasJS.testUtils.approxEquals
+  const layers = KerasJS.layers
+
+  const testParams = [
+    { attrs: { epsilon: 1e-5, mode: 0, axis: -1 } },
+    { attrs: { epsilon: 1e-2, mode: 0, axis: -1 } },
+    { attrs: { epsilon: 1e-5, mode: 0, axis: 1 } },
+    { attrs: { epsilon: 1e-5, mode: 0, axis: 2 } },
+    { attrs: { epsilon: 1e-5, mode: 0, axis: 3 } },
+    { attrs: { epsilon: 1e-5, mode: 1, axis: -1 } },
+    { attrs: { epsilon: 1e-5, mode: 2, axis: -1 } }
+  ]
+
+  before(function () {
+    console.log('\n%cnormalization layer: BatchNormalization', styles.h1)
+  })
+
+  testParams.forEach(({ attrs }, i) => {
+    const key = `normalization.BatchNormalization.${i}`
+    const title = `[${key}] test: epsilon='${attrs.epsilon}', mode=${attrs.mode}, axis=${attrs.axis}`
+
+    it(title, function () {
+      console.log(`\n%c${title}`, styles.h3)
+      let testLayer = new layers.BatchNormalization(attrs)
+      testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
+      let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
+      console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
+      const startTime = performance.now()
+      t = testLayer.call(t)
+      const endTime = performance.now()
+      console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
+      logTime(startTime, endTime)
+      const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
+      const shapeExpected = TEST_DATA[key].expected.shape
+      assert.deepEqual(t.tensor.shape, shapeExpected)
+      assert.isTrue(approxEquals(t.tensor, dataExpected))
+    })
+  })
+})
diff --git a/test/normalization/data_BatchNormalization.js b/test/normalization/data_BatchNormalization.js
new file mode 100644
index 0000000..3f1e091
--- /dev/null
+++ b/test/normalization/data_BatchNormalization.js
@@ -0,0 +1,118 @@
+// TEST DATA
+// Keyed by mocha test ID
+// Python code for generating test data can be found in the matching jupyter notebook in folder `notebooks/`.
+
+(function () {
+  var DATA = {
+    'normalization.BatchNormalization.0': {
+      input: {
+        data: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266],
+        shape: [4, 3]
+      },
+      weights: [
+        { data: [0.307179, -0.769986, 0.900566], shape: [3] },
+        { data: [-0.387536, -0.469873, -0.60788], shape: [3] },
+        { data: [-0.742023, -0.077688, -0.167692], shape: [3] },
+        { data: [0.597806, 0.435934, 0.01687], shape: [3] }
+      ],
+      expected: {
+        data: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193],
+        shape: [4, 3]
+      }
+    },
+    'normalization.BatchNormalization.1': {
+      input: {
+        data: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129],
+        shape: [4, 3]
+      },
+      weights: [
+        { data: [-0.211487, -0.648815, -0.854588], shape: [3] },
+        { data: [0.311362, -0.228519, 0.253024], shape: [3] },
+        { data: [-0.946541, 0.585593, -0.49527], shape: [3] },
+        { data: [0.557039, 0.055171, 0.263987], shape: [3] }
+      ],
+      expected: {
+        data: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487],
+        shape: [4, 3]
+      }
+    },
+    'normalization.BatchNormalization.2': {
+      input: {
+        data: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617],
+        shape: [4, 3, 2]
+      },
+      weights: [
+        { data: [0.517906, -0.666537, 0.378665, -0.380062], shape: [4] },
+        { data: [0.400557, 0.743871, 0.437134, -0.5548], shape: [4] },
+        { data: [-0.726393, 0.961405, -0.352651, -0.616831], shape: [4] },
+        { data: [0.91407, 0.630071, 0.508933, 0.052897], shape: [4] }
+      ],
+      expected: {
+        data: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177],
+        shape: [4, 3, 2]
+      }
+    },
+    'normalization.BatchNormalization.3': {
+      input: {
+        data: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809],
+        shape: [4, 3, 2]
+      },
+      weights: [
+        { data: [-0.089873, -0.210305, -0.554444], shape: [3] },
+        { data: [-0.997964, -0.487343, -0.350362], shape: [3] },
+        { data: [-0.145437, 0.11872, -0.368563], shape: [3] },
+        { data: [0.486933, 0.587355, 0.517041], shape: [3] }
+      ],
+      expected: {
+        data: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342],
+        shape: [4, 3, 2]
+      }
+    },
+    'normalization.BatchNormalization.4': {
+      input: {
+        data: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951],
+        shape: [4, 3, 2]
+      },
+      weights: [
+        { data: [0.660544, -0.047716], shape: [2] },
+        { data: [-0.956656, 0.127814], shape: [2] },
+        { data: [-0.359373, -0.71013], shape: [2] },
+        { data: [0.551285, 0.957558], shape: [2] }
+      ],
+      expected: {
+        data: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732],
+        shape: [4, 3, 2]
+      }
+    },
+    'normalization.BatchNormalization.5': {
+      input: {
+        data: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567],
+        shape: [4, 3, 2]
+      },
+      weights: [
+        { data: [0.833294, -0.732828], shape: [2] },
+        { data: [0.013718, -0.842599], shape: [2] }
+      ],
+      expected: {
+        data: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066],
+        shape: [4, 3, 2]
+      }
+    },
+    'normalization.BatchNormalization.6': {
+      input: {
+        data: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054],
+        shape: [4, 3, 2]
+      },
+      weights: [
+        { data: [0.128705, -0.956067], shape: [2] },
+        { data: [-0.168081, -0.9895], shape: [2] }
+      ],
+      expected: {
+        data: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298],
+        shape: [4, 3, 2]
+      }
+    }
+  }
+
+  window.TEST_DATA = Object.assign({}, window.TEST_DATA, DATA)
+})()