{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.models import Model\n",
    "from keras.layers import Input\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def format_decimal(arr, places=6):\n",
    "    return [round(x * 10**places) / 10**places for x in arr]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BatchNormalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.0] epsilon=1e-05, mode=0, axis=-1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (3,)\n",
      "gamma: [0.307179, -0.769986, 0.900566]\n",
      "beta shape: (3,)\n",
      "beta: [-0.387536, -0.469873, -0.60788]\n",
      "running_mean shape: (3,)\n",
      "running_mean: [-0.742023, -0.077688, -0.167692]\n",
      "running_std shape: (3,)\n",
      "running_std: [0.597806, 0.435934, 0.01687]\n",
      "\n",
      "in shape: (4, 3)\n",
      "in: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266]\n",
      "out shape: (4, 3)\n",
      "out: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=-1)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1000 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.1] epsilon=1e-02, mode=0, axis=-1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (3,)\n",
      "gamma: [-0.211487, -0.648815, -0.854588]\n",
      "beta shape: (3,)\n",
      "beta: [0.311362, -0.228519, 0.253024]\n",
      "running_mean shape: (3,)\n",
      "running_mean: [-0.946541, 0.585593, -0.49527]\n",
      "running_std shape: (3,)\n",
      "running_std: [0.557039, 0.055171, 0.263987]\n",
      "\n",
      "in shape: (4, 3)\n",
      "in: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129]\n",
      "out shape: (4, 3)\n",
      "out: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3)\n",
    "norm = BatchNormalization(epsilon=1e-02, mode=0, axis=-1)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1010 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.2] epsilon=1e-05, mode=0, axis=1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (4,)\n",
      "gamma: [0.517906, -0.666537, 0.378665, -0.380062]\n",
      "beta shape: (4,)\n",
      "beta: [0.400557, 0.743871, 0.437134, -0.5548]\n",
      "running_mean shape: (4,)\n",
      "running_mean: [-0.726393, 0.961405, -0.352651, -0.616831]\n",
      "running_std shape: (4,)\n",
      "running_std: [0.91407, 0.630071, 0.508933, 0.052897]\n",
      "\n",
      "in shape: (4, 3, 2)\n",
      "in: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617]\n",
      "out shape: (4, 3, 2)\n",
      "out: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3, 2)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=1)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1020 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.3] epsilon=1e-05, mode=0, axis=2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (3,)\n",
      "gamma: [-0.089873, -0.210305, -0.554444]\n",
      "beta shape: (3,)\n",
      "beta: [-0.997964, -0.487343, -0.350362]\n",
      "running_mean shape: (3,)\n",
      "running_mean: [-0.145437, 0.11872, -0.368563]\n",
      "running_std shape: (3,)\n",
      "running_std: [0.486933, 0.587355, 0.517041]\n",
      "\n",
      "in shape: (4, 3, 2)\n",
      "in: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809]\n",
      "out shape: (4, 3, 2)\n",
      "out: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3, 2)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=2)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1030 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.4] epsilon=1e-05, mode=0, axis=3**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (2,)\n",
      "gamma: [0.660544, -0.047716]\n",
      "beta shape: (2,)\n",
      "beta: [-0.956656, 0.127814]\n",
      "running_mean shape: (2,)\n",
      "running_mean: [-0.359373, -0.71013]\n",
      "running_std shape: (2,)\n",
      "running_std: [0.551285, 0.957558]\n",
      "\n",
      "in shape: (4, 3, 2)\n",
      "in: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951]\n",
      "out shape: (4, 3, 2)\n",
      "out: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3, 2)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=0, axis=3)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1040 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.5] epsilon=1e-05, mode=1, axis=-1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (2,)\n",
      "gamma: [0.833294, -0.732828]\n",
      "beta shape: (2,)\n",
      "beta: [0.013718, -0.842599]\n",
      "running_mean shape: (2,)\n",
      "running_mean: [0.331031, -0.238168]\n",
      "running_std shape: (2,)\n",
      "running_std: [0.644294, 0.114097]\n",
      "\n",
      "in shape: (4, 3, 2)\n",
      "in: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567]\n",
      "out shape: (4, 3, 2)\n",
      "out: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3, 2)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=1, axis=-1)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1050 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**[normalization.BatchNormalization.6] epsilon=1e-05, mode=2, axis=-1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma shape: (2,)\n",
      "gamma: [0.128705, -0.956067]\n",
      "beta shape: (2,)\n",
      "beta: [-0.168081, -0.9895]\n",
      "running_mean shape: (2,)\n",
      "running_mean: [0.188775, -0.258915]\n",
      "running_std shape: (2,)\n",
      "running_std: [0.318404, 0.231636]\n",
      "\n",
      "in shape: (4, 3, 2)\n",
      "in: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054]\n",
      "out shape: (4, 3, 2)\n",
      "out: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298]\n"
     ]
    }
   ],
   "source": [
    "data_in_shape = (4, 3, 2)\n",
    "norm = BatchNormalization(epsilon=1e-05, mode=2, axis=-1)\n",
    "\n",
    "layer_0 = Input(shape=data_in_shape)\n",
    "layer_1 = norm(layer_0)\n",
    "model = Model(input=layer_0, output=layer_1)\n",
    "\n",
    "# set weights to random (use seed for reproducibility)\n",
    "weights = []\n",
    "for i, w in enumerate(model.get_weights()):\n",
    "    np.random.seed(1060 + i)\n",
    "    if i == 3:\n",
    "        # std should be positive\n",
    "        weights.append(np.random.random(w.shape))\n",
    "    else:\n",
    "        weights.append(2 * np.random.random(w.shape) - 1)\n",
    "model.set_weights(weights)\n",
    "print('gamma shape:', weights[0].shape)\n",
    "print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
    "print('beta shape:', weights[1].shape)\n",
    "print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
    "print('running_mean shape:', weights[2].shape)\n",
    "print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
    "print('running_std shape:', weights[3].shape)\n",
    "print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
    "\n",
    "data_in = 2 * np.random.random(data_in_shape) - 1\n",
    "print('')\n",
    "print('in shape:', data_in_shape)\n",
    "print('in:', format_decimal(data_in.ravel().tolist()))\n",
    "result = model.predict(np.array([data_in]))\n",
    "print('out shape:', result[0].shape)\n",
    "print('out:', format_decimal(result[0].ravel().tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}