mirror of
https://github.com/wassname/keras-js.git
synced 2026-09-09 11:25:25 +08:00
implement BatchNormalization in all modes, with tests
This commit is contained in:
Leon Chen
@@ -94,6 +94,9 @@
|
||||
<script src="/test/pooling/data_GlobalAveragePooling2D.js"></script>
|
||||
<script src="/test/pooling/GlobalAveragePooling2D.js"></script>
|
||||
|
||||
<script src="/test/normalization/data_BatchNormalization.js"></script>
|
||||
<script src="/test/normalization/BatchNormalization.js"></script>
|
||||
|
||||
<script>
|
||||
// mocha.checkLeaks();
|
||||
mocha.globals(['jQuery']);
|
||||
|
||||
@@ -0,0 +1,559 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from keras.models import Model\n",
|
||||
"from keras.layers import Input\n",
|
||||
"from keras.layers.normalization import BatchNormalization\n",
|
||||
"from keras import backend as K"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_decimal(arr, places=6):\n",
|
||||
" return [round(x * 10**places) / 10**places for x in arr]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### BatchNormalization"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.0] epsilon=1e-05, mode=0, axis=-1**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 68,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (3,)\n",
|
||||
"gamma: [0.307179, -0.769986, 0.900566]\n",
|
||||
"beta shape: (3,)\n",
|
||||
"beta: [-0.387536, -0.469873, -0.60788]\n",
|
||||
"running_mean shape: (3,)\n",
|
||||
"running_mean: [-0.742023, -0.077688, -0.167692]\n",
|
||||
"running_std shape: (3,)\n",
|
||||
"running_std: [0.597806, 0.435934, 0.01687]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3)\n",
|
||||
"in: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266]\n",
|
||||
"out shape: (4, 3)\n",
|
||||
"out: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=-1)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1000 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.1] epsilon=1e-02, mode=0, axis=-1**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 69,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (3,)\n",
|
||||
"gamma: [-0.211487, -0.648815, -0.854588]\n",
|
||||
"beta shape: (3,)\n",
|
||||
"beta: [0.311362, -0.228519, 0.253024]\n",
|
||||
"running_mean shape: (3,)\n",
|
||||
"running_mean: [-0.946541, 0.585593, -0.49527]\n",
|
||||
"running_std shape: (3,)\n",
|
||||
"running_std: [0.557039, 0.055171, 0.263987]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3)\n",
|
||||
"in: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129]\n",
|
||||
"out shape: (4, 3)\n",
|
||||
"out: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-02, mode=0, axis=-1)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1010 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.2] epsilon=1e-05, mode=0, axis=1**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 70,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (4,)\n",
|
||||
"gamma: [0.517906, -0.666537, 0.378665, -0.380062]\n",
|
||||
"beta shape: (4,)\n",
|
||||
"beta: [0.400557, 0.743871, 0.437134, -0.5548]\n",
|
||||
"running_mean shape: (4,)\n",
|
||||
"running_mean: [-0.726393, 0.961405, -0.352651, -0.616831]\n",
|
||||
"running_std shape: (4,)\n",
|
||||
"running_std: [0.91407, 0.630071, 0.508933, 0.052897]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3, 2)\n",
|
||||
"in: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617]\n",
|
||||
"out shape: (4, 3, 2)\n",
|
||||
"out: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3, 2)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=1)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1020 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.3] epsilon=1e-05, mode=0, axis=2**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 71,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (3,)\n",
|
||||
"gamma: [-0.089873, -0.210305, -0.554444]\n",
|
||||
"beta shape: (3,)\n",
|
||||
"beta: [-0.997964, -0.487343, -0.350362]\n",
|
||||
"running_mean shape: (3,)\n",
|
||||
"running_mean: [-0.145437, 0.11872, -0.368563]\n",
|
||||
"running_std shape: (3,)\n",
|
||||
"running_std: [0.486933, 0.587355, 0.517041]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3, 2)\n",
|
||||
"in: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809]\n",
|
||||
"out shape: (4, 3, 2)\n",
|
||||
"out: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3, 2)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=2)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1030 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.4] epsilon=1e-05, mode=0, axis=3**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (2,)\n",
|
||||
"gamma: [0.660544, -0.047716]\n",
|
||||
"beta shape: (2,)\n",
|
||||
"beta: [-0.956656, 0.127814]\n",
|
||||
"running_mean shape: (2,)\n",
|
||||
"running_mean: [-0.359373, -0.71013]\n",
|
||||
"running_std shape: (2,)\n",
|
||||
"running_std: [0.551285, 0.957558]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3, 2)\n",
|
||||
"in: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951]\n",
|
||||
"out shape: (4, 3, 2)\n",
|
||||
"out: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3, 2)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=3)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1040 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.5] epsilon=1e-05, mode=1, axis=-1**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 102,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (2,)\n",
|
||||
"gamma: [0.833294, -0.732828]\n",
|
||||
"beta shape: (2,)\n",
|
||||
"beta: [0.013718, -0.842599]\n",
|
||||
"running_mean shape: (2,)\n",
|
||||
"running_mean: [0.331031, -0.238168]\n",
|
||||
"running_std shape: (2,)\n",
|
||||
"running_std: [0.644294, 0.114097]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3, 2)\n",
|
||||
"in: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567]\n",
|
||||
"out shape: (4, 3, 2)\n",
|
||||
"out: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3, 2)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=1, axis=-1)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1050 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[normalization.BatchNormalization.6] epsilon=1e-05, mode=2, axis=-1**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 97,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"gamma shape: (2,)\n",
|
||||
"gamma: [0.128705, -0.956067]\n",
|
||||
"beta shape: (2,)\n",
|
||||
"beta: [-0.168081, -0.9895]\n",
|
||||
"running_mean shape: (2,)\n",
|
||||
"running_mean: [0.188775, -0.258915]\n",
|
||||
"running_std shape: (2,)\n",
|
||||
"running_std: [0.318404, 0.231636]\n",
|
||||
"\n",
|
||||
"in shape: (4, 3, 2)\n",
|
||||
"in: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054]\n",
|
||||
"out shape: (4, 3, 2)\n",
|
||||
"out: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4, 3, 2)\n",
|
||||
"norm = BatchNormalization(epsilon=1e-05, mode=2, axis=-1)\n",
|
||||
"\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = norm(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(1060 + i)\n",
|
||||
" if i == 3:\n",
|
||||
" # std should be positive\n",
|
||||
" weights.append(np.random.random(w.shape))\n",
|
||||
" else:\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('gamma shape:', weights[0].shape)\n",
|
||||
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('beta shape:', weights[1].shape)\n",
|
||||
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('running_mean shape:', weights[2].shape)\n",
|
||||
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('running_std shape:', weights[3].shape)\n",
|
||||
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python [default]",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.5.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
@@ -2,3 +2,4 @@ export * from './advanced_activations'
|
||||
export * from './core'
|
||||
export * from './convolutional'
|
||||
export * from './pooling'
|
||||
export * from './normalization'
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import Layer from '../../engine/Layer'
|
||||
import Tensor from '../../Tensor'
|
||||
import ops from 'ndarray-ops'
|
||||
import unpack from 'ndarray-unpack'
|
||||
import flattenDeep from 'lodash/flattenDeep'
|
||||
|
||||
/**
|
||||
* BatchNormalization layer class
|
||||
*/
|
||||
export default class BatchNormalization extends Layer {
|
||||
/**
|
||||
* Creates an BatchNormalization layer
|
||||
*/
|
||||
constructor (attrs = {}) {
|
||||
super(attrs)
|
||||
const {
|
||||
epsilon = 1e-5,
|
||||
mode = 0,
|
||||
axis = -1
|
||||
} = attrs
|
||||
|
||||
this.epsilon = epsilon
|
||||
this.mode = mode
|
||||
this.axis = axis
|
||||
|
||||
// Layer weights specification
|
||||
// running mean and std are non_trainable_weights in mode 0
|
||||
this.params = this.mode === 0
|
||||
? ['gamma', 'beta', 'running_mean', 'running_std']
|
||||
: ['gamma', 'beta']
|
||||
}
|
||||
|
||||
/**
|
||||
* Method for layer computational logic
|
||||
* @param {Tensor} x
|
||||
* @returns {Tensor} x
|
||||
*/
|
||||
call (x) {
|
||||
// no batch axis, so axis is less 1 compared to representation in keras
|
||||
this.axis = this.axis < 0 ? x.tensor.shape.length + this.axis : this.axis - 1
|
||||
|
||||
let broadcast = []
|
||||
for (let d = 0; d < x.tensor.shape.length; d++) {
|
||||
if (d === this.axis) broadcast.push(1)
|
||||
else broadcast.push(null)
|
||||
}
|
||||
|
||||
// broadcast weights
|
||||
let _gamma = new Tensor([], x.tensor.shape)
|
||||
let _beta = new Tensor([], x.tensor.shape)
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
ops.assigns(_gamma.tensor.pick(...broadcast), this.weights.gamma.tensor.get(i))
|
||||
ops.assigns(_beta.tensor.pick(...broadcast), this.weights.beta.tensor.get(i))
|
||||
}
|
||||
|
||||
let _mean = new Tensor([], x.tensor.shape)
|
||||
let _std = new Tensor([], x.tensor.shape)
|
||||
|
||||
if (this.mode === 0) {
|
||||
// feature-wise normalization
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
ops.assigns(_mean.tensor.pick(...broadcast), this.weights.running_mean.tensor.get(i))
|
||||
ops.assigns(_std.tensor.pick(...broadcast), this.weights.running_std.tensor.get(i) + this.epsilon)
|
||||
}
|
||||
ops.sqrteq(_std.tensor)
|
||||
} else if (this.mode === 1) {
|
||||
// sample-wise normalization
|
||||
let reducedShape = x.tensor.shape.slice()
|
||||
reducedShape.splice(this.axis, 1)
|
||||
|
||||
// mean
|
||||
let sampleMean = new Tensor([], reducedShape)
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
ops.addeq(sampleMean.tensor, x.tensor.pick(...broadcast))
|
||||
}
|
||||
ops.divseq(sampleMean.tensor, x.tensor.shape[this.axis])
|
||||
|
||||
// stddev
|
||||
let sampleStd = new Tensor([], reducedShape)
|
||||
let stdTemp = new Tensor([], reducedShape)
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
ops.sub(stdTemp.tensor, x.tensor.pick(...broadcast), sampleMean.tensor)
|
||||
ops.powseq(stdTemp.tensor, 2)
|
||||
ops.addeq(sampleStd.tensor, stdTemp.tensor)
|
||||
}
|
||||
ops.divseq(sampleStd.tensor, x.tensor.shape[this.axis])
|
||||
ops.addseq(sampleStd.tensor, this.epsilon)
|
||||
ops.sqrteq(sampleStd.tensor)
|
||||
ops.addseq(sampleStd.tensor, this.epsilon)
|
||||
|
||||
// broadcast
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
ops.assign(_mean.tensor.pick(...broadcast), sampleMean.tensor)
|
||||
ops.assign(_std.tensor.pick(...broadcast), sampleStd.tensor)
|
||||
}
|
||||
} else if (this.mode === 2) {
|
||||
// feature-wise normalization, using per-batch statistics
|
||||
// here, batch size always = 1
|
||||
for (let i = 0; i < x.tensor.shape[this.axis]; i++) {
|
||||
broadcast[this.axis] = i
|
||||
let reduction = flattenDeep(unpack(x.tensor.pick(...broadcast)))
|
||||
let axisMean = reduction.reduce((a, b) => a + b, 0) / reduction.length
|
||||
let axisStd = reduction.map(x => (x - axisMean) * (x - axisMean)).reduce((a, b) => a + b, 0) / reduction.length
|
||||
ops.assigns(_mean.tensor.pick(...broadcast), axisMean)
|
||||
ops.assigns(_std.tensor.pick(...broadcast), axisStd + this.epsilon)
|
||||
}
|
||||
ops.sqrteq(_std.tensor)
|
||||
} else {
|
||||
throw new Error(`[normalization.BatchNormalization] Invalid mode ${this.mode}.`)
|
||||
}
|
||||
|
||||
ops.subeq(x.tensor, _mean.tensor)
|
||||
ops.diveq(x.tensor, _std.tensor)
|
||||
ops.muleq(x.tensor, _gamma.tensor)
|
||||
ops.addeq(x.tensor, _beta.tensor)
|
||||
|
||||
return x
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import BatchNormalization from './BatchNormalization'
|
||||
|
||||
export {
|
||||
BatchNormalization
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/* eslint-env browser, mocha */
|
||||
|
||||
describe('normalization layer: BatchNormalization', function () {
|
||||
const assert = chai.assert
|
||||
const styles = testGlobals.styles
|
||||
const logTime = testGlobals.logTime
|
||||
const stringifyCondensed = testGlobals.stringifyCondensed
|
||||
const approxEquals = KerasJS.testUtils.approxEquals
|
||||
const layers = KerasJS.layers
|
||||
|
||||
const testParams = [
|
||||
{ attrs: { epsilon: 1e-5, mode: 0, axis: -1 } },
|
||||
{ attrs: { epsilon: 1e-2, mode: 0, axis: -1 } },
|
||||
{ attrs: { epsilon: 1e-5, mode: 0, axis: 1 } },
|
||||
{ attrs: { epsilon: 1e-5, mode: 0, axis: 2 } },
|
||||
{ attrs: { epsilon: 1e-5, mode: 0, axis: 3 } },
|
||||
{ attrs: { epsilon: 1e-5, mode: 1, axis: -1 } },
|
||||
{ attrs: { epsilon: 1e-5, mode: 2, axis: -1 } }
|
||||
]
|
||||
|
||||
before(function () {
|
||||
console.log('\n%cnormalization layer: BatchNormalization', styles.h1)
|
||||
})
|
||||
|
||||
testParams.forEach(({ attrs }, i) => {
|
||||
const key = `normalization.BatchNormalization.${i}`
|
||||
const title = `[${key}] test: epsilon='${attrs.epsilon}', mode=${attrs.mode}, axis=${attrs.axis}`
|
||||
|
||||
it(title, function () {
|
||||
console.log(`\n%c${title}`, styles.h3)
|
||||
let testLayer = new layers.BatchNormalization(attrs)
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,118 @@
|
||||
// TEST DATA
|
||||
// Keyed by mocha test ID
|
||||
// Python code for generating test data can be found in the matching jupyter notebook in folder `notebooks/`.
|
||||
|
||||
(function () {
|
||||
var DATA = {
|
||||
'normalization.BatchNormalization.0': {
|
||||
input: {
|
||||
data: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266],
|
||||
shape: [4, 3]
|
||||
},
|
||||
weights: [
|
||||
{ data: [0.307179, -0.769986, 0.900566], shape: [3] },
|
||||
{ data: [-0.387536, -0.469873, -0.60788], shape: [3] },
|
||||
{ data: [-0.742023, -0.077688, -0.167692], shape: [3] },
|
||||
{ data: [0.597806, 0.435934, 0.01687], shape: [3] }
|
||||
],
|
||||
expected: {
|
||||
data: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193],
|
||||
shape: [4, 3]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.1': {
|
||||
input: {
|
||||
data: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129],
|
||||
shape: [4, 3]
|
||||
},
|
||||
weights: [
|
||||
{ data: [-0.211487, -0.648815, -0.854588], shape: [3] },
|
||||
{ data: [0.311362, -0.228519, 0.253024], shape: [3] },
|
||||
{ data: [-0.946541, 0.585593, -0.49527], shape: [3] },
|
||||
{ data: [0.557039, 0.055171, 0.263987], shape: [3] }
|
||||
],
|
||||
expected: {
|
||||
data: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487],
|
||||
shape: [4, 3]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.2': {
|
||||
input: {
|
||||
data: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617],
|
||||
shape: [4, 3, 2]
|
||||
},
|
||||
weights: [
|
||||
{ data: [0.517906, -0.666537, 0.378665, -0.380062], shape: [4] },
|
||||
{ data: [0.400557, 0.743871, 0.437134, -0.5548], shape: [4] },
|
||||
{ data: [-0.726393, 0.961405, -0.352651, -0.616831], shape: [4] },
|
||||
{ data: [0.91407, 0.630071, 0.508933, 0.052897], shape: [4] }
|
||||
],
|
||||
expected: {
|
||||
data: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177],
|
||||
shape: [4, 3, 2]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.3': {
|
||||
input: {
|
||||
data: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809],
|
||||
shape: [4, 3, 2]
|
||||
},
|
||||
weights: [
|
||||
{ data: [-0.089873, -0.210305, -0.554444], shape: [3] },
|
||||
{ data: [-0.997964, -0.487343, -0.350362], shape: [3] },
|
||||
{ data: [-0.145437, 0.11872, -0.368563], shape: [3] },
|
||||
{ data: [0.486933, 0.587355, 0.517041], shape: [3] }
|
||||
],
|
||||
expected: {
|
||||
data: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342],
|
||||
shape: [4, 3, 2]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.4': {
|
||||
input: {
|
||||
data: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951],
|
||||
shape: [4, 3, 2]
|
||||
},
|
||||
weights: [
|
||||
{ data: [0.660544, -0.047716], shape: [2] },
|
||||
{ data: [-0.956656, 0.127814], shape: [2] },
|
||||
{ data: [-0.359373, -0.71013], shape: [2] },
|
||||
{ data: [0.551285, 0.957558], shape: [2] }
|
||||
],
|
||||
expected: {
|
||||
data: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732],
|
||||
shape: [4, 3, 2]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.5': {
|
||||
input: {
|
||||
data: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567],
|
||||
shape: [4, 3, 2]
|
||||
},
|
||||
weights: [
|
||||
{ data: [0.833294, -0.732828], shape: [2] },
|
||||
{ data: [0.013718, -0.842599], shape: [2] }
|
||||
],
|
||||
expected: {
|
||||
data: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066],
|
||||
shape: [4, 3, 2]
|
||||
}
|
||||
},
|
||||
'normalization.BatchNormalization.6': {
|
||||
input: {
|
||||
data: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054],
|
||||
shape: [4, 3, 2]
|
||||
},
|
||||
weights: [
|
||||
{ data: [0.128705, -0.956067], shape: [2] },
|
||||
{ data: [-0.168081, -0.9895], shape: [2] }
|
||||
],
|
||||
expected: {
|
||||
data: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298],
|
||||
shape: [4, 3, 2]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
window.TEST_DATA = Object.assign({}, window.TEST_DATA, DATA)
|
||||
})()
|
||||
Reference in New Issue
Block a user