wassname/keras-js
1
0
Fork 0
mirror of https://github.com/wassname/keras-js.git synced 2026-08-30 11:25:40 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
9b352c1935d5bc70553cd2fcb52445f194ef2e8d
keras-js/notebooks/normalization/BatchNormalization.ipynb
T

560 lines
20 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 66,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import numpy as np\n",
"from keras.models import Model\n",
"from keras.layers import Input\n",
"from keras.layers.normalization import BatchNormalization\n",
"from keras import backend as K"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def format_decimal(arr, places=6):\n",
" return [round(x * 10**places) / 10**places for x in arr]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### BatchNormalization"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.0] epsilon=1e-05, mode=0, axis=-1**"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (3,)\n",
"gamma: [0.307179, -0.769986, 0.900566]\n",
"beta shape: (3,)\n",
"beta: [-0.387536, -0.469873, -0.60788]\n",
"running_mean shape: (3,)\n",
"running_mean: [-0.742023, -0.077688, -0.167692]\n",
"running_std shape: (3,)\n",
"running_std: [0.597806, 0.435934, 0.01687]\n",
"\n",
"in shape: (4, 3)\n",
"in: [0.193375, 0.789956, 0.069255, -0.988089, 0.804359, 0.509039, -0.655792, 0.460058, -0.25375, -0.635374, -0.109318, -0.426266]\n",
"out shape: (4, 3)\n",
"out: [-0.015911, -1.481707, 1.034528, -0.485295, -1.498503, 4.082906, -0.353277, -1.096984, -1.204392, -0.345165, -0.432986, -2.400193]\n"
]
}
],
"source": [
"data_in_shape = (4, 3)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=-1)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1000 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.1] epsilon=1e-02, mode=0, axis=-1**"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (3,)\n",
"gamma: [-0.211487, -0.648815, -0.854588]\n",
"beta shape: (3,)\n",
"beta: [0.311362, -0.228519, 0.253024]\n",
"running_mean shape: (3,)\n",
"running_mean: [-0.946541, 0.585593, -0.49527]\n",
"running_std shape: (3,)\n",
"running_std: [0.557039, 0.055171, 0.263987]\n",
"\n",
"in shape: (4, 3)\n",
"in: [0.718808, -0.159628, -0.955364, -0.945802, -0.433707, -0.559921, 0.418526, 0.035668, 0.099925, 0.537048, 0.914591, -0.504129]\n",
"out shape: (4, 3)\n",
"out: [-0.156355, 1.665477, 1.004194, 0.311154, 2.362055, 0.358576, -0.07202, 1.169128, -0.718718, -0.105307, -1.064677, 0.267487]\n"
]
}
],
"source": [
"data_in_shape = (4, 3)\n",
"norm = BatchNormalization(epsilon=1e-02, mode=0, axis=-1)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1010 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.2] epsilon=1e-05, mode=0, axis=1**"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (4,)\n",
"gamma: [0.517906, -0.666537, 0.378665, -0.380062]\n",
"beta shape: (4,)\n",
"beta: [0.400557, 0.743871, 0.437134, -0.5548]\n",
"running_mean shape: (4,)\n",
"running_mean: [-0.726393, 0.961405, -0.352651, -0.616831]\n",
"running_std shape: (4,)\n",
"running_std: [0.91407, 0.630071, 0.508933, 0.052897]\n",
"\n",
"in shape: (4, 3, 2)\n",
"in: [-0.920023, 0.466664, 0.332702, 0.693696, -0.705391, 0.385741, 0.296606, 0.805754, -0.876679, -0.140908, -0.817375, 0.222433, -0.406426, 0.144013, 0.566715, 0.215483, -0.18429, 0.901878, 0.31492, -0.560544, 0.989602, 0.265541, 0.755267, 0.664617]\n",
"out shape: (4, 3, 2)\n",
"out: [0.295668, 1.046836, 0.974269, 1.16982, 0.411934, 1.003001, 1.302105, 0.874571, 2.287317, 1.669487, 2.237519, 1.364389, 0.408591, 0.700756, 0.925121, 0.738691, 0.526497, 1.103021, -2.094362, -0.647805, -3.209161, -2.012771, -2.821961, -2.672177]\n"
]
}
],
"source": [
"data_in_shape = (4, 3, 2)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=1)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1020 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.3] epsilon=1e-05, mode=0, axis=2**"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (3,)\n",
"gamma: [-0.089873, -0.210305, -0.554444]\n",
"beta shape: (3,)\n",
"beta: [-0.997964, -0.487343, -0.350362]\n",
"running_mean shape: (3,)\n",
"running_mean: [-0.145437, 0.11872, -0.368563]\n",
"running_std shape: (3,)\n",
"running_std: [0.486933, 0.587355, 0.517041]\n",
"\n",
"in shape: (4, 3, 2)\n",
"in: [0.177239, -0.818004, -0.955256, -0.625241, 0.936162, 0.471789, -0.251405, 0.671525, -0.819759, 0.468665, 0.492218, 0.014732, -0.415985, 0.354683, -0.445545, -0.401559, 0.68942, 0.302571, -0.247906, -0.871673, -0.153346, -0.940188, 0.924139, 0.554809]\n",
"out shape: (4, 3, 2)\n",
"out: [-1.039522, -0.911343, -0.192635, -0.283194, -1.35639, -0.998328, -0.984316, -1.103182, -0.229817, -0.58337, -1.01408, -0.645908, -0.96312, -1.062375, -0.332504, -0.344574, -1.166136, -0.86785, -0.984767, -0.904431, -0.412686, -0.19677, -1.347119, -1.062342]\n"
]
}
],
"source": [
"data_in_shape = (4, 3, 2)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=2)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1030 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.4] epsilon=1e-05, mode=0, axis=3**"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (2,)\n",
"gamma: [0.660544, -0.047716]\n",
"beta shape: (2,)\n",
"beta: [-0.956656, 0.127814]\n",
"running_mean shape: (2,)\n",
"running_mean: [-0.359373, -0.71013]\n",
"running_std shape: (2,)\n",
"running_std: [0.551285, 0.957558]\n",
"\n",
"in shape: (4, 3, 2)\n",
"in: [-0.929182, 0.875771, -0.883762, 0.72433, -0.585509, -0.152172, -0.775304, 0.788189, -0.508879, 0.009276, 0.928936, -0.407103, 0.022937, -0.299328, -0.616092, -0.951088, -0.779181, 0.39148, -0.023818, 0.37291, -0.119784, -0.941465, -0.9535, -0.687951]\n",
"out shape: (4, 3, 2)\n",
"out: [-1.463575, 0.050482, -1.423168, 0.057867, -1.157833, 0.100607, -1.326681, 0.054753, -1.089661, 0.092734, 0.189464, 0.113038, -0.616541, 0.107782, -1.18504, 0.139563, -1.33013, 0.074097, -0.658135, 0.075003, -0.74351, 0.139094, -1.485209, 0.126732]\n"
]
}
],
"source": [
"data_in_shape = (4, 3, 2)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=0, axis=3)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1040 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.5] epsilon=1e-05, mode=1, axis=-1**"
]
},
{
"cell_type": "code",
"execution_count": 102,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (2,)\n",
"gamma: [0.833294, -0.732828]\n",
"beta shape: (2,)\n",
"beta: [0.013718, -0.842599]\n",
"running_mean shape: (2,)\n",
"running_mean: [0.331031, -0.238168]\n",
"running_std shape: (2,)\n",
"running_std: [0.644294, 0.114097]\n",
"\n",
"in shape: (4, 3, 2)\n",
"in: [0.295997, -0.197309, 0.752622, -0.502156, -0.322833, 0.698127, 0.740562, -0.668644, -0.229012, -0.967928, -0.411317, 0.458195, 0.94757, 0.256302, -0.571923, 0.259773, -0.737641, -0.119598, -0.999641, 0.772751, -0.900172, 0.100827, 0.17941, 0.288567]\n",
"out shape: (4, 3, 2)\n",
"out: [0.84691, -0.10986, 0.846988, -0.109791, -0.819544, -1.575399, 0.846992, -0.109788, 0.846959, -0.109817, -0.819535, -1.575391, 0.846953, -0.109822, -0.819532, -1.575388, -0.819506, -1.575365, -0.819562, -1.575414, -0.819543, -1.575398, -0.818029, -1.574066]\n"
]
}
],
"source": [
"data_in_shape = (4, 3, 2)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=1, axis=-1)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1050 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**[normalization.BatchNormalization.6] epsilon=1e-05, mode=2, axis=-1**"
]
},
{
"cell_type": "code",
"execution_count": 97,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"gamma shape: (2,)\n",
"gamma: [0.128705, -0.956067]\n",
"beta shape: (2,)\n",
"beta: [-0.168081, -0.9895]\n",
"running_mean shape: (2,)\n",
"running_mean: [0.188775, -0.258915]\n",
"running_std shape: (2,)\n",
"running_std: [0.318404, 0.231636]\n",
"\n",
"in shape: (4, 3, 2)\n",
"in: [-0.435983, -0.714349, -0.557696, -0.838492, 0.885101, -0.155057, 0.36079, 0.909788, 0.061332, -0.616654, 0.853256, -0.279329, -0.308602, 0.003971, -0.539738, 0.186558, 0.261057, 0.11644, -0.980123, -0.455651, 0.528218, 0.999738, -0.665862, 0.374054]\n",
"out shape: (4, 3, 2)\n",
"out: [-0.252569, 0.142873, -0.27886, 0.351052, 0.032798, -0.795018, -0.080458, -2.580686, -0.145144, -0.020954, 0.025919, -0.586624, -0.225054, -1.061697, -0.274981, -1.367882, -0.102002, -1.250299, -0.370108, -0.290944, -0.044292, -2.731525, -0.302225, -1.682298]\n"
]
}
],
"source": [
"data_in_shape = (4, 3, 2)\n",
"norm = BatchNormalization(epsilon=1e-05, mode=2, axis=-1)\n",
"\n",
"layer_0 = Input(shape=data_in_shape)\n",
"layer_1 = norm(layer_0)\n",
"model = Model(input=layer_0, output=layer_1)\n",
"\n",
"# set weights to random (use seed for reproducibility)\n",
"weights = []\n",
"for i, w in enumerate(model.get_weights()):\n",
" np.random.seed(1060 + i)\n",
" if i == 3:\n",
" # std should be positive\n",
" weights.append(np.random.random(w.shape))\n",
" else:\n",
" weights.append(2 * np.random.random(w.shape) - 1)\n",
"model.set_weights(weights)\n",
"print('gamma shape:', weights[0].shape)\n",
"print('gamma:', format_decimal(weights[0].ravel().tolist()))\n",
"print('beta shape:', weights[1].shape)\n",
"print('beta:', format_decimal(weights[1].ravel().tolist()))\n",
"print('running_mean shape:', weights[2].shape)\n",
"print('running_mean:', format_decimal(weights[2].ravel().tolist()))\n",
"print('running_std shape:', weights[3].shape)\n",
"print('running_std:', format_decimal(weights[3].ravel().tolist()))\n",
"\n",
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
"print('')\n",
"print('in shape:', data_in_shape)\n",
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
"result = model.predict(np.array([data_in]))\n",
"print('out shape:', result[0].shape)\n",
"print('out:', format_decimal(result[0].ravel().tolist()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [default]",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Reference in New Issue View Git Blame Copy Permalink