mirror of
https://github.com/wassname/keras-js.git
synced 2026-09-09 11:25:25 +08:00
add MaxoutDense and Highway layers, with tests
This commit is contained in:
Leon Chen
@@ -40,6 +40,10 @@
|
||||
<script src="/test/core/RepeatVector.js"></script>
|
||||
<script src="/test/core/data_Merge.js"></script>
|
||||
<script src="/test/core/Merge.js"></script>
|
||||
<script src="/test/core/data_Highway.js"></script>
|
||||
<script src="/test/core/Highway.js"></script>
|
||||
<script src="/test/core/data_MaxoutDense.js"></script>
|
||||
<script src="/test/core/MaxoutDense.js"></script>
|
||||
|
||||
<script src="/test/convolutional/data_Convolution2D.js"></script>
|
||||
<script src="/test/convolutional/Convolution2D.js"></script>
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using TensorFlow backend.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from keras.models import Model\n",
|
||||
"from keras.layers import Input\n",
|
||||
"from keras.layers.core import Highway\n",
|
||||
"from keras import backend as K"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_decimal(arr, places=6):\n",
|
||||
" return [round(x * 10**places) / 10**places for x in arr]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Highway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[core.Highway.0]**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W shape: (6, 6)\n",
|
||||
"W: [0.176262, 0.795427, 0.783061, 0.631675, -0.928221, 0.383515, -0.242638, 0.037022, 0.315903, -0.6123, -0.455367, 0.437212, 0.566007, 0.700655, 0.55049, -0.926671, -0.766613, 0.502561, -0.521564, -0.490388, 0.715251, 0.899558, 0.123374, -0.642439, 0.540504, -0.015238, 0.262506, 0.678996, -0.077921, -0.00412, 0.358822, 0.301572, -0.46241, -0.865351, 0.54289, -0.038032]\n",
|
||||
"b shape: (6,)\n",
|
||||
"b: [-0.90255, -0.421781, 0.441933, -0.956768, -0.588154, -0.898453]\n",
|
||||
"W_carry shape: (6, 6)\n",
|
||||
"W_carry: [-0.583079, -0.036638, -0.158924, 0.718364, -0.657677, -0.322272, -0.458934, 0.382083, -0.559191, 0.623902, -0.978946, 0.122407, 0.627452, 0.490201, -0.621777, -0.987718, 0.544088, 0.915664, 0.403876, -0.404843, 0.535985, 0.376437, -0.225633, 0.230412, -0.14489, 0.168579, 0.405271, -0.77621, 0.84654, 0.977773, 0.354822, 0.59033, -0.941847, -0.644482, 0.749855, 0.489864]\n",
|
||||
"b_carry shape: (6,)\n",
|
||||
"b_carry: [0.034596, 0.893925, 0.53092, -0.435208, -0.557909, 0.372444]\n",
|
||||
"\n",
|
||||
"in shape: (6,)\n",
|
||||
"in: [-0.665722, -0.215115, 0.236105, -0.17614, -0.99507, 0.768064]\n",
|
||||
"out shape: (6,)\n",
|
||||
"out: [-0.914347, -0.408456, -0.114281, -0.888056, -0.290505, -0.199544]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (6,)\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = Highway(transform_bias=-2, activation='linear', bias=True)(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(20+i)\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('W shape:', weights[0].shape)\n",
|
||||
"print('W:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('b shape:', weights[1].shape)\n",
|
||||
"print('b:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('W_carry shape:', weights[2].shape)\n",
|
||||
"print('W_carry:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('b_carry shape:', weights[3].shape)\n",
|
||||
"print('b_carry:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[core.Highway.1]**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W shape: (5, 5)\n",
|
||||
"W: [0.288287, -0.238503, 0.326096, -0.672699, 0.925216, -0.306676, 0.983502, -0.529884, 0.171389, -0.18662, -0.727531, 0.088273, 0.036353, 0.53371, 0.8677, -0.820593, -0.608457, 0.988387, -0.529639, -0.522027, 0.2582, 0.469905, 0.376689, -0.937739, 0.805028]\n",
|
||||
"b shape: (5,)\n",
|
||||
"b: [-0.427892, 0.916211, 0.540626, 0.97374, -0.583669]\n",
|
||||
"W_carry shape: (5, 5)\n",
|
||||
"W_carry: [0.717779, -0.254578, 0.110258, 0.911313, 0.473339, 0.63241, -0.797827, 0.856976, 0.218218, 0.193107, -0.816432, -0.309628, 0.325505, -0.116573, 0.102976, 0.407425, 0.178802, -0.900134, 0.123584, 0.532717, 0.821817, -0.81418, 0.805043, -0.078079, -0.095963]\n",
|
||||
"b_carry shape: (5,)\n",
|
||||
"b_carry: [-0.50298, -0.100049, -0.178118, -0.479401, 0.740791]\n",
|
||||
"\n",
|
||||
"in shape: (5,)\n",
|
||||
"in: [-0.62992, -0.960677, 0.906504, 0.360902, -0.026824]\n",
|
||||
"out shape: (5,)\n",
|
||||
"out: [-0.652907, -0.353259, 0.890362, 0.477292, -0.256096]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (5,)\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = Highway(transform_bias=-5, activation='tanh', bias=True)(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(30+i)\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('W shape:', weights[0].shape)\n",
|
||||
"print('W:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('b shape:', weights[1].shape)\n",
|
||||
"print('b:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"print('W_carry shape:', weights[2].shape)\n",
|
||||
"print('W_carry:', format_decimal(weights[2].ravel().tolist()))\n",
|
||||
"print('b_carry shape:', weights[3].shape)\n",
|
||||
"print('b_carry:', format_decimal(weights[3].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W shape: (4, 4)\n",
|
||||
"W: [-0.184626, -0.889268, 0.57707, -0.42539, -0.099299, -0.392175, 0.052799, 0.247624, 0.553551, 0.372483, 0.961878, 0.201632, 0.627937, 0.41729, -0.944931, 0.808534]\n",
|
||||
"W_carry shape: (4, 4)\n",
|
||||
"W_carry: [-0.498153, -0.907808, 0.353632, -0.913061, -0.767153, 0.207731, -0.618139, 0.337031, 0.834896, -0.16244, -0.33548, -0.433933, -0.627435, -0.365779, -0.037663, -0.860959]\n",
|
||||
"\n",
|
||||
"in shape: (4,)\n",
|
||||
"in: [0.409965, -0.370646, 0.490565, -0.203574]\n",
|
||||
"out shape: (4,)\n",
|
||||
"out: [0.482075, -0.04199, 0.593448, 0.031503]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (4,)\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = Highway(transform_bias=1, activation='hard_sigmoid', bias=False)(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for i, w in enumerate(model.get_weights()):\n",
|
||||
" np.random.seed(40+i)\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('W shape:', weights[0].shape)\n",
|
||||
"print('W:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('W_carry shape:', weights[1].shape)\n",
|
||||
"print('W_carry:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.5.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from keras.models import Model\n",
|
||||
"from keras.layers import Input\n",
|
||||
"from keras.layers.core import MaxoutDense\n",
|
||||
"from keras import backend as K"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_decimal(arr, places=6):\n",
|
||||
" return [round(x * 10**places) / 10**places for x in arr]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### MaxoutDense"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[core.MaxoutDense.0] nb_feature=4, biase=True**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W shape: (4, 6, 3)\n",
|
||||
"W: [0.542641, -0.958496, 0.267296, 0.497608, -0.002986, -0.550407, -0.603874, 0.521061, -0.661778, -0.82332, 0.37072, 0.906787, -0.992103, 0.024385, 0.625242, 0.225052, 0.443511, -0.416248, 0.835548, 0.429152, 0.085089, -0.71566, -0.253318, 0.348267, -0.116334, -0.131972, 0.235534, 0.026276, 0.300794, 0.202078, 0.610446, 0.043294, 0.817298, -0.361528, -0.819081, -0.3986, -0.772031, 0.657363, -0.906207, 0.252574, 0.095172, 0.638574, -0.602105, 0.713701, -0.296695, 0.509295, -0.408077, 0.767873, -0.348977, -0.669968, -0.214942, -0.813079, 0.642211, -0.697696, -0.231771, 0.888521, 0.975251, -0.087391, 0.652246, -0.497252, 0.194743, 0.805664, 0.069116, 0.180403, -0.921436, -0.285636, -0.840774, -0.38908, -0.338561, 0.547661, -0.920082, -0.141016]\n",
|
||||
"b shape: (4, 3)\n",
|
||||
"b: [0.542641, -0.958496, 0.267296, 0.497608, -0.002986, -0.550407, -0.603874, 0.521061, -0.661778, -0.82332, 0.37072, 0.906787]\n",
|
||||
"\n",
|
||||
"in shape: (6,)\n",
|
||||
"in: [-0.992103, 0.024385, 0.625242, 0.225052, 0.443511, -0.416248]\n",
|
||||
"out shape: (3,)\n",
|
||||
"out: [0.090044, 0.227783, 0.435236]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (6,)\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = MaxoutDense(3, nb_feature=4, bias=True)(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for w in model.get_weights():\n",
|
||||
" np.random.seed(10)\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('W shape:', weights[0].shape)\n",
|
||||
"print('W:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"print('b shape:', weights[1].shape)\n",
|
||||
"print('b:', format_decimal(weights[1].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**[core.MaxoutDense.1] nb_feature=7, biase=False**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W shape: (7, 6, 3)\n",
|
||||
"W: [-0.639461, -0.96105, -0.073563, 0.449868, -0.159593, -0.029146, -0.974438, -0.025257, 0.883613, 0.70159, 0.459929, -0.782528, 0.787808, 0.714308, -0.669827, 0.264668, -0.959033, -0.766525, -0.367265, -0.684175, 0.517959, 0.636551, -0.310751, -0.362402, -0.776678, -0.832094, 0.425452, 0.199087, -0.888653, -0.040405, -0.196647, 0.695958, 0.435698, 0.204128, 0.104768, 0.898205, 0.973347, -0.323892, -0.520251, 0.592872, -0.872627, -0.270769, -0.859954, -0.361265, -0.859235, -0.419473, 0.580202, 0.810801, 0.585243, 0.123637, 0.232037, -0.277033, -0.662365, -0.127518, 0.465651, -0.874225, -0.958534, 0.541096, -0.400096, 0.402329, 0.469335, 0.865809, -0.199343, -0.283124, 0.613134, 0.528982, 0.305229, 0.621933, 0.28443, 0.914888, -0.332251, 0.476505, 0.899667, -0.331272, 0.223264, -0.26866, -0.076919, -0.849996, -0.961313, 0.519299, -0.094482, 0.245668, 0.479903, -0.162673, -0.264765, -0.661942, 0.587745, 0.666075, 0.468426, 0.750589, 0.296113, 0.392132, -0.613964, 0.535264, -0.331834, -0.124082, -0.36226, 0.136579, 0.317385, 0.151117, -0.363626, -0.563993, 0.689877, -0.395671, -0.123559, -0.817171, -0.398042, -0.82861, -0.312573, 0.418602, 0.947937, 0.25045, -0.461393, 0.313418, 0.253996, 0.651304, 0.067388, 0.822243, -0.168993, -0.427911, 0.038015, 0.8449, 0.447823, -0.012174, -0.005007, 0.297444]\n",
|
||||
"\n",
|
||||
"in shape: (6,)\n",
|
||||
"in: [-0.104458, 0.101279, 0.94235, 0.864827, 0.681371, -0.745903]\n",
|
||||
"out shape: (3,)\n",
|
||||
"out: [1.043451, 2.068543, 0.396771]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_in_shape = (6,)\n",
|
||||
"layer_0 = Input(shape=data_in_shape)\n",
|
||||
"layer_1 = MaxoutDense(3, nb_feature=7, bias=False)(layer_0)\n",
|
||||
"model = Model(input=layer_0, output=layer_1)\n",
|
||||
"\n",
|
||||
"# set weights to random (use seed for reproducibility)\n",
|
||||
"weights = []\n",
|
||||
"for w in model.get_weights():\n",
|
||||
" np.random.seed(11)\n",
|
||||
" weights.append(2 * np.random.random(w.shape) - 1)\n",
|
||||
"model.set_weights(weights)\n",
|
||||
"print('W shape:', weights[0].shape)\n",
|
||||
"print('W:', format_decimal(weights[0].ravel().tolist()))\n",
|
||||
"\n",
|
||||
"data_in = 2 * np.random.random(data_in_shape) - 1\n",
|
||||
"print('')\n",
|
||||
"print('in shape:', data_in_shape)\n",
|
||||
"print('in:', format_decimal(data_in.ravel().tolist()))\n",
|
||||
"result = model.predict(np.array([data_in]))\n",
|
||||
"print('out shape:', result[0].shape)\n",
|
||||
"print('out:', format_decimal(result[0].ravel().tolist()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.5.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import * as activations from '../../activations'
|
||||
import Tensor from '../../Tensor'
|
||||
import Layer from '../../engine/Layer'
|
||||
import { gemv } from 'ndarray-blas-level2'
|
||||
import ops from 'ndarray-ops'
|
||||
import cwise from 'cwise'
|
||||
|
||||
/**
|
||||
* Highway layer class
|
||||
* From Keras docs: Densely connected highway network, a natural extension of LSTMs to feedforward networks.
|
||||
*/
|
||||
export default class Highway extends Layer {
|
||||
/**
|
||||
* Creates a Highway layer
|
||||
* @param {number} outputDim - output dimension size
|
||||
* @param {Object} [attrs] - layer attributes
|
||||
*/
|
||||
constructor (attrs = {}) {
|
||||
super(attrs)
|
||||
const {
|
||||
transformBias = -2,
|
||||
activation = 'linear',
|
||||
bias = true
|
||||
} = attrs
|
||||
|
||||
this.transformBias = transformBias
|
||||
this.activation = activations[activation]
|
||||
this.bias = bias
|
||||
|
||||
/**
|
||||
* Layer weights specification
|
||||
*/
|
||||
this.params = this.bias ? ['W', 'b', 'W_carry', 'b_carry'] : ['W', 'W_carry']
|
||||
}
|
||||
|
||||
_computeOutput = cwise({
|
||||
args: ['array', 'array', 'array'],
|
||||
body: function (_x, _y, _transform) {
|
||||
_x = _y * _transform + (1 - _transform) * _x
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Method for layer computational logic
|
||||
* @param {Tensor} x
|
||||
* @returns {Tensor} x
|
||||
*/
|
||||
call = x => {
|
||||
let y = new Tensor([], [this.weights.W.tensor.shape[1]])
|
||||
if (this.bias) {
|
||||
ops.assign(y.tensor, this.weights.b.tensor)
|
||||
}
|
||||
gemv(1.0, this.weights.W.tensor.transpose(1, 0), x.tensor, 1.0, y.tensor)
|
||||
this.activation(y)
|
||||
|
||||
let transform = new Tensor([], [this.weights.W_carry.tensor.shape[1]])
|
||||
if (this.bias) {
|
||||
ops.assign(transform.tensor, this.weights.b_carry.tensor)
|
||||
}
|
||||
gemv(1.0, this.weights.W_carry.tensor.transpose(1, 0), x.tensor, 1.0, transform.tensor)
|
||||
activations.sigmoid(transform)
|
||||
|
||||
this._computeOutput(x.tensor, y.tensor, transform.tensor)
|
||||
|
||||
return x
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import Tensor from '../../Tensor'
|
||||
import Layer from '../../engine/Layer'
|
||||
import { gemv } from 'ndarray-blas-level2'
|
||||
import ops from 'ndarray-ops'
|
||||
|
||||
/**
|
||||
* MaxoutDense layer class
|
||||
* From Keras docs: takes the element-wise maximum of nb_feature Dense(input_dim, output_dim) linear layers
|
||||
* Note that `nb_feature` is implicit in the weights tensors, with shapes:
|
||||
* - W: [nb_feature, input_dim, output_dim]
|
||||
* - b: [nb_feature, output_dim]
|
||||
*/
|
||||
export default class MaxoutDense extends Layer {
|
||||
/**
|
||||
* Creates a MaxoutDense layer
|
||||
* @param {number} outputDim - output dimension size
|
||||
* @param {Object} [attrs] - layer attributes
|
||||
*/
|
||||
constructor (outputDim, attrs = {}) {
|
||||
super(attrs)
|
||||
const {
|
||||
inputDim = null,
|
||||
bias = true
|
||||
} = attrs
|
||||
this.outputDim = outputDim
|
||||
this.inputDim = inputDim
|
||||
this.bias = bias
|
||||
|
||||
/**
|
||||
* Layer weights specification
|
||||
*/
|
||||
this.params = this.bias ? ['W', 'b'] : ['W']
|
||||
}
|
||||
|
||||
/**
|
||||
* Method for layer computational logic
|
||||
* @param {Tensor} x
|
||||
* @returns {Tensor} x
|
||||
*/
|
||||
call = x => {
|
||||
const nbFeature = this.weights.W.tensor.shape[0]
|
||||
|
||||
let featMax = new Tensor([], [this.outputDim])
|
||||
for (let i = 0; i < nbFeature; i++) {
|
||||
let y = new Tensor([], [this.outputDim])
|
||||
if (this.bias) {
|
||||
ops.assign(y.tensor, this.weights.b.tensor.pick(i, null))
|
||||
}
|
||||
gemv(1.0, this.weights.W.tensor.pick(i, null, null).transpose(1, 0), x.tensor, 1.0, y.tensor)
|
||||
ops.maxeq(featMax.tensor, y.tensor)
|
||||
}
|
||||
|
||||
x.tensor = featMax.tensor
|
||||
return x
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import Reshape from './Reshape'
|
||||
import Permute from './Permute'
|
||||
import RepeatVector from './RepeatVector'
|
||||
import Merge from './Merge'
|
||||
import Highway from './Highway'
|
||||
import MaxoutDense from './MaxoutDense'
|
||||
|
||||
export {
|
||||
Dense,
|
||||
@@ -15,5 +17,7 @@ export {
|
||||
Reshape,
|
||||
Permute,
|
||||
RepeatVector,
|
||||
Merge
|
||||
Merge,
|
||||
Highway,
|
||||
MaxoutDense
|
||||
}
|
||||
|
||||
+2
-2
@@ -18,7 +18,7 @@ describe('core layer: Dense', function () {
|
||||
|
||||
describe('CPU', function () {
|
||||
before(function () {
|
||||
console.log('\n%cDense', styles.h2)
|
||||
console.log('\n%cCPU', styles.h2)
|
||||
})
|
||||
|
||||
it('[core.Dense.0] [CPU] should produce expected values', function () {
|
||||
@@ -82,7 +82,7 @@ describe('core layer: Dense', function () {
|
||||
|
||||
describe('CPU', function () {
|
||||
before(function () {
|
||||
console.log('\n%cDense', styles.h2)
|
||||
console.log('\n%cGPU', styles.h2)
|
||||
})
|
||||
|
||||
it('[core.Dense.3] [GPU] should produce expected values', function () {
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/* eslint-env browser, mocha */
|
||||
|
||||
describe('core layer: Highway', function () {
|
||||
const assert = chai.assert
|
||||
const styles = testGlobals.styles
|
||||
const logTime = testGlobals.logTime
|
||||
const stringifyCondensed = testGlobals.stringifyCondensed
|
||||
const approxEquals = KerasJS.testUtils.approxEquals
|
||||
const layers = KerasJS.layers
|
||||
|
||||
before(function () {
|
||||
console.log('\n%ccore layer: Highway', styles.h1)
|
||||
})
|
||||
|
||||
it('[core.Highway.0] should produce expected values, transformBias=-2, activation=linear bias=true', function () {
|
||||
const key = 'core.Highway.0'
|
||||
console.log(`\n%c[${key}] transformBias=-2, activation=linear bias=true`, styles.h3)
|
||||
let testLayer = new layers.Highway({ transformBias: -2, activation: 'linear', bias: true })
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
|
||||
it('[core.Highway.1] should produce expected values, transformBias=-5, activation=tanh bias=true', function () {
|
||||
const key = 'core.Highway.1'
|
||||
console.log(`\n%c[${key}] transformBias=-5, activation=tanh bias=true`, styles.h3)
|
||||
let testLayer = new layers.Highway({ transformBias: -5, activation: 'tanh', bias: true })
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
|
||||
it('[core.Highway.2] should produce expected values, transformBias=1, activation=hardSigmoid bias=false', function () {
|
||||
const key = 'core.Highway.2'
|
||||
console.log(`\n%c[${key}] transformBias=1, activation=hardSigmoid bias=false`, styles.h3)
|
||||
let testLayer = new layers.Highway({ transformBias: 1, activation: 'hardSigmoid', bias: false })
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,50 @@
|
||||
/* eslint-env browser, mocha */
|
||||
|
||||
describe('core layer: MaxoutDense', function () {
|
||||
const assert = chai.assert
|
||||
const styles = testGlobals.styles
|
||||
const logTime = testGlobals.logTime
|
||||
const stringifyCondensed = testGlobals.stringifyCondensed
|
||||
const approxEquals = KerasJS.testUtils.approxEquals
|
||||
const layers = KerasJS.layers
|
||||
|
||||
before(function () {
|
||||
console.log('\n%ccore layer: MaxoutDense', styles.h1)
|
||||
})
|
||||
|
||||
it('[core.MaxoutDense.0] should produce expected values, nbFeature=4, bias=true', function () {
|
||||
const key = 'core.MaxoutDense.0'
|
||||
console.log(`\n%c[${key}] nbFeature=4, bias=true`, styles.h3)
|
||||
let testLayer = new layers.MaxoutDense(3)
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
|
||||
it('[core.MaxoutDense.1] should produce expected values, nbFeature=7, bias=false', function () {
|
||||
const key = 'core.MaxoutDense.1'
|
||||
console.log(`\n%c[${key}] nbFeature=7, bias=false`, styles.h3)
|
||||
let testLayer = new layers.MaxoutDense(3, { bias: false })
|
||||
testLayer.setWeights(TEST_DATA[key].weights.map(w => new KerasJS.Tensor(w.data, w.shape)))
|
||||
let t = new KerasJS.Tensor(TEST_DATA[key].input.data, TEST_DATA[key].input.shape)
|
||||
console.log('%cin', styles.h4, stringifyCondensed(t.tensor))
|
||||
const startTime = performance.now()
|
||||
t = testLayer.call(t)
|
||||
const endTime = performance.now()
|
||||
console.log('%cout', styles.h4, stringifyCondensed(t.tensor))
|
||||
logTime(startTime, endTime)
|
||||
const dataExpected = new Float32Array(TEST_DATA[key].expected.data)
|
||||
const shapeExpected = TEST_DATA[key].expected.shape
|
||||
assert.deepEqual(t.tensor.shape, shapeExpected)
|
||||
assert.isTrue(approxEquals(t.tensor, dataExpected))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
// TEST DATA
|
||||
// Keyed by mocha test ID
|
||||
// Python code for generating test data can be found in the matching jupyter notebook in folder `notebooks/`.
|
||||
|
||||
(function () {
|
||||
var DATA = {
|
||||
'core.Highway.0': {
|
||||
input: {
|
||||
data: [-0.665722, -0.215115, 0.236105, -0.17614, -0.99507, 0.768064],
|
||||
shape: [6]
|
||||
},
|
||||
weights: [
|
||||
{
|
||||
data: [0.176262, 0.795427, 0.783061, 0.631675, -0.928221, 0.383515, -0.242638, 0.037022, 0.315903, -0.6123, -0.455367, 0.437212, 0.566007, 0.700655, 0.55049, -0.926671, -0.766613, 0.502561, -0.521564, -0.490388, 0.715251, 0.899558, 0.123374, -0.642439, 0.540504, -0.015238, 0.262506, 0.678996, -0.077921, -0.00412, 0.358822, 0.301572, -0.46241, -0.865351, 0.54289, -0.038032],
|
||||
shape: [6, 6]
|
||||
},
|
||||
{
|
||||
data: [-0.90255, -0.421781, 0.441933, -0.956768, -0.588154, -0.898453],
|
||||
shape: [6]
|
||||
},
|
||||
{
|
||||
data: [-0.583079, -0.036638, -0.158924, 0.718364, -0.657677, -0.322272, -0.458934, 0.382083, -0.559191, 0.623902, -0.978946, 0.122407, 0.627452, 0.490201, -0.621777, -0.987718, 0.544088, 0.915664, 0.403876, -0.404843, 0.535985, 0.376437, -0.225633, 0.230412, -0.14489, 0.168579, 0.405271, -0.77621, 0.84654, 0.977773, 0.354822, 0.59033, -0.941847, -0.644482, 0.749855, 0.489864],
|
||||
shape: [6, 6]
|
||||
},
|
||||
{
|
||||
data: [0.034596, 0.893925, 0.53092, -0.435208, -0.557909, 0.372444],
|
||||
shape: [6]
|
||||
}
|
||||
],
|
||||
expected: {
|
||||
data: [-0.914347, -0.408456, -0.114281, -0.888056, -0.290505, -0.199544],
|
||||
shape: [6]
|
||||
}
|
||||
},
|
||||
'core.Highway.1': {
|
||||
input: {
|
||||
data: [-0.62992, -0.960677, 0.906504, 0.360902, -0.026824],
|
||||
shape: [5]
|
||||
},
|
||||
weights: [
|
||||
{
|
||||
data: [0.288287, -0.238503, 0.326096, -0.672699, 0.925216, -0.306676, 0.983502, -0.529884, 0.171389, -0.18662, -0.727531, 0.088273, 0.036353, 0.53371, 0.8677, -0.820593, -0.608457, 0.988387, -0.529639, -0.522027, 0.2582, 0.469905, 0.376689, -0.937739, 0.805028],
|
||||
shape: [5, 5]
|
||||
},
|
||||
{
|
||||
data: [-0.427892, 0.916211, 0.540626, 0.97374, -0.583669],
|
||||
shape: [5]
|
||||
},
|
||||
{
|
||||
data: [0.717779, -0.254578, 0.110258, 0.911313, 0.473339, 0.63241, -0.797827, 0.856976, 0.218218, 0.193107, -0.816432, -0.309628, 0.325505, -0.116573, 0.102976, 0.407425, 0.178802, -0.900134, 0.123584, 0.532717, 0.821817, -0.81418, 0.805043, -0.078079, -0.095963],
|
||||
shape: [5, 5]
|
||||
},
|
||||
{
|
||||
data: [-0.50298, -0.100049, -0.178118, -0.479401, 0.740791],
|
||||
shape: [5]
|
||||
}
|
||||
],
|
||||
expected: {
|
||||
data: [-0.652907, -0.353259, 0.890362, 0.477292, -0.256096],
|
||||
shape: [5]
|
||||
}
|
||||
},
|
||||
'core.Highway.2': {
|
||||
input: {
|
||||
data: [0.409965, -0.370646, 0.490565, -0.203574],
|
||||
shape: [4]
|
||||
},
|
||||
weights: [
|
||||
{
|
||||
data: [-0.184626, -0.889268, 0.57707, -0.42539, -0.099299, -0.392175, 0.052799, 0.247624, 0.553551, 0.372483, 0.961878, 0.201632, 0.627937, 0.41729, -0.944931, 0.808534],
|
||||
shape: [4, 4]
|
||||
},
|
||||
{
|
||||
data: [-0.498153, -0.907808, 0.353632, -0.913061, -0.767153, 0.207731, -0.618139, 0.337031, 0.834896, -0.16244, -0.33548, -0.433933, -0.627435, -0.365779, -0.037663, -0.860959],
|
||||
shape: [4, 4]
|
||||
}
|
||||
],
|
||||
expected: {
|
||||
data: [0.482075, -0.04199, 0.593448, 0.031503],
|
||||
shape: [4]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
window.TEST_DATA = Object.assign({}, window.TEST_DATA, DATA)
|
||||
})()
|
||||
@@ -0,0 +1,46 @@
|
||||
// TEST DATA
|
||||
// Keyed by mocha test ID
|
||||
// Python code for generating test data can be found in the matching jupyter notebook in folder `notebooks/`.
|
||||
|
||||
(function () {
|
||||
var DATA = {
|
||||
'core.MaxoutDense.0': {
|
||||
input: {
|
||||
data: [-0.992103, 0.024385, 0.625242, 0.225052, 0.443511, -0.416248],
|
||||
shape: [6]
|
||||
},
|
||||
weights: [
|
||||
{
|
||||
data: [0.542641, -0.958496, 0.267296, 0.497608, -0.002986, -0.550407, -0.603874, 0.521061, -0.661778, -0.82332, 0.37072, 0.906787, -0.992103, 0.024385, 0.625242, 0.225052, 0.443511, -0.416248, 0.835548, 0.429152, 0.085089, -0.71566, -0.253318, 0.348267, -0.116334, -0.131972, 0.235534, 0.026276, 0.300794, 0.202078, 0.610446, 0.043294, 0.817298, -0.361528, -0.819081, -0.3986, -0.772031, 0.657363, -0.906207, 0.252574, 0.095172, 0.638574, -0.602105, 0.713701, -0.296695, 0.509295, -0.408077, 0.767873, -0.348977, -0.669968, -0.214942, -0.813079, 0.642211, -0.697696, -0.231771, 0.888521, 0.975251, -0.087391, 0.652246, -0.497252, 0.194743, 0.805664, 0.069116, 0.180403, -0.921436, -0.285636, -0.840774, -0.38908, -0.338561, 0.547661, -0.920082, -0.141016],
|
||||
shape: [4, 6, 3]
|
||||
},
|
||||
{
|
||||
data: [0.542641, -0.958496, 0.267296, 0.497608, -0.002986, -0.550407, -0.603874, 0.521061, -0.661778, -0.82332, 0.37072, 0.906787],
|
||||
shape: [4, 3]
|
||||
}
|
||||
],
|
||||
expected: {
|
||||
data: [0.090044, 0.227783, 0.435236],
|
||||
shape: [3]
|
||||
}
|
||||
},
|
||||
'core.MaxoutDense.1': {
|
||||
input: {
|
||||
data: [-0.104458, 0.101279, 0.94235, 0.864827, 0.681371, -0.745903],
|
||||
shape: [6]
|
||||
},
|
||||
weights: [
|
||||
{
|
||||
data: [-0.639461, -0.96105, -0.073563, 0.449868, -0.159593, -0.029146, -0.974438, -0.025257, 0.883613, 0.70159, 0.459929, -0.782528, 0.787808, 0.714308, -0.669827, 0.264668, -0.959033, -0.766525, -0.367265, -0.684175, 0.517959, 0.636551, -0.310751, -0.362402, -0.776678, -0.832094, 0.425452, 0.199087, -0.888653, -0.040405, -0.196647, 0.695958, 0.435698, 0.204128, 0.104768, 0.898205, 0.973347, -0.323892, -0.520251, 0.592872, -0.872627, -0.270769, -0.859954, -0.361265, -0.859235, -0.419473, 0.580202, 0.810801, 0.585243, 0.123637, 0.232037, -0.277033, -0.662365, -0.127518, 0.465651, -0.874225, -0.958534, 0.541096, -0.400096, 0.402329, 0.469335, 0.865809, -0.199343, -0.283124, 0.613134, 0.528982, 0.305229, 0.621933, 0.28443, 0.914888, -0.332251, 0.476505, 0.899667, -0.331272, 0.223264, -0.26866, -0.076919, -0.849996, -0.961313, 0.519299, -0.094482, 0.245668, 0.479903, -0.162673, -0.264765, -0.661942, 0.587745, 0.666075, 0.468426, 0.750589, 0.296113, 0.392132, -0.613964, 0.535264, -0.331834, -0.124082, -0.36226, 0.136579, 0.317385, 0.151117, -0.363626, -0.563993, 0.689877, -0.395671, -0.123559, -0.817171, -0.398042, -0.82861, -0.312573, 0.418602, 0.947937, 0.25045, -0.461393, 0.313418, 0.253996, 0.651304, 0.067388, 0.822243, -0.168993, -0.427911, 0.038015, 0.8449, 0.447823, -0.012174, -0.005007, 0.297444],
|
||||
shape: [7, 6, 3]
|
||||
}
|
||||
],
|
||||
expected: {
|
||||
data: [1.043451, 2.068543, 0.396771],
|
||||
shape: [3]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
window.TEST_DATA = Object.assign({}, window.TEST_DATA, DATA)
|
||||
})()
|
||||
Reference in New Issue
Block a user