diff --git a/notebooks/recurrent/LSTM.ipynb b/notebooks/recurrent/LSTM.ipynb new file mode 100644 index 0000000..3b1c0c4 --- /dev/null +++ b/notebooks/recurrent/LSTM.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras.models import Model\n", + "from keras.layers import Input\n", + "from keras.layers.recurrent import LSTM\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def format_decimal(arr, places=6):\n", + " return [round(x * 10**places) / 10**places for x in arr]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### LSTM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[recurrent.LSTM.0] output_dim=4, activation='tanh', inner_activation='hard_sigmoid'**\n", + "\n", + "Note dropout_W and dropout_U are only applied during training phase" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W_i shape: (6, 4)\n", + "W_i: [0.021587, 0.171208, 0.42254, -0.348744, 0.040626, 0.479164, 0.684427, -0.694989, 0.064855, 0.056754, -0.076463, 0.184194, -0.793392, 0.627222, -0.855503, -0.362412, 0.855828, 0.681804, -0.326362, -0.218483, 0.302837, 0.321287, -0.052374, 0.066035]\n", + "U_i shape: (4, 4)\n", + "U_i: [-0.486398, -0.924962, 0.924244, -0.099505, 0.697209, -0.988196, 0.312034, -0.82215, -0.450764, -0.702362, 0.86183, -0.505312, 0.224025, -0.210654, -0.378863, 0.270893]\n", + "b_i shape: (4,)\n", + "b_i: [-0.752966, 0.132946, 0.249025, -0.746897]\n", + "W_c shape: (6, 4)\n", + "W_c: [-0.321918, 0.410663, -0.775844, -0.651728, 0.078787, 0.493956, -0.207594, -0.280414, -0.172642, -0.673013, -0.577208, -0.485994, 0.501094, 0.138545, 0.010672, 0.470218, -0.560627, 0.188022, 0.383913, 0.692257, -0.360569, -0.077626, -0.009927, 0.454896]\n", + "U_c shape: (4, 4)\n", + "U_c: [-0.008879, -0.535629, 0.500476, -0.443752, -0.370062, 0.766849, 0.348937, 0.530872, 0.151555, -0.41474, -0.538834, -0.242793, -0.809167, 0.950484, 0.326277, -0.861456]\n", + "b_c shape: (4,)\n", + "b_c: [-0.852859, 0.946774, 0.323797, -0.109777]\n", + "W_f shape: (6, 4)\n", + "W_f: [-0.793815, -0.518475, -0.579862, 0.302531, 0.680386, 0.692198, -0.682758, -0.752123, 0.743339, 0.221127, 0.042567, 0.215791, -0.256099, 0.022924, 0.157546, 0.535038, -0.591736, 0.443027, -0.51415, 0.467614, -0.391062, 0.993584, 0.965207, 0.509182]\n", + "U_f shape: (4, 4)\n", + "U_f: [0.476861, -0.902137, 0.156377, 0.778033, 0.679346, 0.284676, 0.042898, 0.289311, -0.861268, -0.784418, 0.932704, -0.825608, -0.765997, -0.944845, -0.980398, -0.006388]\n", + "b_f shape: (4,)\n", + "b_f: [0.824388, -0.068399, 0.018677, -0.595055]\n", + "W_o shape: (6, 4)\n", + "W_o: [0.180806, -0.699554, -0.809467, -0.616054, -0.217633, 0.690901, 0.098324, 0.76948, -0.122458, 0.011661, 0.377493, -0.022008, -0.898668, -0.569468, -0.162894, 0.853575, 0.607888, -0.742334, 0.593786, -0.223577, -0.626006, 0.871527, 0.493996, -0.143212]\n", + "U_o shape: (4, 4)\n", + "U_o: [0.672312, -0.799748, 0.482037, -0.833993, 0.825852, -0.939344, -0.227248, -0.287931, 0.996954, 0.391395, 0.122945, -0.582373, 0.170427, 0.895754, -0.548469, 0.313066]\n", + "b_o shape: (4,)\n", + "b_o: [-0.590664, 0.408132, 0.246375, -0.796643]\n", + "\n", + "in shape: (3, 6)\n", + "in: [0.665517, 0.700148, 0.455626, 0.804744, -0.880561, -0.520282, 0.574518, -0.675696, -0.97577, -0.240988, 0.80853, 0.237721, -0.97565, 0.164267, -0.003728, 0.882893, 0.121681, -0.925775]\n", + "out shape: (4,)\n", + "out: [-0.12453, 0.324853, 0.173839, 0.145323]\n" + ] + } + ], + "source": [ + "data_in_shape = (3, 6)\n", + "rnn = LSTM(4, activation='tanh', inner_activation='hard_sigmoid')\n", + "\n", + "layer_0 = Input(shape=data_in_shape)\n", + "layer_1 = rnn(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for i, w in enumerate(model.get_weights()):\n", + " np.random.seed(3000 + i)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "weight_names = ['W_i', 'U_i', 'b_i', 'W_c', 'U_c', 'b_c', 'W_f', 'U_f', 'b_f', 'W_o', 'U_o', 'b_o']\n", + "for w_i, w_name in enumerate(weight_names):\n", + " print('{} shape:'.format(w_name), weights[w_i].shape)\n", + " print('{}:'.format(w_name), format_decimal(weights[w_i].ravel().tolist()))\n", + "\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[recurrent.LSTM.1] output_dim=5, activation='sigmoid', inner_activation='sigmoid'**\n", + "\n", + "Note dropout_W and dropout_U are only applied during training phase" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W_i shape: (5, 5)\n", + "W_i: [0.904105, -0.877164, 0.33871, 0.812244, -0.802633, 0.595832, -0.981528, 0.378119, 0.41493, -0.11458, -0.692809, 0.875448, 0.318268, -0.627292, 0.941065, 0.916899, -0.78002, 0.54744, -0.282447, -0.214613, -0.087847, -0.404508, -0.411046, -0.425703, -0.603508]\n", + "U_i shape: (5, 5)\n", + "U_i: [-0.708522, -0.706998, 0.233695, 0.906727, -0.986561, 0.797093, -0.065787, 0.680991, 0.839202, 0.009629, -0.39912, -0.935261, 0.187645, 0.505894, -0.292493, 0.909563, -0.869531, -0.741938, -0.617129, -0.750299, 0.78066, -0.489417, 0.306455, -0.684443, -0.097986]\n", + "b_i shape: (5,)\n", + "b_i: [0.351652, 0.052999, 0.049473, -0.569237, 0.204228]\n", + "W_c shape: (5, 5)\n", + "W_c: [0.067039, 0.428238, -0.349015, -0.542575, 0.975082, 0.727611, 0.470814, -0.569386, 0.338722, 0.972686, 0.114987, 0.967292, 0.424026, 0.922109, -0.380777, -0.89206, -0.144607, -0.091346, -0.419741, -0.779593, -0.104814, -0.079318, 0.377633, 0.490806, -0.452043]\n", + "U_c shape: (5, 5)\n", + "U_c: [-0.174106, -0.833394, 0.381703, -0.843934, -0.236694, -0.36737, -0.367741, 0.837889, -0.039786, 0.361505, 0.286408, 0.273419, -0.69914, -0.82985, -0.459621, -0.559184, -0.869297, 0.514745, 0.614001, 0.001062, 0.051854, -0.369003, 0.30649, 0.899397, 0.365186]\n", + "b_c shape: (5,)\n", + "b_c: [-0.585018, 0.56124, 0.203839, -0.185823, -0.172334]\n", + "W_f shape: (5, 5)\n", + "W_f: [0.928699, -0.755814, -0.167684, -0.11633, 0.984822, 0.163734, 0.294156, 0.159899, 0.429806, 0.120375, 0.536481, 0.315275, 0.334484, 0.385015, 0.303987, -0.500827, -0.317092, -0.562572, 0.939739, 0.402176, -0.220776, 0.177354, -0.29364, -0.109566, 0.317174]\n", + "U_f shape: (5, 5)\n", + "U_f: [-0.293165, 0.91974, -0.295233, -0.963946, 0.96014, -0.400338, 0.690912, -0.817071, -0.544351, 0.133677, 0.44247, -0.787779, 0.705348, -0.41608, -0.298659, 0.277014, -0.727674, -0.965871, 0.773519, 0.03901, -0.824034, -0.962203, 0.307807, 0.422613, -0.849913]\n", + "b_f shape: (5,)\n", + "b_f: [-0.517725, 0.941461, 0.58815, -0.186298, -0.946213]\n", + "W_o shape: (5, 5)\n", + "W_o: [0.35193, -0.016184, -0.950798, 0.227653, -0.950189, -0.244107, 0.114508, -0.716402, -0.688769, 0.253202, 0.438141, -0.883068, -0.305492, 0.466456, -0.971678, -0.687383, 0.970269, 0.5688, -0.05607, 0.245652, -0.849934, 0.141141, -0.50186, 0.8381, 0.615684]\n", + "U_o shape: (5, 5)\n", + "U_o: [0.737993, -0.987905, 0.734592, 0.869378, -0.219359, 0.16681, 0.618012, 0.167843, -0.129422, -0.743455, 0.398148, 0.849206, -0.941554, 0.796703, 0.939108, 0.23178, -0.652339, 0.019098, -0.925936, -0.892392, 0.785007, 0.794477, -0.682383, 0.542452, 0.94182]\n", + "b_o shape: (5,)\n", + "b_o: [-0.80119, -0.093136, -0.781478, 0.664981, 0.81755]\n", + "\n", + "in shape: (8, 5)\n", + "in: [0.218071, -0.494078, 0.765253, 0.73909, -0.631062, 0.886984, -0.606791, -0.153821, -0.772063, 0.758903, -0.855796, -0.307768, 0.947849, 0.654099, 0.100526, -0.662352, -0.969299, 0.120011, -0.356979, 0.062594, 0.727154, -0.336611, -0.870729, -0.949392, -0.881842, 0.731806, 0.361095, -0.168111, 0.835201, 0.10262, -0.76506, -0.680559, 0.56447, 0.52546, 0.503997, -0.947011, -0.578522, 0.289283, 0.649655, 0.453943]\n", + "out shape: (5,)\n", + "out: [0.143347, 0.400829, 0.363268, 0.449314, 0.438799]\n" + ] + } + ], + "source": [ + "data_in_shape = (8, 5)\n", + "rnn = LSTM(5, activation='sigmoid', inner_activation='sigmoid')\n", + "\n", + "layer_0 = Input(shape=data_in_shape)\n", + "layer_1 = rnn(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for i, w in enumerate(model.get_weights()):\n", + " np.random.seed(3100 + i)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "weight_names = ['W_i', 'U_i', 'b_i', 'W_c', 'U_c', 'b_c', 'W_f', 'U_f', 'b_f', 'W_o', 'U_o', 'b_o']\n", + "for w_i, w_name in enumerate(weight_names):\n", + " print('{} shape:'.format(w_name), weights[w_i].shape)\n", + " print('{}:'.format(w_name), format_decimal(weights[w_i].ravel().tolist()))\n", + "\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/src/layers/index.js b/src/layers/index.js index 4c92d7a..cf61b1c 100644 --- a/src/layers/index.js +++ b/src/layers/index.js @@ -4,3 +4,4 @@ export * from './convolutional' export * from './pooling' export * from './normalization' export * from './embeddings' +export * from './recurrent' diff --git a/src/layers/recurrent/LSTM.js b/src/layers/recurrent/LSTM.js new file mode 100644 index 0000000..0540487 --- /dev/null +++ b/src/layers/recurrent/LSTM.js @@ -0,0 +1,134 @@ +import * as activations from '../../activations' +import Tensor from '../../Tensor' +import Layer from '../../Layer' +import { gemv } from 'ndarray-blas-level2' +import ops from 'ndarray-ops' +import cwise from 'cwise' + +/** + * LSTM layer class + */ +export default class LSTM extends Layer { + /** + * Creates a LSTM layer + * @param {number} attrs.outputDim - Number of convolution filters to use. + * @param {Object} [attrs] - layer attributes + */ + constructor (attrs = {}) { + super(attrs) + this.layerClass = 'LSTM' + + const { + outputDim = 1, + activation = 'tanh', + innerActivation = 'hardSigmoid' + } = attrs + + this.outputDim = outputDim + + this.activation = activations[activation] + this.innerActivation = activations[innerActivation] + + // Layer weights specification + this.params = ['W_i', 'U_i', 'b_i', 'W_c', 'U_c', 'b_c', 'W_f', 'U_f', 'b_f', 'W_o', 'U_o', 'b_o'] + } + + _combine = cwise({ + args: ['array', 'array', 'array', 'array'], + body: function (_y, _x1, _x2, _b) { + _y = _x1 + _x2 + _b + } + }) + + _update = cwise({ + args: ['array', 'array', 'array', 'array'], + body: function (_c, _ctm1, _i, _f) { + _c = _c * _i + _ctm1 * _f + } + }) + + /** + * Method for layer computational logic + * @param {Tensor} x + * @returns {Tensor} x + */ + call (x) { + let currentX = new Tensor([], [x.tensor.shape[1]]) + + const dimInputGate = this.weights['b_i'].tensor.shape[0] + const dimCandidate = this.weights['b_c'].tensor.shape[0] + const dimForgetGate = this.weights['b_f'].tensor.shape[0] + const dimOutputGate = this.weights['b_o'].tensor.shape[0] + + let currentInputGateState = new Tensor([], [dimInputGate]) + let tempXI = new Tensor([], [dimInputGate]) + let tempHI = new Tensor([], [dimInputGate]) + + let currentForgetGateState = new Tensor([], [dimForgetGate]) + let tempXF = new Tensor([], [dimForgetGate]) + let tempHF = new Tensor([], [dimForgetGate]) + + let currentOutputGateState = new Tensor([], [dimOutputGate]) + let tempXO = new Tensor([], [dimOutputGate]) + let tempHO = new Tensor([], [dimOutputGate]) + + let currentCandidate = new Tensor([], [dimCandidate]) + let tempXC = new Tensor([], [dimCandidate]) + let tempHC = new Tensor([], [dimCandidate]) + let previousCandidate = new Tensor([], [dimCandidate]) + + let currentHiddenState = new Tensor([], [dimCandidate]) + let previousHiddenState = new Tensor([], [dimCandidate]) + + const _clearTemp = () => { + const tempTensors = [tempXI, tempHI, tempXF, tempHF, tempXO, tempHO, tempXC, tempHC] + tempTensors.forEach(temp => ops.assigns(temp.tensor, 0)) + } + + const _step = () => { + ops.assign(previousHiddenState.tensor, currentHiddenState.tensor) + + gemv(1.0, this.weights['W_i'].tensor.transpose(1, 0), currentX.tensor, 1.0, tempXI.tensor) + gemv(1.0, this.weights['U_i'].tensor.transpose(1, 0), previousHiddenState.tensor, 1.0, tempHI.tensor) + this._combine(currentInputGateState.tensor, tempXI.tensor, tempHI.tensor, this.weights['b_i'].tensor) + this.innerActivation(currentInputGateState) + + gemv(1.0, this.weights['W_f'].tensor.transpose(1, 0), currentX.tensor, 1.0, tempXF.tensor) + gemv(1.0, this.weights['U_f'].tensor.transpose(1, 0), previousHiddenState.tensor, 1.0, tempHF.tensor) + this._combine(currentForgetGateState.tensor, tempXF.tensor, tempHF.tensor, this.weights['b_f'].tensor) + this.innerActivation(currentForgetGateState) + + gemv(1.0, this.weights['W_o'].tensor.transpose(1, 0), currentX.tensor, 1.0, tempXO.tensor) + gemv(1.0, this.weights['U_o'].tensor.transpose(1, 0), previousHiddenState.tensor, 1.0, tempHO.tensor) + this._combine(currentOutputGateState.tensor, tempXO.tensor, tempHO.tensor, this.weights['b_o'].tensor) + this.innerActivation(currentOutputGateState) + + gemv(1.0, this.weights['W_c'].tensor.transpose(1, 0), currentX.tensor, 1.0, tempXC.tensor) + gemv(1.0, this.weights['U_c'].tensor.transpose(1, 0), previousHiddenState.tensor, 1.0, tempHC.tensor) + this._combine(currentCandidate.tensor, tempXC.tensor, tempHC.tensor, this.weights['b_c'].tensor) + this.activation(currentCandidate) + + this._update( + currentCandidate.tensor, + previousCandidate.tensor, + currentInputGateState.tensor, + currentForgetGateState.tensor + ) + + ops.assign(previousCandidate.tensor, currentCandidate.tensor) + + this.activation(currentCandidate) + ops.mul(currentHiddenState.tensor, currentOutputGateState.tensor, currentCandidate.tensor) + } + + for (let i = 0, steps = x.tensor.shape[0]; i < steps; i++) { + ops.assign(currentX.tensor, x.tensor.pick(i, null)) + _clearTemp() + _step() + } + + x.tensor = currentHiddenState.tensor + + return x + } +} diff --git a/src/layers/recurrent/index.js b/src/layers/recurrent/index.js new file mode 100644 index 0000000..2c2875d --- /dev/null +++ b/src/layers/recurrent/index.js @@ -0,0 +1,5 @@ +import LSTM from './LSTM' + +export { + LSTM +} diff --git a/test/index.html b/test/index.html index 50891fb..c1cb0e7 100644 --- a/test/index.html +++ b/test/index.html @@ -100,6 +100,9 @@ + + +