diff --git a/notebooks/activations.ipynb b/notebooks/activations.ipynb new file mode 100644 index 0000000..749cecf --- /dev/null +++ b/notebooks/activations.ipynb @@ -0,0 +1,1123 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras import activations\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def format_decimal(arr):\n", + " return [round(x * 1e6) / 1e6 for x in arr]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### softmax" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softmax.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.067194, 0.082071, 0.110784, 0.0608, 0.182652, 0.4965]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softmax(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softmax.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.067194, 0.082071, 0.110784, 0.0608, 0.182652, 0.4965, 0.107768, 0.149902, 0.11105, 0.247147, 0.082268, 0.301865]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softmax(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### softplus" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softplus.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softplus(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softplus.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.693147, 0.798139, 0.974077, 0.644397, 1.313262, 2.126928, 0.67826, 0.854355, 0.693147, 1.171101, 0.554355, 1.313262]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softplus(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softplus.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.693147, 0.798139, 0.474077, 0.644397, 1.313262, 2.126928, 0.67826, 2.395545, 0.693147, 1.171101, 0.554355, 1.313262]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softplus(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### softsign" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softsign.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softsign(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softsign.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.0, 0.166667, 0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.230769, 0.0, 0.444444, -0.230769, 0.5]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softsign(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.softsign.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.166667, -0.333333, -0.090909, 0.5, 0.666667, -0.029126, 0.69697, 0.0, 0.444444, -0.230769, 0.5]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.softsign(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ReLU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.relu.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, 0.5, 0.0, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.relu(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.relu.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.0, 0.2, 0.5, 0.0, 1.0, 2.0, 0.0, 0.3, 0.0, 0.8, 0.0, 1.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.relu(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.relu.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.2, 0.0, 0.0, 1.0, 2.0, 0.0, 2.3, 0.0, 0.8, 0.0, 1.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.relu(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.relu.3] 3D, max_value=0.5**" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.2, 0.0, 0.0, 0.5, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.relu(K.variable(np.array([arr_in])), max_value=0.5)\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.relu.4] 3D, alpha=0.3**" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.2, -0.15, -0.03, 1.0, 2.0, -0.009, 2.3, 0.0, 0.8, -0.09, 1.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.relu(K.variable(np.array([arr_in])), alpha=0.3)\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### tanh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.tanh.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.tanh(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.tanh.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.0, 0.197375, 0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.291313, 0.0, 0.664037, -0.291313, 0.761594]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.tanh(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.tanh.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.197375, -0.462117, -0.099668, 0.761594, 0.964028, -0.029991, 0.980096, 0.0, 0.664037, -0.291313, 0.761594]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.tanh(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### sigmoid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.sigmoid.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.sigmoid.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.5, 0.549834, 0.622459, 0.475021, 0.731059, 0.880797, 0.492501, 0.574443, 0.5, 0.689974, 0.425557, 0.731059]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.sigmoid.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.5, 0.549834, 0.377541, 0.475021, 0.731059, 0.880797, 0.492501, 0.908877, 0.5, 0.689974, 0.425557, 0.731059]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### hardSigmoid" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.hardSigmoid.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.5, 0.54, 0.6, 0.48, 0.7, 0.9]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.hard_sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.hardSigmoid.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.5, 0.54, 0.6, 0.48, 0.7, 0.9, 0.494, 0.56, 0.5, 0.66, 0.44, 0.7]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.hard_sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.hardSigmoid.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.5, 0.54, 0.4, 0.48, 0.7, 0.9, 0.494, 0.96, 0.5, 0.66, 0.44, 0.7]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.hard_sigmoid(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### linear" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.linear.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.linear(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.linear.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 6)\n", + "out shape: (2, 6)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0, -0.03, 0.3, 0.0, 0.8, -0.3, 1.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, -0.03, 0.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 6)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.linear(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[activations.linear.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "in shape: (2, 2, 3)\n", + "out shape: (2, 2, 3)\n", + "out: [0.0, 0.2, -0.5, -0.1, 1.0, 2.0, -0.03, 2.3, 0.0, 0.8, -0.3, 1.0]\n" + ] + } + ], + "source": [ + "data_in = [0, 0.2, -0.5, -0.1, 1, 2, -0.03, 2.3, 0, 0.8, -0.3, 1]\n", + "data_in_shape = (2, 2, 3)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "\n", + "result = activations.linear(K.variable(np.array([arr_in])))\n", + "\n", + "arr_out = K.eval(result)[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/advanced_activations.ipynb b/notebooks/advanced_activations.ipynb new file mode 100644 index 0000000..83fe6c0 --- /dev/null +++ b/notebooks/advanced_activations.ipynb @@ -0,0 +1,377 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras.models import Model\n", + "from keras.layers import Input\n", + "from keras.layers.advanced_activations import LeakyReLU, PReLU, ELU, ParametricSoftplus, ThresholdedReLU, SReLU\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def format_decimal(arr, places=6):\n", + " return [round(x * 10**places) / 10**places for x in arr]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### LeakyReLU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.LeakyReLU.0] alpha=0.4**" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, -0.2, -0.04, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = LeakyReLU(alpha=0.4)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, -0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### PReLU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.PReLU.0] weights: alphas**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, -0.01, 0.003, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = PReLU()(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "alphas = np.array([-0.03, -0.02, 0.02, -0.03, -0.03, -0.01])\n", + "model.set_weights([alphas])\n", + "\n", + "data_in = [0, 0.2, -0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ELU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.ELU.0] alpha=1.1**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, -0.432816, -0.104679, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = ELU(alpha=1.1)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, -0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ParametricSoftplus" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.ParametricSoftplus.0] weights: alphas, betas**" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.090109, -0.013664, 0.013763, -0.020054, -0.023944, -0.006931]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = ParametricSoftplus()(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "alphas = np.array([0.13, -0.02, 0.02, -0.03, -0.03, -0.01])\n", + "betas = np.array([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0])\n", + "model.set_weights([alphas, betas])\n", + "\n", + "data_in = [0, 0.2, -0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ThresholdedReLU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.ThresholdedReLU.0] theta=0.9**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = ThresholdedReLU(theta=0.9)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SReLU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[advanced_activations.SReLU.0] weights: t_left, a_left, t_right, a_right**" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, -0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6,)\n", + "out: [0.1339, 0.2, 0.0096, -0.065, 0.835, 0.068]\n" + ] + } + ], + "source": [ + "xxx = SReLU()\n", + "layer_0 = Input(shape=(6,))\n", + "layer_1 = xxx(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "t_left, a_left, t_right, a_right = (np.array([0.13, -0.02, 0.02, -0.03, -0.03, -0.01]),\n", + " np.array([-0.03, -0.1, 0.02, 0.5, 0.2, 0.0]),\n", + " np.array([-0.9, 0.8, 0.0, -1.0, 0.7, 0.4]),\n", + " np.array([0.1, 0.2, 0.3, 0.0, 0.5, -0.2]))\n", + "model.set_weights([t_left, a_left, t_right, a_right])\n", + "\n", + "data_in = [0, 0.2, -0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/convolutional.ipynb b/notebooks/convolutional.ipynb new file mode 100644 index 0000000..4410417 --- /dev/null +++ b/notebooks/convolutional.ipynb @@ -0,0 +1,524 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras.models import Model\n", + "from keras.layers import Input\n", + "from keras.layers.convolutional import Convolution2D\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def format_decimal(arr, places=6):\n", + " return [round(x * 10**places) / 10**places for x in arr]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convolution1D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convolution2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.0] 4 3x3 filters on 5x5x2 input, activation='linear', border_mode='valid', subsample=(1,1), dim_ordering='tf', bias=True**" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (3, 3, 2, 4)\n", + "W: [0.08681, -0.443261, -0.150965, 0.689552, -0.990562, -0.756862, 0.341498, 0.651706, -0.726587, 0.150187, 0.782644, -0.581596, -0.629344, -0.783246, -0.560605, 0.957248, 0.623366, -0.656118, 0.632449, -0.451853, -0.136592, 0.88006, 0.635299, -0.327776, -0.649179, -0.254336, -0.988623, -0.495147, 0.591325, -0.96949, 0.197687, 0.207609, -0.789705, -0.236113, -0.927048, 0.780823, 0.961842, -0.880116, 0.781092, 0.153803, 0.484959, 0.260368, 0.163684, -0.959122, -0.579947, 0.08937, 0.53823, -0.49861, -0.428209, 0.70479, 0.950013, 0.769707, -0.280984, 0.197718, -0.290409, -0.31962, -0.643838, -0.524612, -0.910275, 0.010863, -0.247495, 0.185611, 0.259884, -0.714799, 0.867683, 0.89276, 0.204593, -0.224467, -0.273624, -0.591309, -0.44647, -0.506928]\n", + "b shape: (4,)\n", + "b: [0.08681, -0.443261, -0.150965, 0.689552]\n", + "\n", + "in shape: (5, 5, 2)\n", + "in: [-0.990562, -0.756862, 0.341498, 0.651706, -0.726587, 0.150187, 0.782644, -0.581596, -0.629344, -0.783246, -0.560605, 0.957248, 0.623366, -0.656118, 0.632449, -0.451853, -0.136592, 0.88006, 0.635299, -0.327776, -0.649179, -0.254336, -0.988623, -0.495147, 0.591325, -0.96949, 0.197687, 0.207609, -0.789705, -0.236113, -0.927048, 0.780823, 0.961842, -0.880116, 0.781092, 0.153803, 0.484959, 0.260368, 0.163684, -0.959122, -0.579947, 0.08937, 0.53823, -0.49861, -0.428209, 0.70479, 0.950013, 0.769707, -0.280984, 0.197718]\n", + "out shape: (3, 3, 4)\n", + "out: [1.881069, 1.507598, -0.42648, 0.981946, -1.295304, -2.802102, -3.379112, 1.371861, -0.618295, -0.672642, 0.74116, -0.092844, 1.342198, -0.536709, -0.195162, -1.44153, -1.020649, 3.006844, 0.951307, 0.750237, -0.934043, -0.375223, -1.075853, 2.961484, 0.996102, -2.28647, -2.774244, 0.264827, -0.138134, 2.738469, 0.662859, -2.902777, -0.663609, -1.414437, -4.081861, 0.236701]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(4, 3, 3, activation='linear', border_mode='valid', \n", + " subsample=(1, 1), dim_ordering='tf', bias=True)\n", + "\n", + "layer_0 = Input(shape=(5, 5, 2))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(100)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "print('b shape:', weights[1].shape)\n", + "print('b:', format_decimal(weights[1].ravel().tolist()))\n", + "\n", + "data_in_shape = (5, 5, 2)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.1] 4 3x3 filters on 5x5x2 input, activation='linear', border_mode='valid', subsample=(1,1), dim_ordering='tf', bias=False**" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (3, 3, 2, 4)\n", + "W: [0.032797, 0.141335, -0.943052, -0.656957, 0.370554, 0.667794, -0.386068, 0.787226, 0.443088, -0.620122, 0.108455, -0.295736, -0.636215, 0.571204, 0.930966, -0.535293, -0.832877, 0.207097, 0.457986, -0.447522, 0.370613, 0.035735, -0.903031, -0.724262, -0.626065, 0.988636, 0.041331, 0.157579, 0.469638, 0.083924, 0.826307, 0.61584, -0.194004, -0.285551, 0.905753, -0.312737, 0.7302, 0.660555, 0.076323, 0.844939, -0.805707, -0.794305, 0.403015, 0.78096, -0.680879, -0.448855, 0.344983, -0.671394, 0.402742, -0.02473, 0.361356, 0.043096, -0.913207, -0.552127, 0.15041, -0.759133, 0.000233, -0.723981, -0.894383, -0.643446, -0.115264, 0.755175, 0.898528, -0.043665, -0.077761, 0.274578, -0.350784, -0.764844, -0.897798, 0.275317, 0.624532, 0.340521]\n", + "\n", + "in shape: (5, 5, 2)\n", + "in: [0.303535, -0.150862, 0.313191, -0.581677, 0.319849, 0.059247, 0.497041, -0.812486, 0.569044, 0.374484, 0.390157, -0.006267, 0.950722, -0.592945, -0.401959, -0.544688, -0.903662, 0.807943, -0.839793, 0.214433, 0.261693, -0.244116, -0.973518, 0.684439, -0.230125, 0.103332, 0.421076, 0.350558, 0.389129, -0.315086, -0.175219, -0.520418, 0.937672, -0.422886, -0.705377, -0.741319, 0.888112, -0.297131, 0.467135, 0.827779, 0.401975, -0.222937, 0.884519, 0.472983, -0.523071, 0.647547, 0.521227, -0.210582, -0.599624, 0.425193]\n", + "out shape: (3, 3, 4)\n", + "out: [0.222896, 1.085101, 1.523122, 0.029914, -3.208341, 0.668446, 0.159678, -2.842651, 1.751455, 0.291728, -3.055066, -1.414293, 2.641148, -0.979586, -3.159913, 0.963647, 2.551851, -1.421387, -2.003495, -0.273586, -0.434787, 0.075452, 0.122831, 0.496359, -0.500579, 0.818305, 0.353606, -1.248895, -1.872087, 1.321958, 0.976701, 0.640393, -2.795104, -2.835678, 1.354631, -1.549929]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(4, 3, 3, activation='linear', border_mode='valid', \n", + " subsample=(1, 1), dim_ordering='tf', bias=False)\n", + "\n", + "layer_0 = Input(shape=(5, 5, 2))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(101)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "\n", + "data_in_shape = (5, 5, 2)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.2] 4 3x3 filters on 5x5x2 input, activation='relu', border_mode='valid', subsample=(2,2), dim_ordering='tf', bias=True**" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (3, 3, 2, 4)\n", + "W: [0.195363, 0.351974, -0.401437, 0.461481, 0.157479, 0.618035, -0.665503, -0.37571, -0.284137, -0.016505, -0.019604, 0.78882, -0.635179, -0.277676, 0.621183, -0.333106, 0.037901, -0.47676, -0.738144, -0.755166, 0.103315, -0.941233, 0.20555, 0.501765, -0.501859, -0.604645, 0.147912, 0.803597, 0.032433, -0.197603, 0.034979, 0.874782, 0.700816, -0.648994, 0.039638, 0.011529, -0.302879, -0.858314, 0.151964, -0.196575, 0.82283, 0.617771, -0.735886, 0.429729, -0.306674, 0.228823, -0.425021, 0.971139, -0.455574, 0.638783, 0.199032, 0.44091, 0.91975, -0.814286, -0.088979, 0.765096, 0.395408, -0.357503, -0.275416, 0.467275, 0.245085, -0.563436, 0.96541, -0.199388, 0.43292, -0.389609, 0.298369, -0.25199, -0.266434, -0.949491, -0.303867, -0.024641]\n", + "b shape: (4,)\n", + "b: [0.195363, 0.351974, -0.401437, 0.461481]\n", + "\n", + "in shape: (5, 5, 2)\n", + "in: [0.157479, 0.618035, -0.665503, -0.37571, -0.284137, -0.016505, -0.019604, 0.78882, -0.635179, -0.277676, 0.621183, -0.333106, 0.037901, -0.47676, -0.738144, -0.755166, 0.103315, -0.941233, 0.20555, 0.501765, -0.501859, -0.604645, 0.147912, 0.803597, 0.032433, -0.197603, 0.034979, 0.874782, 0.700816, -0.648994, 0.039638, 0.011529, -0.302879, -0.858314, 0.151964, -0.196575, 0.82283, 0.617771, -0.735886, 0.429729, -0.306674, 0.228823, -0.425021, 0.971139, -0.455574, 0.638783, 0.199032, 0.44091, 0.91975, -0.814286]\n", + "out shape: (2, 2, 4)\n", + "out: [0.20599, 0.342823, 0.742175, 0.0, 0.815729, 2.184798, 1.251954, 0.0, 0.0, 0.0, 1.112864, 0.010656, 0.731076, 0.0, 0.855229, 0.0]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(4, 3, 3, activation='relu', border_mode='valid', \n", + " subsample=(2, 2), dim_ordering='tf', bias=True)\n", + "\n", + "layer_0 = Input(shape=(5, 5, 2))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(102)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "print('b shape:', weights[1].shape)\n", + "print('b:', format_decimal(weights[1].ravel().tolist()))\n", + "\n", + "data_in_shape = (5, 5, 2)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.3] 5 4x4 filters on 7x7x3 input, activation='relu', border_mode='valid', subsample=(2,1), dim_ordering='tf', bias=True**" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (4, 4, 3, 5)\n", + "W: [-0.135778, -0.651569, -0.658113, 0.655264, 0.174343, -0.081293, 0.64537, 0.643096, -0.385759, -0.598211, -0.193535, 0.895779, 0.355089, 0.21795, 0.344727, -0.984035, -0.32688, -0.285455, -0.02788, 0.756565, 0.509817, 0.26093, -0.247033, 0.180323, 0.474657, 0.192293, -0.815041, -0.195458, 0.179373, -0.769174, -0.223611, -0.880477, 0.167916, -0.666549, -0.303809, -0.156202, -0.501052, 0.539906, 0.10262, -0.947416, 0.541952, -0.813565, 0.405592, 0.159078, 0.198436, 0.71454, 0.102339, -0.835312, 0.466992, 0.067715, -0.315822, -0.056778, -0.556856, -0.647049, -0.050928, -0.277526, 0.64888, -0.708, 0.639138, -0.632262, 0.126027, -0.751824, 0.387838, -0.924068, -0.411933, 0.793635, -0.432359, -0.568279, -0.942986, -0.929277, 0.746907, 0.954599, -0.470971, 0.887461, -0.919761, -0.423457, 0.359481, 0.23833, 0.4367, 0.136035, 0.837921, 0.01252, 0.76166, -0.067006, -0.569453, 0.707085, -0.685992, -0.972653, 0.424172, -0.023413, -0.494544, 0.239537, -0.357146, -0.832116, 0.911805, -0.493355, 0.799492, 0.907763, 0.590698, -0.657319, -0.899599, 0.66576, -0.026552, -0.808517, 0.761201, 0.060031, 0.911167, -0.541875, -0.052366, -0.677874, -0.540037, 0.858037, 0.841428, 0.664757, 0.159247, -0.408513, 0.157763, 0.953228, -0.419758, 0.13644, 0.857617, 0.935383, -0.725736, 0.482768, 0.919937, 0.516163, 0.078243, 0.504286, 0.149417, 0.153285, -0.686413, -0.515197, 0.529735, -0.671812, 0.737835, 0.760286, -0.825563, 0.862511, -0.185154, 0.436687, -0.452091, -0.048684, 0.142544, 0.590622, -0.014887, 0.081443, -0.873841, -0.178904, 0.779569, 0.836992, -0.238607, 0.944873, -0.037027, 0.774056, -0.735437, -0.253303, 0.60782, -0.992794, -0.798253, -0.273809, -0.777994, -0.466779, 0.512164, 0.312941, 0.447827, -0.619183, -0.679821, 0.28787, 0.256291, -0.157439, -0.23489, 0.178363, 0.477582, -0.679932, -0.698904, -0.04713, -0.596417, 0.683988, 0.128674, 0.14461, -0.053412, -0.598701, -0.28704, -0.083479, 0.252272, -0.817686, 0.195913, -0.564344, -0.987454, 0.1461, -0.206776, 0.336421, 0.457537, -0.656402, 0.828352, -0.410226, 0.319745, -0.742681, -0.203616, 0.331785, 0.514347, -0.517425, -0.796471, 0.25001, -0.495738, -0.667485, 0.156551, 0.958574, -0.644056, 0.765525, -0.200055, -0.415118, -0.292941, -0.599639, 0.416564, -0.366227, 0.181605, 0.062272, -0.277941, -0.991833, 0.459158, 0.237737, -0.007369, -0.433549, 0.091637, -0.048628, 0.386804, 0.402629, -0.215112, -0.178129, -0.974124, 0.785804, -0.987586, -0.342624, 0.042113, -0.991281, -0.011006, -0.075192, -0.375827, -0.229347]\n", + "b shape: (5,)\n", + "b: [-0.135778, -0.651569, -0.658113, 0.655264, 0.174343]\n", + "\n", + "in shape: (7, 7, 3)\n", + "in: [-0.081293, 0.64537, 0.643096, -0.385759, -0.598211, -0.193535, 0.895779, 0.355089, 0.21795, 0.344727, -0.984035, -0.32688, -0.285455, -0.02788, 0.756565, 0.509817, 0.26093, -0.247033, 0.180323, 0.474657, 0.192293, -0.815041, -0.195458, 0.179373, -0.769174, -0.223611, -0.880477, 0.167916, -0.666549, -0.303809, -0.156202, -0.501052, 0.539906, 0.10262, -0.947416, 0.541952, -0.813565, 0.405592, 0.159078, 0.198436, 0.71454, 0.102339, -0.835312, 0.466992, 0.067715, -0.315822, -0.056778, -0.556856, -0.647049, -0.050928, -0.277526, 0.64888, -0.708, 0.639138, -0.632262, 0.126027, -0.751824, 0.387838, -0.924068, -0.411933, 0.793635, -0.432359, -0.568279, -0.942986, -0.929277, 0.746907, 0.954599, -0.470971, 0.887461, -0.919761, -0.423457, 0.359481, 0.23833, 0.4367, 0.136035, 0.837921, 0.01252, 0.76166, -0.067006, -0.569453, 0.707085, -0.685992, -0.972653, 0.424172, -0.023413, -0.494544, 0.239537, -0.357146, -0.832116, 0.911805, -0.493355, 0.799492, 0.907763, 0.590698, -0.657319, -0.899599, 0.66576, -0.026552, -0.808517, 0.761201, 0.060031, 0.911167, -0.541875, -0.052366, -0.677874, -0.540037, 0.858037, 0.841428, 0.664757, 0.159247, -0.408513, 0.157763, 0.953228, -0.419758, 0.13644, 0.857617, 0.935383, -0.725736, 0.482768, 0.919937, 0.516163, 0.078243, 0.504286, 0.149417, 0.153285, -0.686413, -0.515197, 0.529735, -0.671812, 0.737835, 0.760286, -0.825563, 0.862511, -0.185154, 0.436687, -0.452091, -0.048684, 0.142544, 0.590622, -0.014887, 0.081443, -0.873841, -0.178904, 0.779569, 0.836992, -0.238607, 0.944873]\n", + "out shape: (2, 4, 5)\n", + "out: [0.0, 0.0, 3.453939, 0.0, 1.228864, 0.0, 0.469417, 0.0, 0.0, 2.662399, 0.0526, 0.0, 0.0, 0.0, 4.369679, 0.0, 0.0, 0.0, 3.251074, 0.602914, 0.0, 4.518651, 0.0, 2.820123, 1.26282, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.960698, 1.478059, 0.0, 0.0, 1.973457, 0.620029, 0.0, 0.0, 3.609938]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(5, 4, 4, activation='relu', border_mode='valid', \n", + " subsample=(2, 1), dim_ordering='tf', bias=True)\n", + "\n", + "layer_0 = Input(shape=(7, 7, 3))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(103)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "print('b shape:', weights[1].shape)\n", + "print('b:', format_decimal(weights[1].ravel().tolist()))\n", + "\n", + "data_in_shape = (7, 7, 3)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.4] 4 3x3 filters on 5x5x2 input, activation='relu', border_mode='same', subsample=(1,1), dim_ordering='tf', bias=True**" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (3, 3, 2, 4)\n", + "W: [-0.704159, -0.543403, 0.614987, -0.403051, -0.628587, 0.544275, -0.189121, 0.991809, -0.028349, 0.30284, 0.201092, -0.953154, -0.343083, 0.640537, -0.117391, -0.526633, -0.163892, 0.861458, -0.665795, 0.51919, -0.753596, 0.337708, 0.402785, 0.315812, 0.541968, 0.870102, -0.141283, -0.469735, -0.574406, -0.571765, 0.046045, -0.790043, -0.953079, -0.774733, -0.58544, -0.882147, 0.282331, 0.214534, 0.862629, -0.446078, 0.230215, 0.308982, -0.361978, -0.850926, 0.409799, 0.710712, -0.391058, -0.542441, -0.41638, 0.558765, -0.149949, -0.864061, -0.759997, -0.265417, -0.331629, -0.845702, 0.426935, -0.731659, 0.062865, -0.255582, 0.396332, 0.639196, -0.078254, -0.253246, 0.792036, 0.044105, 0.879927, -0.987868, -0.35422, 0.924714, 0.304954, -0.870234]\n", + "b shape: (4,)\n", + "b: [-0.704159, -0.543403, 0.614987, -0.403051]\n", + "\n", + "in shape: (5, 5, 2)\n", + "in: [-0.628587, 0.544275, -0.189121, 0.991809, -0.028349, 0.30284, 0.201092, -0.953154, -0.343083, 0.640537, -0.117391, -0.526633, -0.163892, 0.861458, -0.665795, 0.51919, -0.753596, 0.337708, 0.402785, 0.315812, 0.541968, 0.870102, -0.141283, -0.469735, -0.574406, -0.571765, 0.046045, -0.790043, -0.953079, -0.774733, -0.58544, -0.882147, 0.282331, 0.214534, 0.862629, -0.446078, 0.230215, 0.308982, -0.361978, -0.850926, 0.409799, 0.710712, -0.391058, -0.542441, -0.41638, 0.558765, -0.149949, -0.864061, -0.759997, -0.265417]\n", + "out shape: (5, 5, 4)\n", + "out: [0.0, 1.245448, 1.285421, 0.0, 0.0, 0.359707, 1.273985, 0.0, 0.0, 0.0, 0.361441, 0.0, 0.0, 0.0, 0.0, 0.0, 0.814408, 0.0, 1.297703, 0.455072, 0.0, 0.045489, 0.0, 0.324618, 0.0, 0.74526, 0.0, 0.661333, 0.0, 0.0, 0.778435, 1.99095, 0.0, 0.0, 1.025771, 2.929728, 0.0, 0.0, 1.189183, 0.0, 0.0, 0.0, 2.123506, 0.0, 0.526917, 0.0, 2.237532, 0.0, 0.0, 0.0, 1.200334, 1.153394, 0.0, 0.0, 0.0, 4.087014, 0.0, 1.205699, 0.0, 1.322063, 0.159978, 0.037334, 0.0, 0.0, 0.0, 0.0, 0.4114, 0.0, 0.558253, 0.0, 0.0, 0.245042, 0.063339, 0.0, 0.433151, 0.396075, 0.39434, 0.0, 0.433619, 1.579657, 0.0, 0.0, 1.226101, 0.782336, 0.542313, 0.515257, 0.0, 0.0, 0.0, 0.0, 2.093277, 0.652337, 0.0, 0.0, 0.944013, 0.0, 0.306167, 0.0, 0.922607, 2.145665]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(4, 3, 3, activation='relu', border_mode='same', \n", + " subsample=(1, 1), dim_ordering='tf', bias=True)\n", + "\n", + "layer_0 = Input(shape=(5, 5, 2))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(104)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "print('b shape:', weights[1].shape)\n", + "print('b:', format_decimal(weights[1].ravel().tolist()))\n", + "\n", + "data_in_shape = (5, 5, 2)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[convolutional.Convolution2D.5] 4 3x3 filters on 5x5x2 input, activation='relu', border_mode='same', subsample=(2,2), dim_ordering='tf', bias=True**" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W shape: (3, 3, 2, 4)\n", + "W: [-0.83252, -0.332987, 0.718711, -0.788626, 0.07128, -0.872621, -0.406331, -0.99551, -0.097768, 0.767239, 0.154115, -0.502245, -0.316894, -0.859603, 0.076951, 0.050967, 0.637593, 0.372661, 0.261934, -0.259222, 0.072217, -0.653676, -0.668125, -0.779061, -0.348294, 0.794828, 0.096874, -0.517259, -0.343993, 0.177857, 0.590341, -0.037656, 0.152372, 0.180228, 0.595796, 0.556247, 0.333295, 0.022451, 0.07271, -0.653556, 0.35149, 0.816316, 0.070974, -0.631558, -0.817281, 0.142117, 0.649301, -0.685559, 0.317358, -0.816128, 0.896987, -0.808449, -0.141181, 0.254, 0.540822, -0.053505, -0.440647, -0.503204, -0.804487, 0.509112, 0.050405, 0.601983, -0.385832, 0.238291, 0.998461, 0.496045, 0.077139, -0.153403, -0.960801, 0.630448, -0.808006, -0.801654]\n", + "b shape: (4,)\n", + "b: [-0.83252, -0.332987, 0.718711, -0.788626]\n", + "\n", + "in shape: (4, 4, 2)\n", + "in: [0.07128, -0.872621, -0.406331, -0.99551, -0.097768, 0.767239, 0.154115, -0.502245, -0.316894, -0.859603, 0.076951, 0.050967, 0.637593, 0.372661, 0.261934, -0.259222, 0.072217, -0.653676, -0.668125, -0.779061, -0.348294, 0.794828, 0.096874, -0.517259, -0.343993, 0.177857, 0.590341, -0.037656, 0.152372, 0.180228, 0.595796, 0.556247]\n", + "out shape: (2, 2, 4)\n", + "out: [0.0, 0.565042, 0.12593, 0.0, 0.0, 0.320489, 0.979856, 0.0, 0.0, 0.0, 0.799525, 0.0, 0.0, 0.0, 0.637126, 0.0]\n" + ] + } + ], + "source": [ + "conv = Convolution2D(4, 3, 3, activation='relu', border_mode='same', \n", + " subsample=(2, 2), dim_ordering='tf', bias=True)\n", + "\n", + "layer_0 = Input(shape=(4, 4, 2))\n", + "layer_1 = conv(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# set weights to random (use seed for reproducibility)\n", + "weights = []\n", + "for w in model.get_weights():\n", + " np.random.seed(105)\n", + " weights.append(2 * np.random.random(w.shape) - 1)\n", + "model.set_weights(weights)\n", + "print('W shape:', weights[0].shape)\n", + "print('W:', format_decimal(weights[0].ravel().tolist()))\n", + "print('b shape:', weights[1].shape)\n", + "print('b:', format_decimal(weights[1].ravel().tolist()))\n", + "\n", + "data_in_shape = (4, 4, 2)\n", + "data_in = 2 * np.random.random(data_in_shape) - 1\n", + "print('')\n", + "print('in shape:', data_in_shape)\n", + "print('in:', format_decimal(data_in.ravel().tolist()))\n", + "result = model.predict(np.array([data_in]))\n", + "print('out shape:', result[0].shape)\n", + "print('out:', format_decimal(result[0].ravel().tolist()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deconvolution2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### AtrousConvolution2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SeparableConvolution2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convolution3D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### UpSampling1D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### UpSampling2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### UpSampling3D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ZeroPadding1D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ZeroPadding2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ZeroPadding3D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cropping1D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cropping2D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cropping3D" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/core.ipynb b/notebooks/core.ipynb new file mode 100644 index 0000000..48ad92b --- /dev/null +++ b/notebooks/core.ipynb @@ -0,0 +1,1600 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras.models import Model\n", + "from keras.engine import merge\n", + "from keras.layers import Input\n", + "from keras.layers.core import Dense, Activation, Dropout\n", + "from keras.layers.core import Flatten, Reshape, Permute, RepeatVector\n", + "from keras.layers.core import Merge, Lambda, ActivityRegularization, Masking\n", + "from keras.layers.core import Highway, MaxoutDense, TimeDistributedDense\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def format_decimal(arr, places=6):\n", + " return [round(x * 10**places) / 10**places for x in arr]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dense" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.0] test 1**" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [7.3, -0.21]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='linear')(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.1] test 2 (with sigmoid activation)**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [0.999325, 0.447692]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='sigmoid')(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.2] test 3 (with softplus activation and no bias)**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [6.801113, 0.338274]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='softplus', bias=False)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "model.set_weights([W])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.3] [GPU] test 1**" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [7.3, -0.21]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='linear')(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.4] [GPU] test 2 (with sigmoid activation)**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [0.999325, 0.447692]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='sigmoid')(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dense.5] [GPU] test 3 (with softplus activation and no bias)**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [6.801113, 0.338274]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2, activation='softplus', bias=False)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "model.set_weights([W])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Activation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Activation.0] test 1 (tanh)**" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [0.999999, -0.206966]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2)(layer_0)\n", + "layer_2 = Activation('tanh')(layer_1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Activation.1] test 2 (hardSigmoid)**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [1.0, 0.458]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2)(layer_0)\n", + "layer_2 = Activation('hard_sigmoid')(layer_1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dropout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Dropout.0] should pass through**" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [7.3, -0.21]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Dense(2)(layer_0)\n", + "layer_2 = Dropout(0.5)(layer_1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b = np.array([0.5, 0.7])\n", + "model.set_weights([W, b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Flatten" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Flatten.0] 1D**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# layer_0 = Input(shape=(6,))\n", + "# layer_1 = Flatten()(layer_0)\n", + "# model = Model(input=layer_0, output=layer_1)\n", + "\n", + "# data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "# data_in_shape = (6,)\n", + "# print('in:', data_in)\n", + "# print('in shape:', data_in_shape)\n", + "# arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "# result = model.predict(np.array([arr_in]))\n", + "# arr_out = result[0]\n", + "# print('out shape:', arr_out.shape)\n", + "# data_out = format_decimal(arr_out.ravel().tolist())\n", + "# print('out:', data_out)\n", + "\n", + "# exception with TF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Flatten.1] 2D**" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (3, 2)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(3, 2))\n", + "layer_1 = Flatten()(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (3, 2)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Flatten.2] 3D**" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (3, 2, 2)\n", + "out shape: (12,)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(3, 2, 2))\n", + "layer_1 = Flatten()(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (3, 2, 2)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reshape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Reshape.0] shape [6] -> [2, 3]**" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2, 3)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = Reshape((2, 3))(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Reshape.1] shape [3, 2] -> [6]**" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (3, 2)\n", + "out shape: (6,)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(3, 2))\n", + "layer_1 = Reshape((6,))(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (3, 2)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Reshape.2] shape [3, 2, 2] -> [4, 3]**" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (3, 2, 2)\n", + "out shape: (4, 3)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(3, 2, 2))\n", + "layer_1 = Reshape((4, 3))(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (3, 2, 2)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Permute" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Permute.0] shape [3, 2] -> [2, 3]**" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (3, 2)\n", + "out shape: (2, 3)\n", + "out: [0.0, 0.5, 1.0, 0.2, -0.1, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(3, 2))\n", + "layer_1 = Permute((2, 1))(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (3, 2)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Permute.1] shape [2, 3, 4] -> [4, 3, 2]**" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (2, 3, 4)\n", + "out shape: (4, 3, 2)\n", + "out: [0.0, 0.0, 1.0, 1.0, 0.5, 0.5, 0.2, 0.2, 2.0, 2.0, -0.1, -0.1, 0.5, 0.5, 0.0, 0.0, 1.0, 1.0, -0.1, -0.1, 0.2, 0.2, 2.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(2, 3, 4))\n", + "layer_1 = Permute((3, 2, 1))(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2, 0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (2, 3, 4)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RepeatVector" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.RepeatVector.0] repeat vector, shape [6] -> [7, 6]**" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (7, 6)\n", + "out: [0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0, 0.0, 0.2, 0.5, -0.1, 1.0, 2.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1 = RepeatVector(7)(layer_0)\n", + "model = Model(input=layer_0, output=layer_1)\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Merge" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.0] mode: sum**" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [4.85, 4.27]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='sum')\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.1] mode: mul**" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [-17.885, -0.9408]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='mul')\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.2] mode: ave**" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [2.425, 2.135]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='ave')\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.3] mode: max**" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2,)\n", + "out: [7.3, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='max')\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.4] mode: concat (1D)**" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (4,)\n", + "out: [7.3, -0.21, -2.45, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='concat', concat_axis=-1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.5] mode: concat (2D, concatAxis=-1)**" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (3, 4)\n", + "out: [7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='concat', concat_axis=-1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.6] mode: concat (2D, concatAxis=-2)**" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6, 2)\n", + "out: [7.3, -0.21, 7.3, -0.21, 7.3, -0.21, -2.45, 4.48, -2.45, 4.48, -2.45, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='concat', concat_axis=-2)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.7] mode: concat (2D, concatAxis=1)**" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (6, 2)\n", + "out: [7.3, -0.21, 7.3, -0.21, 7.3, -0.21, -2.45, 4.48, -2.45, 4.48, -2.45, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='concat', concat_axis=1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.8] mode: concat (2D, concatAxis=2)**" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (3, 4)\n", + "out: [7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48, 7.3, -0.21, -2.45, 4.48]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='concat', concat_axis=2)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.9] mode: dot (2D x 2D, dotAxes=1)**" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (2, 2)\n", + "out: [-53.655003, 98.112007, 1.5435, -2.8224]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='dot', dot_axes=1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.10] mode: dot (2D x 2D, dotAxes=2)**" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (3, 3)\n", + "out: [-18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258, -18.8258]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='dot', dot_axes=2)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.11] mode: cos (2D x 2D, dotAxes=1)**" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (1, 2, 2)\n", + "out: [-1.0, 7.972744, 0.125427, -1.0]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='cos', dot_axes=1)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**[core.Merge.12] mode: cos (2D x 2D, dotAxes=2)**" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "in: [0, 0.2, 0.5, -0.1, 1, 2]\n", + "in shape: (6,)\n", + "out shape: (1, 3, 3)\n", + "out: [-0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843, -0.504843]\n" + ] + } + ], + "source": [ + "layer_0 = Input(shape=(6,))\n", + "layer_1a = Dense(2, activation='linear')(layer_0)\n", + "layer_1a = RepeatVector(3)(layer_1a)\n", + "layer_1b = Dense(2, activation='linear')(layer_0)\n", + "layer_1b = RepeatVector(3)(layer_1b)\n", + "layer_2 = merge([layer_1a, layer_1b], mode='cos', dot_axes=2)\n", + "model = Model(input=layer_0, output=layer_2)\n", + "\n", + "W_1a = np.array([0.1, 0.4, 0.5, 0.1, 1, -2, 0, 0.3, 0.2, 0.1, 3, 0]).reshape((6, 2))\n", + "b_1a = np.array([0.5, 0.7])\n", + "W_1b = np.array([1, 0, -0.9, 0.6, -0.7, 0, 0.2, 0.4, 0, 0, -1, 2.3]).reshape((6, 2))\n", + "b_1b = np.array([0.1, -0.2])\n", + "model.set_weights([W_1a, b_1a, W_1b, b_1b])\n", + "\n", + "data_in = [0, 0.2, 0.5, -0.1, 1, 2]\n", + "data_in_shape = (6,)\n", + "print('in:', data_in)\n", + "print('in shape:', data_in_shape)\n", + "arr_in = np.array(data_in, dtype='float32').reshape(data_in_shape)\n", + "result = model.predict(np.array([arr_in]))\n", + "arr_out = result[0]\n", + "print('out shape:', arr_out.shape)\n", + "data_out = format_decimal(arr_out.ravel().tolist())\n", + "print('out:', data_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Highway" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### MaxoutDense" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### TimeDistributedDense" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}