From aae13f01263acbed8ab45d51f1c1ee57b1a6ef24 Mon Sep 17 00:00:00 2001 From: Mike Clark Date: Tue, 16 Oct 2018 08:14:51 +0800 Subject: [PATCH] NaN's with multiple pooling layers with MNIST clouds :( Tried building a model with these ops, but with multiple layers of pooling I get NaN value after a while. I think there is a problem with an unstable gradient in the this layer. --- mnist_clouds.ipynb | 1009 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1009 insertions(+) create mode 100644 mnist_clouds.ipynb diff --git a/mnist_clouds.ipynb b/mnist_clouds.ipynb new file mode 100644 index 0000000..32d3017 --- /dev/null +++ b/mnist_clouds.ipynb @@ -0,0 +1,1009 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There seems to be a error when using more than one layer of pooling of conv_transpose. Perhaps it will be fixed in a later flex-conv commit. currently at commit e2fdad61d93a2ec830f68b9cf130b4688a4c8040 (20181016)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:28.366086Z", + "start_time": "2018-10-16T00:03:28.342706Z" + } + }, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:28.404059Z", + "start_time": "2018-10-16T00:03:28.387011Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" # see issue #152\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:38.756642Z", + "start_time": "2018-10-16T00:03:28.406461Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/wassname/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + " from ._conv import register_converters as _register_converters\n", + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "\n", + "from keras import backend as K\n", + "K.set_image_data_format('channels_first')\n", + "import tensorflow as tf\n", + "\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "from tabulate import tabulate\n", + "from layers import flex_convolution, flex_convolution_transpose, flex_pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:38.821985Z", + "start_time": "2018-10-16T00:03:38.760136Z" + } + }, + "outputs": [], + "source": [ + "###### img_rows = 512\n", + "img_cols = 512\n", + "img_channels = 3\n", + "batch_size = 1\n", + "num_epochs = 100\n", + "input_positions = 3\n", + "k = 5\n", + "network_channel_sizes = (8, 16, 16, 32, 32, 64, 64, 128, 128, 256, 256, 512, 512)\n", + "model_name = '3dflex_2_t_minst'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-03T08:03:19.091280Z", + "start_time": "2018-10-03T08:03:19.080019Z" + } + }, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:38.888021Z", + "start_time": "2018-10-16T00:03:38.824873Z" + } + }, + "outputs": [], + "source": [ + "# https://github.com/Harry-Zhi/3DMNIST/blob/master/3DMNIST.ipynb\n", + "def img_to_point_cloud(input_image, voxel):\n", + "\n", + " non_zero_coord = np.transpose(np.nonzero(input_image))\n", + " \n", + " # dict for fast looking of neighboor ocupancy\n", + " non_zero_dict = {}\n", + " for i in range(input_image.shape[0]):\n", + " for j in range(input_image.shape[1]):\n", + " non_zero_dict[str([i,j])] = any(np.all([i,j] == non_zero_coord, axis=1))\n", + "\n", + " cloud = []\n", + " \n", + " for n in range(len(non_zero_coord)):\n", + " x = non_zero_coord[n][0]\n", + " y = non_zero_coord[n][1]\n", + " \n", + " components = [0,1]\n", + " \n", + " # top\n", + " if not non_zero_dict[str([x-1, y])]:\n", + " components.append(2)\n", + " \n", + " # bottom\n", + " if not non_zero_dict[str([x+1, y])]:\n", + " components.append(3)\n", + " \n", + " # left\n", + " if not non_zero_dict[str([x, y-1])]:\n", + " components.append(4)\n", + " \n", + " # right\n", + " if not non_zero_dict[str([x, y+1])]:\n", + " components.append(5)\n", + " \n", + " pixel_cloud = np.concatenate(voxel[components])\n", + " \n", + " # move the voxel to its position\n", + " pixel_cloud[:,0] +=x\n", + " pixel_cloud[:,1] += y\n", + " \n", + " cloud.append(pixel_cloud)\n", + " \n", + " cloud = np.concatenate(cloud)\n", + " \n", + " # make max range 0-1\n", + " xyzmin = np.min(cloud[:,:3], axis=0)\n", + " xyzmax = np.max(cloud[:,:3], axis=0)\n", + " diff = xyzmax - xyzmin\n", + " cloud[:,:3] = ((cloud[:,:3] - xyzmin[np.argmax(diff)]) / diff[np.argmax(diff)])\n", + " \n", + " # 0 mean\n", + " cloud[:,:3] -= np.mean(cloud[:,:3], axis=0)\n", + "\n", + " return cloud" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:38.948978Z", + "start_time": "2018-10-16T00:03:38.890743Z" + } + }, + "outputs": [], + "source": [ + "MIN_X, MAX_X = (-0.5, 0.5)\n", + "MIN_Y, MAX_Y = (-0.5, 0.5)\n", + "MIN_Z, MAX_Z = (-3, 3)\n", + "\n", + "N_X = 5\n", + "N_Y = 5\n", + "N_Z = 30" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.016126Z", + "start_time": "2018-10-16T00:03:38.951042Z" + } + }, + "outputs": [], + "source": [ + "# VOXEL CREATION\n", + "# with normals\n", + "\n", + "front = np.array(np.meshgrid(np.linspace(MIN_X, MAX_X, N_X),\n", + " np.linspace(MIN_Y, MAX_Y, N_Y),\n", + " MAX_Z )).T.reshape(-1,3) \n", + "front = np.concatenate((front, [[1,0,0]] * len(front)), axis=1)\n", + "\n", + "\n", + "back = np.array(np.meshgrid(np.linspace(MIN_X, MAX_X, N_X), \n", + " np.linspace(MIN_Y, MAX_Y, N_Y), \n", + " MIN_Z )).T.reshape(-1,3) \n", + "back = np.concatenate((back, [[-1,0,0]] * len(back)), axis=1)\n", + "\n", + "\n", + "top = np.array(np.meshgrid(MIN_X,\n", + " np.linspace(MIN_Y, MAX_Y, N_Y), \n", + " np.linspace(MIN_Z, MAX_Z, N_Z))).T.reshape(-1,3)\n", + "top = np.concatenate((top, [[0,0,1]] * len(top)), axis=1)\n", + "\n", + "\n", + "bottom = np.array(np.meshgrid(MAX_X, \n", + " np.linspace(MIN_Y, MAX_Y, N_Y), \n", + " np.linspace(MIN_Z, MAX_Z, N_Z))).T.reshape(-1,3) \n", + "bottom = np.concatenate((bottom, [[0,0,-1]] * len(bottom)), axis=1)\n", + "\n", + "\n", + "left = np.array(np.meshgrid(np.linspace(MIN_X, MAX_X, N_X), \n", + " MIN_Y,\n", + " np.linspace(MIN_Z, MAX_Z, N_Z))).T.reshape(-1,3)\n", + "left = np.concatenate((left, [[0,-1,0]] * len(left)), axis=1)\n", + "\n", + "\n", + "right = np.array(np.meshgrid(np.linspace(MIN_X, MAX_X, N_X), \n", + " MAX_Y, \n", + " np.linspace(MIN_Z, MAX_Z, N_Z))).T.reshape(-1,3) \n", + "right = np.concatenate((right, [[0,1,0]] * len(right)), axis=1)\n", + "\n", + "\n", + "voxel = np.array([front, back, top, bottom, left, right])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.370688Z", + "start_time": "2018-10-16T00:03:39.018380Z" + } + }, + "outputs": [], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.428144Z", + "start_time": "2018-10-16T00:03:39.372881Z" + } + }, + "outputs": [], + "source": [ + "from scipy.spatial import cKDTree\n", + "\n", + "def cloud_to_tftree(xv, K=5):\n", + " positions = xv[:, :3]\n", + " features = xv[:, 3:]\n", + " tree = cKDTree(positions)\n", + " neighbors = tree.query(positions, k=K)[1]\n", + "\n", + " positions = positions.transpose(1, 0).astype(np.float32)\n", + " features = features.transpose(1, 0).astype(np.float32)\n", + " neighbors = neighbors.transpose(1, 0).astype(np.int32)\n", + " return features, positions, neighbors" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.663879Z", + "start_time": "2018-10-16T00:03:39.430990Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([[1., 1., 1., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 1., 1., 1.],\n", + " [0., 0., 0., ..., 0., 0., 0.]], dtype=float32),\n", + " array([[-0.47305447, -0.47305447, -0.47305447, ..., 0.5019455 ,\n", + " 0.51444554, 0.52694553],\n", + " [-0.1094358 , -0.0969358 , -0.0844358 , ..., -0.1094358 ,\n", + " -0.1094358 , -0.1094358 ],\n", + " [ 0.15 , 0.15 , 0.15 , ..., 0.15 ,\n", + " 0.15 , 0.15 ]], dtype=float32),\n", + " array([[ 345, 196, 197, ..., 25364, 25369, 25699],\n", + " [ 0, 1, 2, ..., 25697, 25698, 25374],\n", + " [ 195, 191, 192, ..., 25692, 25693, 25549],\n", + " [ 340, 345, 7, ..., 25696, 25699, 25694],\n", + " [ 190, 195, 3, ..., 25359, 25549, 25544]], dtype=int32))" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cloud_to_tftree(img_to_point_cloud(x_train[0], voxel))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:10:03.566867Z", + "start_time": "2018-10-16T00:10:03.494666Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "<__main__.DataLoader at 0x7f02759a12e8>" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class DataLoader(tf.keras.utils.Sequence):\n", + " \n", + " def __init__(self, x_train, y_train, batch_size=4):\n", + " super().__init__()\n", + " self.batch_size = batch_size\n", + " self.x_train = x_train\n", + " self.y_train = y_train\n", + " \n", + " def __getitem__(self, index):\n", + " start = index * self.batch_size\n", + " end = min(start + self.batch_size, len(self.x_train))\n", + "\n", + " xx = [self.x_train[i] for i in range(start, end)]\n", + " yy = [self.y_train[i] for i in range(start, end)]\n", + " \n", + " yy = np.stack(yy, 0)\n", + " \n", + " x2 = []\n", + " for x in xx:\n", + " try:\n", + " x = img_to_point_cloud(x, voxel)\n", + " except KeyError as e:\n", + " print('error', index, e) \n", + " x = np.zeros((512, 6))\n", + " idx = np.random.choice(range(len(x)), size=img_cols, replace=False)\n", + " x = cloud_to_tftree(x[idx])\n", + " x2.append(x)\n", + "# x2 = np.stack(x2, 0)\n", + " x2 = [np.stack(x) for x in zip(*x2)]\n", + " return x2, yy\n", + " \n", + " def __len__(self):\n", + " return len(self.x_train)//self.batch_size-1\n", + "\n", + "\n", + "train_gen = DataLoader(x_train, y_train, batch_size=batch_size)\n", + "test_gen = DataLoader(x_test, y_test, batch_size=batch_size)\n", + "train_gen" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:10:03.811773Z", + "start_time": "2018-10-16T00:10:03.715292Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-03T08:04:08.385788Z", + "start_time": "2018-10-03T08:04:08.374907Z" + } + }, + "source": [ + "# Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-08T09:02:51.764390Z", + "start_time": "2018-10-08T09:02:51.712010Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.789715Z", + "start_time": "2018-10-16T00:03:39.732871Z" + } + }, + "outputs": [], + "source": [ + "# # model\n", + "# # from https://github.com/mikelane/SegNet/blob/master/network.py\n", + "from typing import Tuple, TypeVar\n", + "\n", + "from tensorflow.python.keras.layers import concatenate, Conv2D, Conv2DTranspose, MaxPooling2D, BatchNormalization, Lambda, Input, ReLU, Conv1D, Dense, Reshape\n", + "from tensorflow.python.keras.layers.advanced_activations import LeakyReLU\n", + "from tensorflow.python.keras.models import Model, Sequential\n", + "from tensorflow.python.keras.optimizers import Adam\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.862238Z", + "start_time": "2018-10-16T00:03:39.792646Z" + } + }, + "outputs": [], + "source": [ + "from layers import FlexConvolution, FlexPooling\n", + "\n", + "class DownsampleGroup(tf.keras.Model):\n", + " def __init__(\n", + " self, num_filters: int, pool_strides=2, leaky_alpha=0.3):\n", + " \"\"\"\n", + " A Unet downsampling group using flex-net operations\n", + " \"\"\"\n", + " super().__init__()\n", + " self.num_filters = num_filters\n", + " self.leaky_alpha = leaky_alpha\n", + " self.pool_strides = pool_strides\n", + " \n", + " def build(self, input_shape):\n", + " \n", + "\n", + " input_features_shape, input_positions_shape, input_neighbors_shape = input_shape\n", + " \n", + " input_features = Input(shape=[s.value for s in input_features_shape[1:]])\n", + " input_positions = Input(shape=[s.value for s in input_positions_shape[1:]])\n", + " input_neighbors = Input(shape=[s.value for s in input_neighbors_shape[1:]], dtype='int32')\n", + " \n", + " self.flex0 = FlexConvolution(\n", + " input_features,\n", + " input_positions,\n", + " input_neighbors,\n", + " filters=self.num_filters,\n", + " activation=None,\n", + " kernel_initializer=None,\n", + " position_bias_initializer=tf.zeros_initializer(),\n", + " features_bias_initializer=tf.zeros_initializer(),\n", + " use_feature_bias=True,\n", + " data_format='simple',\n", + " trainable=True,\n", + " name=None)\n", + "# self.bn = BatchNormalization(axis=1)\n", + " self.act = LeakyReLU(alpha=self.leaky_alpha)\n", + " self.pool = FlexPooling(\n", + " input_features,\n", + " input_neighbors,\n", + " data_format='simple',\n", + " name=None)\n", + "\n", + "# conv_trans_features = Input(shape=[self.num_filters, input_features_shape[2]])\n", + "# self.conv_trans = FlexConvolutionTranspose(\n", + "# conv_trans_features,\n", + "# input_positions,\n", + "# input_neighbors,\n", + "# filters=self.num_filters,\n", + "# activation=None,\n", + "# kernel_initializer=None,\n", + "# position_bias_initializer=tf.zeros_initializer(),\n", + "# features_bias_initializer=tf.zeros_initializer(),\n", + "# use_feature_bias=True,\n", + "# data_format=\"simple\",\n", + "# trainable=True,\n", + "# name=None,\n", + "# )\n", + " \n", + " self.subsample_f = Lambda(lambda x: x[:, :, :input_features.shape[2]//self.pool_strides], output_shape=(self.num_filters, input_features.shape[2]//self.pool_strides))\n", + " self.subsample_p = Lambda(lambda x: x[:, :, :input_positions.shape[2]//self.pool_strides], output_shape=(input_positions.shape[1], input_positions.shape[2]//self.pool_strides))\n", + " self.subsample_n = Lambda(lambda x: x[:, :, :input_neighbors.shape[2]//self.pool_strides], output_shape=(input_neighbors.shape[1], input_neighbors.shape[2]//self.pool_strides))\n", + " \n", + " \n", + " def call(self, inputs, training=False):\n", + " \n", + " input_to_group, positions, neighborhoods = inputs\n", + " \n", + " conv_1 = self.flex0.apply([input_to_group, positions, neighborhoods])\n", + "# conv_1 = self.bn(conv_1, training=training)\n", + " conv_1 = self.act(conv_1)\n", + " \n", + " # FIXME the pooling op is unstable and more than 1 downsample module will result in NaN's after a few steps of training\n", + " # you can see this clearly by running this notebook without cuda\n", + " output = self.pool.apply([conv_1, neighborhoods])\n", + "# output = self.conv_trans.apply([conv_1, positions, neighborhoods]) \n", + " output = self.subsample_f(conv_1)\n", + " \n", + " positions = self.subsample_p(positions)\n", + " \n", + " neighborhoods = self.subsample_n(neighborhoods)\n", + " \n", + " return output, positions, neighborhoods" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-10T01:55:09.396077Z", + "start_time": "2018-10-10T01:55:09.330451Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:39.934678Z", + "start_time": "2018-10-16T00:03:39.864702Z" + } + }, + "outputs": [], + "source": [ + "from typing import Tuple\n", + "from layers import FlexConvolutionTranspose, FlexConvolution, FlexPooling\n", + "\n", + "\n", + "class Network2:\n", + " def __init__(self, \n", + " input_height: int,\n", + " input_channels: int,\n", + " network_channel_sizes: Tuple[int, ...],\n", + " channels_last: bool = True,\n", + " conv_padding: str = 'same',\n", + " down_conv_kernel: Tuple[int, int] = (3, 3),\n", + " up_conv_kernel: Tuple[int, int] = (2, 2),\n", + " up_conv_kernel_strides: int = 2,\n", + " leaky_alpha=0.3,\n", + " pool_size: Tuple[int, int] = (2, 2),\n", + " pool_strides: int = 2,\n", + " input_positions: int = 3,\n", + " output_dim: int = 128,\n", + " k: int = 5,\n", + " downsample_layers = 2,\n", + " ):\n", + " self.input_features = Input(shape=(input_channels, input_height))\n", + " self.input_positions = Input(shape=(input_positions, input_height))\n", + " self.input_neighbors = Input(shape=(k, input_height), dtype='int32')\n", + " self.inputs1 = [self.input_features, self.input_positions, self.input_neighbors]\n", + " \n", + " self.output_dim = output_dim\n", + " self.leaky_alpha = leaky_alpha\n", + " self.downsample_layers = downsample_layers\n", + " \n", + " self.inputs = [self.input_features, self.input_positions, self.input_neighbors]\n", + " \n", + " self.seq = []\n", + " for i in range(self.downsample_layers):\n", + " self.seq.append(DownsampleGroup(network_channel_sizes[i], leaky_alpha=leaky_alpha))\n", + "\n", + " self.subtract = Lambda(lambda x: x[0]-x[1], output_shape=(network_channel_sizes[i], 1))\n", + " \n", + " self.reshape = Reshape((network_channel_sizes[i] * input_height//(2**self.downsample_layers),))\n", + "\n", + " def get_model(self):\n", + " \n", + " inputs1 = self.inputs1\n", + " \n", + " for i in range(self.downsample_layers):\n", + " inputs1 = self.seq[i].apply(inputs1)\n", + " \n", + " \n", + " outputs = inputs1[0]\n", + " outputs = self.reshape(outputs)\n", + " \n", + "# outputs = self.reshape(inputs1[0])\n", + " \n", + "# outputs = Dense(128)(outputs)\n", + "# outputs = BatchNormalization(axis=1)(outputs)\n", + "# outputs = LeakyReLU(alpha=self.leaky_alpha)(outputs)\n", + " \n", + "# outputs = Dense(64)(outputs)\n", + "# outputs = BatchNormalization(axis=1)(outputs)\n", + "# outputs = LeakyReLU(alpha=self.leaky_alpha)(outputs)\n", + " \n", + " outputs = Dense(1)(outputs)\n", + " \n", + " self.model = Model(inputs=self.inputs, outputs=outputs)\n", + "\n", + " return self.model\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-15T23:38:13.156468Z", + "start_time": "2018-10-15T23:38:13.094567Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Init" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:40.192919Z", + "start_time": "2018-10-16T00:03:39.936896Z" + }, + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "input_1 (InputLayer) (None, 3, 512) 0 \n", + "__________________________________________________________________________________________________\n", + "input_2 (InputLayer) (None, 3, 512) 0 \n", + "__________________________________________________________________________________________________\n", + "input_3 (InputLayer) (None, 5, 512) 0 \n", + "__________________________________________________________________________________________________\n", + "downsample_group (DownsampleGro [(None, 8, 256), (No 104 input_1[0][0] \n", + " input_2[0][0] \n", + " input_3[0][0] \n", + "__________________________________________________________________________________________________\n", + "downsample_group_1 (DownsampleG [(None, 16, 128), (N 528 downsample_group[0][0] \n", + " downsample_group[0][1] \n", + " downsample_group[0][2] \n", + "__________________________________________________________________________________________________\n", + "reshape (Reshape) (None, 2048) 0 downsample_group_1[0][0] \n", + "__________________________________________________________________________________________________\n", + "dense (Dense) (None, 1) 2049 reshape[0][0] \n", + "==================================================================================================\n", + "Total params: 2,681\n", + "Trainable params: 2,681\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "net1 = Network2(\n", + " input_height=img_cols,\n", + " input_channels=img_channels,\n", + " network_channel_sizes=network_channel_sizes,\n", + " leaky_alpha=0.1, k=k)\n", + "net = net1.get_model()\n", + "\n", + "net.compile(\n", + " optimizer=Adam(lr=1e-7),\n", + " loss='mean_absolute_error',\n", + " metrics=['mean_absolute_error'])\n", + "\n", + "net.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-10T02:04:10.811276Z", + "start_time": "2018-10-10T02:04:10.725334Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:40.262776Z", + "start_time": "2018-10-16T00:03:40.195822Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "flex_convolution (FlexConvol (None, 8, 512) 104 \n", + "_________________________________________________________________\n", + "leaky_re_lu (LeakyReLU) (None, 8, 512) 0 \n", + "_________________________________________________________________\n", + "flex_pooling (FlexPooling) (None, 8, 512) 0 \n", + "_________________________________________________________________\n", + "lambda_1 (Lambda) (None, 8, 256) 0 \n", + "_________________________________________________________________\n", + "lambda_2 (Lambda) (None, 3, 256) 0 \n", + "_________________________________________________________________\n", + "lambda_3 (Lambda) (None, 5, 256) 0 \n", + "=================================================================\n", + "Total params: 104\n", + "Trainable params: 104\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "d = net1.seq[0]\n", + "d.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-08T05:09:34.498114Z", + "start_time": "2018-10-08T05:09:34.302103Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:40.387720Z", + "start_time": "2018-10-16T00:03:40.266718Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "([array([[[-1., -1., 1., ..., 0., 0., 0.],\n", + " [ 0., 0., 0., ..., -1., -1., 0.],\n", + " [ 0., 0., 0., ..., 0., 0., -1.]]], dtype=float32),\n", + " array([[[-0.32888693, 0.07111307, 0.37111306, ..., -0.14138693,\n", + " -0.16638693, -0.17888692],\n", + " [-0.27791518, -0.12791519, -0.2904152 , ..., -0.12791519,\n", + " -0.12791519, 0.3720848 ],\n", + " [-0.15 , -0.15 , 0.15 , ..., -0.11896551,\n", + " -0.08793104, -0.04655172]]], dtype=float32),\n", + " array([[[ 0, 1, 2, ..., 509, 510, 511],\n", + " [ 74, 162, 503, ..., 510, 322, 297],\n", + " [265, 216, 255, ..., 322, 509, 228],\n", + " [414, 358, 337, ..., 481, 181, 277],\n", + " [ 49, 106, 230, ..., 181, 394, 67]]], dtype=int32)],\n", + " array([3], dtype=uint8))" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(test_gen)\n", + "test_gen[200]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:03:40.487172Z", + "start_time": "2018-10-16T00:03:40.391319Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "([array([[[-1., 0., 0., ..., 0., 0., 0.],\n", + " [ 0., -1., 1., ..., 0., 0., 0.],\n", + " [ 0., 0., 0., ..., -1., -1., 1.]]], dtype=float32),\n", + " array([[[ 0.33352602, 0.258526 , 0.471026 , ..., -0.24147399,\n", + " -0.341474 , -0.441474 ],\n", + " [ 0.0800578 , -0.0199422 , 0.2300578 , ..., -0.0199422 ,\n", + " 0.2675578 , -0.0074422 ],\n", + " [-0.15 , 0.05689655, -0.0775862 , ..., 0.00517241,\n", + " -0.09827586, 0.06724138]]], dtype=float32),\n", + " array([[[ 0, 1, 2, ..., 509, 510, 511],\n", + " [418, 450, 322, ..., 434, 43, 461],\n", + " [327, 15, 36, ..., 416, 278, 309],\n", + " [ 66, 49, 430, ..., 323, 361, 424],\n", + " [ 99, 456, 353, ..., 388, 291, 265]]], dtype=int32)],\n", + " array([5], dtype=uint8))" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(train_gen)\n", + "train_gen[100]" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "ExecuteTime": { + "end_time": "2018-10-16T00:10:33.992327Z", + "start_time": "2018-10-16T00:10:18.921204Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6801deb33dc64833b1555e2bb5c32288", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, description='Training', max=1), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "af99a934c85948aba139e22e98b03979", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, description='Epoch 0', max=59999), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error 53214 '[28, 14]'\n", + "error 27485 '[28, 16]'\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m callbacks=[\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mTQDMNotebookCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;31m# TensorBoard(log_dir='logs/'),\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;31m# ModelCheckpoint(epoch_model_path,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1777\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1778\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1779\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1780\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1781\u001b[0m def evaluate_generator(self,\n", + "\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tensorflow/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m outs = model.train_on_batch(\n\u001b[0;32m--> 204\u001b[0;31m x, y, sample_weight=sample_weight, class_weight=class_weight)\n\u001b[0m\u001b[1;32m 205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1550\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1551\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1552\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1554\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2912\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_arrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_symbols\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2913\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2914\u001b[0;31m \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2915\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2916\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.5.3/envs/jupyter3/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1380\u001b[0m ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m 1381\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1382\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 1383\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1384\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "from keras.callbacks import TensorBoard, ModelCheckpoint\n", + "from keras_tqdm import TQDMNotebookCallback\n", + "\n", + "for epoch in range(num_epochs):\n", + " epoch_model_path = 'outputs/{model_name:}_epoch_{epoch:}.h5'.format(model_name=model_name, epoch=epoch)\n", + " net.fit_generator(generator=train_gen,\n", + " validation_data=test_gen,\n", + " verbose=0,\n", + " shuffle=True,\n", + " callbacks=[\n", + " TQDMNotebookCallback()\n", + "# TensorBoard(log_dir='logs/'),\n", + "# ModelCheckpoint(epoch_model_path,\n", + "# monitor='loss',\n", + "# save_best_only=True,\n", + "# save_weights_only=True,\n", + "# mode='auto',\n", + "# period=1)\n", + " ]\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "jupyter3", + "language": "python", + "name": "jupyter3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.3" + }, + "toc": { + "colors": { + "hover_highlight": "#DAA520", + "navigate_num": "#000000", + "navigate_text": "#333333", + "running_highlight": "#FF0000", + "selected_highlight": "#FFD700", + "sidebar_border": "#EEEEEE", + "wrapper_background": "#FFFFFF" + }, + "moveMenuLeft": true, + "nav_menu": { + "height": "12px", + "width": "252px" + }, + "navigate_menu": true, + "number_sections": false, + "sideBar": true, + "threshold": 4, + "toc_cell": false, + "toc_position": { + "height": "857px", + "left": "0px", + "right": "1714px", + "top": "152px", + "width": "191px" + }, + "toc_section_display": "block", + "toc_window_display": true, + "widenNotebook": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}