mirror of
https://github.com/wassname/data_augmentation_talk.git
synced 2026-08-11 11:17:24 +08:00
In [1]:
%pylab --no-import-all inline
import pandas as pd
import seaborn as sns
from tqdm import tqdm_notebook as tqdmPopulating the interactive namespace from numpy and matplotlib
In [2]:
import os
os.sys.path.append(os.path.abspath('.'))In [3]:
import keras
import keras.models
from keras.datasets import cifar10
from keras import backend as K
from sklearn.model_selection import train_test_splitUsing TensorFlow backend.
In [4]:
# init
K.set_image_data_format('channels_last')
seed=0
batch_size = 32Model
We load a small 40 layer densenet model which has been pretrained on the cifar10 dataset. With only 1M params it got 95% accuracy.
The pretrained model was provided by robertomest. Thanks robert.
Densenet is charecterised by skip connects between all layers.
As of 25 Aug 2016 it beat all previous benchmarks in CIFAR 10, CIFAR 100 and SVHN.
In [5]:
# load a pretrained densenet model from https://github.com/robertomest/convnet-study
model = keras.models.model_from_json(
open('pretrained_models/densenet_cifar10_robertomest/densenet.json')
.read())
model.load_weights(
'./pretrained_models/densenet_cifar10_robertomest/densenet.h5')
model.compile(
optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()/media/isisilon/Data/My_Documents/Documents/eclipse-workspace/keras/keras/engine/topology.py:1237: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc. return cls(**config)
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
input_9 (InputLayer) (None, 32, 32, 3) 0
____________________________________________________________________________________________________
conv2d_313 (Conv2D) (None, 32, 32, 16) 432 input_9[0][0]
____________________________________________________________________________________________________
batch_normalization_313 (BatchNo (None, 32, 32, 16) 64 conv2d_313[0][0]
____________________________________________________________________________________________________
activation_321 (Activation) (None, 32, 32, 16) 0 batch_normalization_313[0][0]
____________________________________________________________________________________________________
conv2d_314 (Conv2D) (None, 32, 32, 12) 1728 activation_321[0][0]
____________________________________________________________________________________________________
dropout_115 (Dropout) (None, 32, 32, 12) 0 conv2d_314[0][0]
____________________________________________________________________________________________________
merge_289 (Merge) (None, 32, 32, 28) 0 conv2d_313[0][0]
dropout_115[0][0]
____________________________________________________________________________________________________
batch_normalization_314 (BatchNo (None, 32, 32, 28) 112 merge_289[0][0]
____________________________________________________________________________________________________
activation_322 (Activation) (None, 32, 32, 28) 0 batch_normalization_314[0][0]
____________________________________________________________________________________________________
conv2d_315 (Conv2D) (None, 32, 32, 12) 3024 activation_322[0][0]
____________________________________________________________________________________________________
dropout_116 (Dropout) (None, 32, 32, 12) 0 conv2d_315[0][0]
____________________________________________________________________________________________________
merge_290 (Merge) (None, 32, 32, 40) 0 merge_289[0][0]
dropout_116[0][0]
____________________________________________________________________________________________________
batch_normalization_315 (BatchNo (None, 32, 32, 40) 160 merge_290[0][0]
____________________________________________________________________________________________________
activation_323 (Activation) (None, 32, 32, 40) 0 batch_normalization_315[0][0]
____________________________________________________________________________________________________
conv2d_316 (Conv2D) (None, 32, 32, 12) 4320 activation_323[0][0]
____________________________________________________________________________________________________
dropout_117 (Dropout) (None, 32, 32, 12) 0 conv2d_316[0][0]
____________________________________________________________________________________________________
merge_291 (Merge) (None, 32, 32, 52) 0 merge_290[0][0]
dropout_117[0][0]
____________________________________________________________________________________________________
batch_normalization_316 (BatchNo (None, 32, 32, 52) 208 merge_291[0][0]
____________________________________________________________________________________________________
activation_324 (Activation) (None, 32, 32, 52) 0 batch_normalization_316[0][0]
____________________________________________________________________________________________________
conv2d_317 (Conv2D) (None, 32, 32, 12) 5616 activation_324[0][0]
____________________________________________________________________________________________________
dropout_118 (Dropout) (None, 32, 32, 12) 0 conv2d_317[0][0]
____________________________________________________________________________________________________
merge_292 (Merge) (None, 32, 32, 64) 0 merge_291[0][0]
dropout_118[0][0]
____________________________________________________________________________________________________
batch_normalization_317 (BatchNo (None, 32, 32, 64) 256 merge_292[0][0]
____________________________________________________________________________________________________
activation_325 (Activation) (None, 32, 32, 64) 0 batch_normalization_317[0][0]
____________________________________________________________________________________________________
conv2d_318 (Conv2D) (None, 32, 32, 12) 6912 activation_325[0][0]
____________________________________________________________________________________________________
dropout_119 (Dropout) (None, 32, 32, 12) 0 conv2d_318[0][0]
____________________________________________________________________________________________________
merge_293 (Merge) (None, 32, 32, 76) 0 merge_292[0][0]
dropout_119[0][0]
____________________________________________________________________________________________________
batch_normalization_318 (BatchNo (None, 32, 32, 76) 304 merge_293[0][0]
____________________________________________________________________________________________________
activation_326 (Activation) (None, 32, 32, 76) 0 batch_normalization_318[0][0]
____________________________________________________________________________________________________
conv2d_319 (Conv2D) (None, 32, 32, 12) 8208 activation_326[0][0]
____________________________________________________________________________________________________
dropout_120 (Dropout) (None, 32, 32, 12) 0 conv2d_319[0][0]
____________________________________________________________________________________________________
merge_294 (Merge) (None, 32, 32, 88) 0 merge_293[0][0]
dropout_120[0][0]
____________________________________________________________________________________________________
batch_normalization_319 (BatchNo (None, 32, 32, 88) 352 merge_294[0][0]
____________________________________________________________________________________________________
activation_327 (Activation) (None, 32, 32, 88) 0 batch_normalization_319[0][0]
____________________________________________________________________________________________________
conv2d_320 (Conv2D) (None, 32, 32, 12) 9504 activation_327[0][0]
____________________________________________________________________________________________________
dropout_121 (Dropout) (None, 32, 32, 12) 0 conv2d_320[0][0]
____________________________________________________________________________________________________
merge_295 (Merge) (None, 32, 32, 100) 0 merge_294[0][0]
dropout_121[0][0]
____________________________________________________________________________________________________
batch_normalization_320 (BatchNo (None, 32, 32, 100) 400 merge_295[0][0]
____________________________________________________________________________________________________
activation_328 (Activation) (None, 32, 32, 100) 0 batch_normalization_320[0][0]
____________________________________________________________________________________________________
conv2d_321 (Conv2D) (None, 32, 32, 12) 10800 activation_328[0][0]
____________________________________________________________________________________________________
dropout_122 (Dropout) (None, 32, 32, 12) 0 conv2d_321[0][0]
____________________________________________________________________________________________________
merge_296 (Merge) (None, 32, 32, 112) 0 merge_295[0][0]
dropout_122[0][0]
____________________________________________________________________________________________________
batch_normalization_321 (BatchNo (None, 32, 32, 112) 448 merge_296[0][0]
____________________________________________________________________________________________________
activation_329 (Activation) (None, 32, 32, 112) 0 batch_normalization_321[0][0]
____________________________________________________________________________________________________
conv2d_322 (Conv2D) (None, 32, 32, 12) 12096 activation_329[0][0]
____________________________________________________________________________________________________
dropout_123 (Dropout) (None, 32, 32, 12) 0 conv2d_322[0][0]
____________________________________________________________________________________________________
merge_297 (Merge) (None, 32, 32, 124) 0 merge_296[0][0]
dropout_123[0][0]
____________________________________________________________________________________________________
batch_normalization_322 (BatchNo (None, 32, 32, 124) 496 merge_297[0][0]
____________________________________________________________________________________________________
activation_330 (Activation) (None, 32, 32, 124) 0 batch_normalization_322[0][0]
____________________________________________________________________________________________________
conv2d_323 (Conv2D) (None, 32, 32, 12) 13392 activation_330[0][0]
____________________________________________________________________________________________________
dropout_124 (Dropout) (None, 32, 32, 12) 0 conv2d_323[0][0]
____________________________________________________________________________________________________
merge_298 (Merge) (None, 32, 32, 136) 0 merge_297[0][0]
dropout_124[0][0]
____________________________________________________________________________________________________
batch_normalization_323 (BatchNo (None, 32, 32, 136) 544 merge_298[0][0]
____________________________________________________________________________________________________
activation_331 (Activation) (None, 32, 32, 136) 0 batch_normalization_323[0][0]
____________________________________________________________________________________________________
conv2d_324 (Conv2D) (None, 32, 32, 12) 14688 activation_331[0][0]
____________________________________________________________________________________________________
dropout_125 (Dropout) (None, 32, 32, 12) 0 conv2d_324[0][0]
____________________________________________________________________________________________________
merge_299 (Merge) (None, 32, 32, 148) 0 merge_298[0][0]
dropout_125[0][0]
____________________________________________________________________________________________________
batch_normalization_324 (BatchNo (None, 32, 32, 148) 592 merge_299[0][0]
____________________________________________________________________________________________________
activation_332 (Activation) (None, 32, 32, 148) 0 batch_normalization_324[0][0]
____________________________________________________________________________________________________
conv2d_325 (Conv2D) (None, 32, 32, 12) 15984 activation_332[0][0]
____________________________________________________________________________________________________
dropout_126 (Dropout) (None, 32, 32, 12) 0 conv2d_325[0][0]
____________________________________________________________________________________________________
merge_300 (Merge) (None, 32, 32, 160) 0 merge_299[0][0]
dropout_126[0][0]
____________________________________________________________________________________________________
batch_normalization_325 (BatchNo (None, 32, 32, 160) 640 merge_300[0][0]
____________________________________________________________________________________________________
activation_333 (Activation) (None, 32, 32, 160) 0 batch_normalization_325[0][0]
____________________________________________________________________________________________________
conv2d_326 (Conv2D) (None, 32, 32, 160) 25600 activation_333[0][0]
____________________________________________________________________________________________________
dropout_127 (Dropout) (None, 32, 32, 160) 0 conv2d_326[0][0]
____________________________________________________________________________________________________
average_pooling2d_17 (AveragePoo (None, 16, 16, 160) 0 dropout_127[0][0]
____________________________________________________________________________________________________
batch_normalization_326 (BatchNo (None, 16, 16, 160) 640 average_pooling2d_17[0][0]
____________________________________________________________________________________________________
activation_334 (Activation) (None, 16, 16, 160) 0 batch_normalization_326[0][0]
____________________________________________________________________________________________________
conv2d_327 (Conv2D) (None, 16, 16, 12) 17280 activation_334[0][0]
____________________________________________________________________________________________________
dropout_128 (Dropout) (None, 16, 16, 12) 0 conv2d_327[0][0]
____________________________________________________________________________________________________
merge_301 (Merge) (None, 16, 16, 172) 0 average_pooling2d_17[0][0]
dropout_128[0][0]
____________________________________________________________________________________________________
batch_normalization_327 (BatchNo (None, 16, 16, 172) 688 merge_301[0][0]
____________________________________________________________________________________________________
activation_335 (Activation) (None, 16, 16, 172) 0 batch_normalization_327[0][0]
____________________________________________________________________________________________________
conv2d_328 (Conv2D) (None, 16, 16, 12) 18576 activation_335[0][0]
____________________________________________________________________________________________________
dropout_129 (Dropout) (None, 16, 16, 12) 0 conv2d_328[0][0]
____________________________________________________________________________________________________
merge_302 (Merge) (None, 16, 16, 184) 0 merge_301[0][0]
dropout_129[0][0]
____________________________________________________________________________________________________
batch_normalization_328 (BatchNo (None, 16, 16, 184) 736 merge_302[0][0]
____________________________________________________________________________________________________
activation_336 (Activation) (None, 16, 16, 184) 0 batch_normalization_328[0][0]
____________________________________________________________________________________________________
conv2d_329 (Conv2D) (None, 16, 16, 12) 19872 activation_336[0][0]
____________________________________________________________________________________________________
dropout_130 (Dropout) (None, 16, 16, 12) 0 conv2d_329[0][0]
____________________________________________________________________________________________________
merge_303 (Merge) (None, 16, 16, 196) 0 merge_302[0][0]
dropout_130[0][0]
____________________________________________________________________________________________________
batch_normalization_329 (BatchNo (None, 16, 16, 196) 784 merge_303[0][0]
____________________________________________________________________________________________________
activation_337 (Activation) (None, 16, 16, 196) 0 batch_normalization_329[0][0]
____________________________________________________________________________________________________
conv2d_330 (Conv2D) (None, 16, 16, 12) 21168 activation_337[0][0]
____________________________________________________________________________________________________
dropout_131 (Dropout) (None, 16, 16, 12) 0 conv2d_330[0][0]
____________________________________________________________________________________________________
merge_304 (Merge) (None, 16, 16, 208) 0 merge_303[0][0]
dropout_131[0][0]
____________________________________________________________________________________________________
batch_normalization_330 (BatchNo (None, 16, 16, 208) 832 merge_304[0][0]
____________________________________________________________________________________________________
activation_338 (Activation) (None, 16, 16, 208) 0 batch_normalization_330[0][0]
____________________________________________________________________________________________________
conv2d_331 (Conv2D) (None, 16, 16, 12) 22464 activation_338[0][0]
____________________________________________________________________________________________________
dropout_132 (Dropout) (None, 16, 16, 12) 0 conv2d_331[0][0]
____________________________________________________________________________________________________
merge_305 (Merge) (None, 16, 16, 220) 0 merge_304[0][0]
dropout_132[0][0]
____________________________________________________________________________________________________
batch_normalization_331 (BatchNo (None, 16, 16, 220) 880 merge_305[0][0]
____________________________________________________________________________________________________
activation_339 (Activation) (None, 16, 16, 220) 0 batch_normalization_331[0][0]
____________________________________________________________________________________________________
conv2d_332 (Conv2D) (None, 16, 16, 12) 23760 activation_339[0][0]
____________________________________________________________________________________________________
dropout_133 (Dropout) (None, 16, 16, 12) 0 conv2d_332[0][0]
____________________________________________________________________________________________________
merge_306 (Merge) (None, 16, 16, 232) 0 merge_305[0][0]
dropout_133[0][0]
____________________________________________________________________________________________________
batch_normalization_332 (BatchNo (None, 16, 16, 232) 928 merge_306[0][0]
____________________________________________________________________________________________________
activation_340 (Activation) (None, 16, 16, 232) 0 batch_normalization_332[0][0]
____________________________________________________________________________________________________
conv2d_333 (Conv2D) (None, 16, 16, 12) 25056 activation_340[0][0]
____________________________________________________________________________________________________
dropout_134 (Dropout) (None, 16, 16, 12) 0 conv2d_333[0][0]
____________________________________________________________________________________________________
merge_307 (Merge) (None, 16, 16, 244) 0 merge_306[0][0]
dropout_134[0][0]
____________________________________________________________________________________________________
batch_normalization_333 (BatchNo (None, 16, 16, 244) 976 merge_307[0][0]
____________________________________________________________________________________________________
activation_341 (Activation) (None, 16, 16, 244) 0 batch_normalization_333[0][0]
____________________________________________________________________________________________________
conv2d_334 (Conv2D) (None, 16, 16, 12) 26352 activation_341[0][0]
____________________________________________________________________________________________________
dropout_135 (Dropout) (None, 16, 16, 12) 0 conv2d_334[0][0]
____________________________________________________________________________________________________
merge_308 (Merge) (None, 16, 16, 256) 0 merge_307[0][0]
dropout_135[0][0]
____________________________________________________________________________________________________
batch_normalization_334 (BatchNo (None, 16, 16, 256) 1024 merge_308[0][0]
____________________________________________________________________________________________________
activation_342 (Activation) (None, 16, 16, 256) 0 batch_normalization_334[0][0]
____________________________________________________________________________________________________
conv2d_335 (Conv2D) (None, 16, 16, 12) 27648 activation_342[0][0]
____________________________________________________________________________________________________
dropout_136 (Dropout) (None, 16, 16, 12) 0 conv2d_335[0][0]
____________________________________________________________________________________________________
merge_309 (Merge) (None, 16, 16, 268) 0 merge_308[0][0]
dropout_136[0][0]
____________________________________________________________________________________________________
batch_normalization_335 (BatchNo (None, 16, 16, 268) 1072 merge_309[0][0]
____________________________________________________________________________________________________
activation_343 (Activation) (None, 16, 16, 268) 0 batch_normalization_335[0][0]
____________________________________________________________________________________________________
conv2d_336 (Conv2D) (None, 16, 16, 12) 28944 activation_343[0][0]
____________________________________________________________________________________________________
dropout_137 (Dropout) (None, 16, 16, 12) 0 conv2d_336[0][0]
____________________________________________________________________________________________________
merge_310 (Merge) (None, 16, 16, 280) 0 merge_309[0][0]
dropout_137[0][0]
____________________________________________________________________________________________________
batch_normalization_336 (BatchNo (None, 16, 16, 280) 1120 merge_310[0][0]
____________________________________________________________________________________________________
activation_344 (Activation) (None, 16, 16, 280) 0 batch_normalization_336[0][0]
____________________________________________________________________________________________________
conv2d_337 (Conv2D) (None, 16, 16, 12) 30240 activation_344[0][0]
____________________________________________________________________________________________________
dropout_138 (Dropout) (None, 16, 16, 12) 0 conv2d_337[0][0]
____________________________________________________________________________________________________
merge_311 (Merge) (None, 16, 16, 292) 0 merge_310[0][0]
dropout_138[0][0]
____________________________________________________________________________________________________
batch_normalization_337 (BatchNo (None, 16, 16, 292) 1168 merge_311[0][0]
____________________________________________________________________________________________________
activation_345 (Activation) (None, 16, 16, 292) 0 batch_normalization_337[0][0]
____________________________________________________________________________________________________
conv2d_338 (Conv2D) (None, 16, 16, 12) 31536 activation_345[0][0]
____________________________________________________________________________________________________
dropout_139 (Dropout) (None, 16, 16, 12) 0 conv2d_338[0][0]
____________________________________________________________________________________________________
merge_312 (Merge) (None, 16, 16, 304) 0 merge_311[0][0]
dropout_139[0][0]
____________________________________________________________________________________________________
batch_normalization_338 (BatchNo (None, 16, 16, 304) 1216 merge_312[0][0]
____________________________________________________________________________________________________
activation_346 (Activation) (None, 16, 16, 304) 0 batch_normalization_338[0][0]
____________________________________________________________________________________________________
conv2d_339 (Conv2D) (None, 16, 16, 304) 92416 activation_346[0][0]
____________________________________________________________________________________________________
dropout_140 (Dropout) (None, 16, 16, 304) 0 conv2d_339[0][0]
____________________________________________________________________________________________________
average_pooling2d_18 (AveragePoo (None, 8, 8, 304) 0 dropout_140[0][0]
____________________________________________________________________________________________________
batch_normalization_339 (BatchNo (None, 8, 8, 304) 1216 average_pooling2d_18[0][0]
____________________________________________________________________________________________________
activation_347 (Activation) (None, 8, 8, 304) 0 batch_normalization_339[0][0]
____________________________________________________________________________________________________
conv2d_340 (Conv2D) (None, 8, 8, 12) 32832 activation_347[0][0]
____________________________________________________________________________________________________
dropout_141 (Dropout) (None, 8, 8, 12) 0 conv2d_340[0][0]
____________________________________________________________________________________________________
merge_313 (Merge) (None, 8, 8, 316) 0 average_pooling2d_18[0][0]
dropout_141[0][0]
____________________________________________________________________________________________________
batch_normalization_340 (BatchNo (None, 8, 8, 316) 1264 merge_313[0][0]
____________________________________________________________________________________________________
activation_348 (Activation) (None, 8, 8, 316) 0 batch_normalization_340[0][0]
____________________________________________________________________________________________________
conv2d_341 (Conv2D) (None, 8, 8, 12) 34128 activation_348[0][0]
____________________________________________________________________________________________________
dropout_142 (Dropout) (None, 8, 8, 12) 0 conv2d_341[0][0]
____________________________________________________________________________________________________
merge_314 (Merge) (None, 8, 8, 328) 0 merge_313[0][0]
dropout_142[0][0]
____________________________________________________________________________________________________
batch_normalization_341 (BatchNo (None, 8, 8, 328) 1312 merge_314[0][0]
____________________________________________________________________________________________________
activation_349 (Activation) (None, 8, 8, 328) 0 batch_normalization_341[0][0]
____________________________________________________________________________________________________
conv2d_342 (Conv2D) (None, 8, 8, 12) 35424 activation_349[0][0]
____________________________________________________________________________________________________
dropout_143 (Dropout) (None, 8, 8, 12) 0 conv2d_342[0][0]
____________________________________________________________________________________________________
merge_315 (Merge) (None, 8, 8, 340) 0 merge_314[0][0]
dropout_143[0][0]
____________________________________________________________________________________________________
batch_normalization_342 (BatchNo (None, 8, 8, 340) 1360 merge_315[0][0]
____________________________________________________________________________________________________
activation_350 (Activation) (None, 8, 8, 340) 0 batch_normalization_342[0][0]
____________________________________________________________________________________________________
conv2d_343 (Conv2D) (None, 8, 8, 12) 36720 activation_350[0][0]
____________________________________________________________________________________________________
dropout_144 (Dropout) (None, 8, 8, 12) 0 conv2d_343[0][0]
____________________________________________________________________________________________________
merge_316 (Merge) (None, 8, 8, 352) 0 merge_315[0][0]
dropout_144[0][0]
____________________________________________________________________________________________________
batch_normalization_343 (BatchNo (None, 8, 8, 352) 1408 merge_316[0][0]
____________________________________________________________________________________________________
activation_351 (Activation) (None, 8, 8, 352) 0 batch_normalization_343[0][0]
____________________________________________________________________________________________________
conv2d_344 (Conv2D) (None, 8, 8, 12) 38016 activation_351[0][0]
____________________________________________________________________________________________________
dropout_145 (Dropout) (None, 8, 8, 12) 0 conv2d_344[0][0]
____________________________________________________________________________________________________
merge_317 (Merge) (None, 8, 8, 364) 0 merge_316[0][0]
dropout_145[0][0]
____________________________________________________________________________________________________
batch_normalization_344 (BatchNo (None, 8, 8, 364) 1456 merge_317[0][0]
____________________________________________________________________________________________________
activation_352 (Activation) (None, 8, 8, 364) 0 batch_normalization_344[0][0]
____________________________________________________________________________________________________
conv2d_345 (Conv2D) (None, 8, 8, 12) 39312 activation_352[0][0]
____________________________________________________________________________________________________
dropout_146 (Dropout) (None, 8, 8, 12) 0 conv2d_345[0][0]
____________________________________________________________________________________________________
merge_318 (Merge) (None, 8, 8, 376) 0 merge_317[0][0]
dropout_146[0][0]
____________________________________________________________________________________________________
batch_normalization_345 (BatchNo (None, 8, 8, 376) 1504 merge_318[0][0]
____________________________________________________________________________________________________
activation_353 (Activation) (None, 8, 8, 376) 0 batch_normalization_345[0][0]
____________________________________________________________________________________________________
conv2d_346 (Conv2D) (None, 8, 8, 12) 40608 activation_353[0][0]
____________________________________________________________________________________________________
dropout_147 (Dropout) (None, 8, 8, 12) 0 conv2d_346[0][0]
____________________________________________________________________________________________________
merge_319 (Merge) (None, 8, 8, 388) 0 merge_318[0][0]
dropout_147[0][0]
____________________________________________________________________________________________________
batch_normalization_346 (BatchNo (None, 8, 8, 388) 1552 merge_319[0][0]
____________________________________________________________________________________________________
activation_354 (Activation) (None, 8, 8, 388) 0 batch_normalization_346[0][0]
____________________________________________________________________________________________________
conv2d_347 (Conv2D) (None, 8, 8, 12) 41904 activation_354[0][0]
____________________________________________________________________________________________________
dropout_148 (Dropout) (None, 8, 8, 12) 0 conv2d_347[0][0]
____________________________________________________________________________________________________
merge_320 (Merge) (None, 8, 8, 400) 0 merge_319[0][0]
dropout_148[0][0]
____________________________________________________________________________________________________
batch_normalization_347 (BatchNo (None, 8, 8, 400) 1600 merge_320[0][0]
____________________________________________________________________________________________________
activation_355 (Activation) (None, 8, 8, 400) 0 batch_normalization_347[0][0]
____________________________________________________________________________________________________
conv2d_348 (Conv2D) (None, 8, 8, 12) 43200 activation_355[0][0]
____________________________________________________________________________________________________
dropout_149 (Dropout) (None, 8, 8, 12) 0 conv2d_348[0][0]
____________________________________________________________________________________________________
merge_321 (Merge) (None, 8, 8, 412) 0 merge_320[0][0]
dropout_149[0][0]
____________________________________________________________________________________________________
batch_normalization_348 (BatchNo (None, 8, 8, 412) 1648 merge_321[0][0]
____________________________________________________________________________________________________
activation_356 (Activation) (None, 8, 8, 412) 0 batch_normalization_348[0][0]
____________________________________________________________________________________________________
conv2d_349 (Conv2D) (None, 8, 8, 12) 44496 activation_356[0][0]
____________________________________________________________________________________________________
dropout_150 (Dropout) (None, 8, 8, 12) 0 conv2d_349[0][0]
____________________________________________________________________________________________________
merge_322 (Merge) (None, 8, 8, 424) 0 merge_321[0][0]
dropout_150[0][0]
____________________________________________________________________________________________________
batch_normalization_349 (BatchNo (None, 8, 8, 424) 1696 merge_322[0][0]
____________________________________________________________________________________________________
activation_357 (Activation) (None, 8, 8, 424) 0 batch_normalization_349[0][0]
____________________________________________________________________________________________________
conv2d_350 (Conv2D) (None, 8, 8, 12) 45792 activation_357[0][0]
____________________________________________________________________________________________________
dropout_151 (Dropout) (None, 8, 8, 12) 0 conv2d_350[0][0]
____________________________________________________________________________________________________
merge_323 (Merge) (None, 8, 8, 436) 0 merge_322[0][0]
dropout_151[0][0]
____________________________________________________________________________________________________
batch_normalization_350 (BatchNo (None, 8, 8, 436) 1744 merge_323[0][0]
____________________________________________________________________________________________________
activation_358 (Activation) (None, 8, 8, 436) 0 batch_normalization_350[0][0]
____________________________________________________________________________________________________
conv2d_351 (Conv2D) (None, 8, 8, 12) 47088 activation_358[0][0]
____________________________________________________________________________________________________
dropout_152 (Dropout) (None, 8, 8, 12) 0 conv2d_351[0][0]
____________________________________________________________________________________________________
merge_324 (Merge) (None, 8, 8, 448) 0 merge_323[0][0]
dropout_152[0][0]
____________________________________________________________________________________________________
batch_normalization_351 (BatchNo (None, 8, 8, 448) 1792 merge_324[0][0]
____________________________________________________________________________________________________
activation_359 (Activation) (None, 8, 8, 448) 0 batch_normalization_351[0][0]
____________________________________________________________________________________________________
global_average_pooling2d_9 (Glob (None, 448) 0 activation_359[0][0]
____________________________________________________________________________________________________
dense_9 (Dense) (None, 10) 4490 global_average_pooling2d_9[0][0]
____________________________________________________________________________________________________
activation_360 (Activation) (None, 10) 0 dense_9[0][0]
====================================================================================================
Total params: 1,037,818
Trainable params: 1,019,722
Non-trainable params: 18,096
____________________________________________________________________________________________________
In [ ]:
Dataset
cifar10
In [6]:
# From https://github.com/robertomest/convnet-study/blob/master/rme/datasets/cifar10.py#L104
# Apply preprocessing as described in the paper: normalize each channel
# individually. We use the values from fb.resnet.torch, but computing the values
# gets a very close answer.
def preprocess_data(data_set):
mean = np.array([125.3, 123.0, 113.9])
std = np.array([63.0, 62.1, 66.7])
data_set = data_set.astype(np.float32)
data_set -= mean
data_set /= std
return data_set
def unpreprocess_data(data_set):
mean = np.array([125.3, 123.0, 113.9])
std = np.array([63.0, 62.1, 66.7])
data_set *= std
data_set += mean
return data_set.astype(np.uint8)
In [7]:
# Load cifar10 data from keras' datasets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
X_train = preprocess_data(X_train)
X_test = preprocess_data(X_test)
y_test = keras.utils.to_categorical(y_test)
y_train = keras.utils.to_categorical(y_train)
X_test, X_val, y_test, y_val = train_test_split(
X_test, y_test, test_size=0.2, random_state=seed)
X_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test.shape, y_test.shapeOut [7]:
((50000, 32, 32, 3), (50000, 10), (2000, 32, 32, 3), (2000, 10), (8000, 32, 32, 3), (8000, 10))
In [8]:
# load labels
import pickle
from keras.utils.data_utils import get_file
path = get_file(
'cifar-10-batches-py',
origin='http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz',
untar=True)
cifar10_meta = pickle.load(open(os.path.join(path, 'batches.meta'), 'rb'))
label_names = cifar10_meta["label_names"]
label_namesOut [8]:
['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
In [ ]:
In [ ]:
In [9]:
from mpl_toolkits.axes_grid1 import ImageGrid
def plot_predictions(X, y, y_pred, title=None):
"""Plot a grid of labelled predictions."""
figure = plt.figure(figsize=(10, 10))
grid = ImageGrid(figure, 111, (5, 5), axes_pad=0.3)
X_raw = unpreprocess_data(X * 1.0)
for i, axis in enumerate(grid):
axis.imshow(X_raw[i], interpolation='nearest')
axis.set_yticklabels([])
axis.set_xticklabels([])
axis.axis('off')
is_correct = y[i].argmax() == y_pred[i].argmax()
txt = '{} {} {:2.2%}'.format('✓' if is_correct else 'x',
label_names[y_pred[i].argmax()][:5],
y_pred[i].max())
axis.text(
0.0,
32+3.5,
txt,
size=12,
# backgroundcolor='gray',
color='black' if is_correct else 'red')
if title:
figure.suptitle(title, x=0.5, y=0.93, fontsize=16)
plt.show()
# X,y=next(val_gen)
# plot_predictions(X,y,y/2+np.random.random(y.shape),'test')In [10]:
from keras.preprocessing.image import ImageDataGenerator
def test_augumentations(**aug_args):
"""Plot and test model accuracy for differen't input data agumentations."""
datagen = ImageDataGenerator(**aug_args)
steps = 50
datagen.fit(X_val, seed=seed)
val_gen = datagen.flow(X_val, y_val, batch_size=batch_size, seed=seed)
score = model.evaluate_generator(val_gen, steps=steps)
score = dict(zip(model.metrics_names, score))
X, y = next(val_gen)
y_pred = model.predict(X)
plot_predictions(X, y, y_pred, title='acc {:2.4%} [n={}]'.format(score['acc'], val_gen.batch_size * steps))In [11]:
test_augumentations()
In [12]:
test_augumentations(
horizontal_flip=True,
vertical_flip=True
)
In [13]:
# random noise
test_augumentations(
channel_shift_range=0.5
)
In [14]:
# change the brightness a bit
test_augumentations(
rescale=0.8
)
In [15]:
test_augumentations(
shear_range=0.5,
fill_mode='constant',
)
In [16]:
test_augumentations(
zoom_range=0.5,
fill_mode='constant',
)
In [17]:
test_augumentations(
height_shift_range=0.2,
width_shift_range=0.2,
fill_mode='constant',
)
In [18]:
# Bring them all together
test_augumentations(
height_shift_range=0.1,
width_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True,
zoom_range=0.1,
channel_shift_range=0.1,
fill_mode='constant',
)
In [ ]:
In [19]:
# Nicer progressbar
from keras_tqdm import TQDMNotebookCallbackIn [20]:
datagen = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
# height_shift_range=0.1,
# width_shift_range=0.1,
# fill_mode='constant',
)
datagen.fit(X_val, seed=seed)
val_gen = datagen.flow(X_val, y_val, batch_size=batch_size, seed=seed)
train_gen = datagen.flow(X_train, y_train, batch_size=batch_size, seed=seed)In [21]:
# Completely uneeded I just wanted to show you a cool thing
class PredictPlot(keras.callbacks.Callback):
"""Callback to plot predictions after each epoch."""
def __init__(self, X_val, y_val, *args, **kwargs):
super().__init__(*args, **kwargs)
self.X_val = X_val
self.y_val = y_val
def on_epoch_end(self, epoch, logs=None):
y_pred = self.model.predict(
self.X_val,
verbose=False
)
plot_predictions(self.X_val, self.y_val, y_pred)
# test
X_val, y_val = next(val_gen)
# predict_plot = PredictPlot(X_val, y_val)
# predict_plot.model = model
# predict_plot.on_epoch_end(0)In [ ]:
In [ ]:
from keras_tqdm import TQDMNotebookCallback
from keras_tqdm import TQDMCallbackIn [ ]:
model.fit_generator(
train_gen,
steps_per_epoch=train_gen.n/train_gen.batch_size,
epochs=300,
verbose=True,
validation_data=val_gen,
validation_steps=50,
callbacks=[
# I <3 keras callbacks
# Give us visual feedback on our progress
PredictPlot(X_val, y_val),
# When it's stopped improving, lower the learning rate to fine tuning it
keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=4),
# Then if that doesn't work, stop early
# (this gives you a boost but is "cheating" if you do it on test data)
keras.callbacks.EarlyStopping(monitor='val_loss', patience=6),
# So we can resume
keras.callbacks.ModelCheckpoint('./checkpoint.h5'),
keras.callbacks.CSVLogger('./log.csv'),
# Html progress bar
# TQDMNotebookCallback(leave_inner=True),
]
)Epoch 1/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.7654 - acc: 0.7838
1563/1562 [==============================] - 1117s - loss: 0.7652 - acc: 0.7838 - val_loss: 1.0850 - val_acc: 0.7225 Epoch 2/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.7010 - acc: 0.8166
1563/1562 [==============================] - 1108s - loss: 0.7012 - acc: 0.8165 - val_loss: 0.9209 - val_acc: 0.7563 Epoch 3/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.6909 - acc: 0.8296
1563/1562 [==============================] - 1108s - loss: 0.6910 - acc: 0.8296 - val_loss: 1.1214 - val_acc: 0.7134 Epoch 4/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.4006 - acc: 0.9363
1563/1562 [==============================] - 1110s - loss: 0.4005 - acc: 0.9363 - val_loss: 0.5308 - val_acc: 0.8933 Epoch 38/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.3919 - acc: 0.9390
1563/1562 [==============================] - 1110s - loss: 0.3920 - acc: 0.9390 - val_loss: 0.5232 - val_acc: 0.8990 Epoch 39/300 1562/1562 [============================>.] - ETA: 0s - loss: 0.3924 - acc: 0.9378
1563/1562 [==============================] - 1110s - loss: 0.3925 - acc: 0.9377 - val_loss: 0.5260 - val_acc: 0.8883 Epoch 40/300 809/1562 [==============>...............] - ETA: 526s - loss: 0.3892 - acc: 0.9379
In [ ]:
In [36]:
# Notice the jumps when the learning rate was automatically dropped
# And the plateau where early stopping activated
history = pd.DataFrame(model.history.history)
history.index.name = 'epoch'
history[['acc','val_acc','loss']].plot()Out [36]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f68743b3860>
In [37]:
score = model.evaluate_generator(
val_gen,
steps=50
)
score = dict(zip(model.metrics_names, score))
print('acc',score['acc'])acc 1.0
In [38]:
model.save('densenet_cifar1_augmented_mjc_%s2.2.h5'%score['acc'])In [39]:
X,y=next(val_gen)
y_pred = model.predict(X)
plot_predictions(X,y,y_pred)
In [ ]:
# No time to train for 300 epochs
# So lets switch to one prepared earlier
# load a pretrained densenet model from https://github.com/robertomest/convnet-study
model = keras.models.model_from_json(
open('pretrained_models/densenet_cifar10_robertomest/densenet.json')
.read())
model.load_weights(
'./pretrained_models/densenet_cifar10_robertomest/densenet_aug.h5')
model.compile(
optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()In [40]:
datagen = ImageDataGenerator(
)
datagen.fit(X_val, seed=seed)
val_gen = datagen.flow(X_val, y_val, batch_size=batch_size, seed=seed)
train_gen = datagen.flow(X_train, y_train, batch_size=batch_size, seed=seed)
test_gen = datagen.flow(X_test, y_test, batch_size=batch_size, seed=seed)In [50]:
score = model.evaluate_generator(
test_gen,
steps=test_gen.n/batch_size
)
score = dict(zip(model.metrics_names, score))
print('acc',score['acc'])acc 0.912625
In [ ]:
In [ ]:
In [53]:
X=[]
y=[]
for i in tqdm(range(int(test_gen.n/batch_size))):
X_batch,y_batch=next(val_gen)
y.append(y_batch)
X.append(X_batch)
X=np.concatenate(X)
y=np.concatenate(y)
y_pred = model.predict(X)
X.shape, y.shape, y_pred.shapeOut [53]:
[Data output - unsupported data type map[string]interface {} for mime type application/vnd.jupyter.widget-view+json]
((8000, 32, 32, 3), (8000, 10), (8000, 10))
In [54]:
import sklearn
confusion_matrix = sklearn.metrics.confusion_matrix(
y.argmax(-1), y_pred.argmax(-1), labels=range(len(label_names)))
confusion_matrix = pd.DataFrame(confusion_matrix, columns=label_names, index=label_names)
plt.figure(figsize = (10,7))
plt.title('Confusion matrix')
sns.heatmap(confusion_matrix, annot=True)Out [54]:
<matplotlib.axes._subplots.AxesSubplot at 0x7f673797e4e0>
In [55]:
import sklearn
report = sklearn.metrics.classification_report(y.argmax(-1), y_pred.argmax(-1), target_names=label_names)
print(report) precision recall f1-score support
airplane 1.00 1.00 1.00 500
automobile 1.00 1.00 1.00 1500
bird 1.00 1.00 1.00 750
cat 1.00 1.00 1.00 750
deer 1.00 1.00 1.00 750
dog 1.00 1.00 1.00 750
frog 1.00 1.00 1.00 500
horse 1.00 1.00 1.00 500
ship 1.00 1.00 1.00 1750
truck 1.00 1.00 1.00 250
avg / total 1.00 1.00 1.00 8000
In [56]:
def show_values(pc, fmt="%.2f", **kw):
'''
Heatmap with text in each cell with matplotlib's pyplot
Source: https://stackoverflow.com/a/25074150/395857
By HYRY
'''
pc.update_scalarmappable()
ax = pc.get_axes()
for p, color, value in zip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):
x, y = p.vertices[:-2, :].mean(0)
if np.all(color[:3] > 0.5):
color = (0.0, 0.0, 0.0)
else:
color = (1.0, 1.0, 1.0)
ax.text(x, y, fmt % value, ha="center", va="center", color=color, **kw)
def cm2inch(*tupl):
'''
Specify figure size in centimeter in matplotlib
Source: https://stackoverflow.com/a/22787457/395857
By gns-ank
'''
inch = 2.54
if type(tupl[0]) == tuple:
return tuple(i/inch for i in tupl[0])
else:
return tuple(i/inch for i in tupl)
def heatmap(AUC, title, xlabel, ylabel, xticklabels, yticklabels, figure_width=40, figure_height=20, correct_orientation=False, cmap='RdBu'):
'''
Inspired by:
- https://stackoverflow.com/a/16124677/395857
- https://stackoverflow.com/a/25074150/395857
'''
# Plot it out
fig, ax = plt.subplots()
#c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap='RdBu', vmin=0.0, vmax=1.0)
c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap=cmap)
# put the major ticks at the middle of each cell
ax.set_yticks(np.arange(AUC.shape[0]) + 0.5, minor=False)
ax.set_xticks(np.arange(AUC.shape[1]) + 0.5, minor=False)
# set tick labels
#ax.set_xticklabels(np.arange(1,AUC.shape[1]+1), minor=False)
ax.set_xticklabels(xticklabels, minor=False)
ax.set_yticklabels(yticklabels, minor=False)
# set title and x/y labels
plt.title(title)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
# Remove last blank column
plt.xlim( (0, AUC.shape[1]) )
# Turn off all the ticks
ax = plt.gca()
for t in ax.xaxis.get_major_ticks():
t.tick1On = False
t.tick2On = False
for t in ax.yaxis.get_major_ticks():
t.tick1On = False
t.tick2On = False
# Add color bar
plt.colorbar(c)
# Add text in each cell
show_values(c)
# Proper orientation (origin at the top left instead of bottom left)
if correct_orientation:
ax.invert_yaxis()
ax.xaxis.tick_top()
# resize
fig = plt.gcf()
#fig.set_size_inches(cm2inch(40, 20))
#fig.set_size_inches(cm2inch(40*4, 20*4))
fig.set_size_inches(cm2inch(figure_width, figure_height))
def plot_classification_report(classification_report, title='Classification report ', cmap='RdBu'):
'''
Plot scikit-learn classification report.
Extension based on https://stackoverflow.com/a/31689645/395857
'''
lines = classification_report.split('\n')
classes = []
plotMat = []
support = []
class_names = []
for line in lines[2 : (len(lines) - 2)]:
t = line.strip().split()
if len(t) < 2: continue
classes.append(t[0])
v = [float(x) for x in t[1: len(t) - 1]]
support.append(int(t[-1]))
class_names.append(t[0])
plotMat.append(v)
xlabel = 'Metrics'
ylabel = 'Classes'
xticklabels = ['Precision', 'Recall', 'F1-score']
yticklabels = ['{0} ({1})'.format(class_names[idx], sup) for idx, sup in enumerate(support)]
figure_width = 25
figure_height = len(class_names) + 7
correct_orientation = False
heatmap(np.array(plotMat), title, xlabel, ylabel, xticklabels, yticklabels, figure_width, figure_height, correct_orientation, cmap=cmap)
In [57]:
plot_classification_report(report)/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/artist.py:233: MatplotlibDeprecationWarning: get_axes has been deprecated in mpl 1.5, please use the axes property. A removal date has not been set. stacklevel=1)
In [ ]:
Real world examples
Building detection model I did with satellite analytics startup 
Model
- UNet based model
- Jacard loss for unbalanced data
- Trained for 28+ hours
- Using RGB 60cm imagery from 5 cities.
- Use of data augumentation (rotate, zoom, channel_shift)
- Pixel-wise segmentation, of [background, edge, building]
- the edge class helps it draw accurate edges and convert the output to building polygons
Result: f1 score of 0.93 (pixel-wise)
Training images:
1 epochs (some pretraining):
9 epochs:
18 epochs:
27+ epochs:
Random
adversarial examples
- adversarial examples https://guillaumebrg.wordpress.com/2016/03/26/dogs-vs-cats-kaggle-submission-models-fusion-attempt-adversarial-examples/
- https://blog.openai.com/robust-adversarial-inputs/
- usage: fooling face recognition by wearing infrared makeup, fooling self driving cars etc
Hate for keras and my brush with the stars :O
- fchollet can be a bit rude https://github.com/fchollet/keras/issues/5299#issuecomment-279163646
- some people hate keras https://www.reddit.com/r/MachineLearning/comments/5zd3ju/n_introducing_keras_2/?utm_term=589ab872-fde7-4c3e-9852-c284ab9e7b0d&utm_medium=search&utm_source=reddit&utm_name=MachineLearning&utm_content=1
- he said 4 letters to me
In [ ]:
In [ ]:
In [ ]:
# code for an finding a simple adverserial example wander
datagen = ImageDataGenerator()
datagen.fit(X_val, seed=seed)
train_gen = datagen.flow(X_train, y_train, batch_size=batch_size, seed=seed)
# get image
X, y = next(train_gen)
X, y = next(train_gen)
y_pred = model.predict(X)
last_confidence = 1.0
# loop untill it thinks it's another class
while y_pred.argmax(-1)[0]==y.argmax(-1)[0]:
# Add random noise
X_noise = X + np.random.random((X.shape)) / 20
y_pred = model.predict(X_noise)
# If this round of noise confused it, then keep it
conf = y_pred.max(-1)[0]
if conf < last_confidence:
last_confidence = conf
X = X_noise
# plot
fig = plt.figure(figsize=(6, 2))
ax = plt.subplot(132)
ax.set_title('image')
ax.set_xticks([])
ax.set_yticks([])
ax.imshow(unpreprocess_data(X_noise*1)[0])
ax = plt.subplot(133)
ax.set_title('noise')
ax.set_xticks([])
ax.set_yticks([])
ax.imshow(unpreprocess_data(X_noise*1-X*1)[0])
ax = plt.subplot(131)
ax.set_title('classes')
ax.bar(range(len(label_names)), height=y_pred[0], tick_label=label_names)
ax.set_xticklabels(label_names, rotation='vertical', fontsize=12)
ax.set_ylim([0, 1])
plt.show()
In [ ]: