diff --git a/README.md b/README.md index d7d839b..2edff60 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Tensor operations are extended on top of the [ndarray](https://github.com/scijs/ - Inception V3, trained on ImageNet -- Xception V1, trained on ImageNet +- Xception, trained on ImageNet - Bidirectional LSTM for IMDB sentiment classification diff --git a/demos/src/home.js b/demos/src/home.js index d9e78dc..ae93df1 100644 --- a/demos/src/home.js +++ b/demos/src/home.js @@ -22,6 +22,11 @@ const DEMO_INFO_DEV = [ path: 'inception-v3', imagePath: '/demos/assets/inception-v3.png' }, + { + title: 'Xception, trained on ImageNet', + path: 'xception', + imagePath: '/demos/assets/inception-v3.png' + }, { title: 'Bidirectional LSTM for IMDB sentiment classification', path: 'imdb-bidirectional-lstm', @@ -50,6 +55,11 @@ const DEMO_INFO_PROD = [ path: 'inception-v3', imagePath: 'demos/assets/inception-v3.png' }, + { + title: 'Xception, trained on ImageNet', + path: 'xception', + imagePath: 'demos/assets/inception-v3.png' + }, { title: 'Bidirectional LSTM for IMDB sentiment classification', path: 'imdb-bidirectional-lstm', diff --git a/demos/src/inception-v3/inception-v3.js b/demos/src/inception-v3/inception-v3.js index 31ac21b..16b2686 100644 --- a/demos/src/inception-v3/inception-v3.js +++ b/demos/src/inception-v3/inception-v3.js @@ -190,6 +190,7 @@ export const InceptionV3 = Vue.extend({ // data processing // see https://github.com/fchollet/keras/blob/master/keras/applications/imagenet_utils.py + // and https://github.com/fchollet/keras/blob/master/keras/applications/inception_v3.py let dataTensor = ndarray(new Float32Array(data), [width, height, 4]) let dataProcessedTensor = ndarray(new Float32Array(width * height * 3), [width, height, 3]) ops.divseq(dataTensor, 255) diff --git a/demos/src/index.js b/demos/src/index.js index 4251240..80b6b86 100644 --- a/demos/src/index.js +++ b/demos/src/index.js @@ -8,6 +8,7 @@ import { MnistCnn } from './mnist-cnn/mnist-cnn' import { MnistVae } from './mnist-vae/mnist-vae' import { ResNet50 } from './resnet50/resnet50' import { InceptionV3 } from './inception-v3/inception-v3' +import { Xception } from './xception/xception' import { ImdbBidirectionalLstm } from './imdb-bidirectional-lstm/imdb-bidirectional-lstm' Vue.component('menu', Menu) @@ -16,6 +17,7 @@ Vue.component('mnist-cnn', MnistCnn) Vue.component('mnist-vae', MnistVae) Vue.component('resnet50', ResNet50) Vue.component('inception-v3', InceptionV3) +Vue.component('xception', Xception) Vue.component('imdb-bidirectional-lstm', ImdbBidirectionalLstm) Vue.use(VueMdl.default) @@ -50,6 +52,7 @@ function matchRoute () { 'mnist-vae', 'resnet50', 'inception-v3', + 'xception', 'imdb-bidirectional-lstm' ] diff --git a/demos/src/menu.template.html b/demos/src/menu.template.html index 9e9a77a..a2c9fb5 100644 --- a/demos/src/menu.template.html +++ b/demos/src/menu.template.html @@ -31,6 +31,12 @@ ImageNet +
  • + + Xception + ImageNet + +
  • Bidirectional LSTM diff --git a/demos/src/xception/xception-arch.js b/demos/src/xception/xception-arch.js new file mode 100644 index 0000000..73bfdc0 --- /dev/null +++ b/demos/src/xception/xception-arch.js @@ -0,0 +1,1114 @@ +export const ARCHITECTURE_DIAGRAM = [ + { + className: 'Convolution2D', + name: 'block1_conv1', + row: 0, + details: '32 3x3 filters, 2x2 strides, border mode valid', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block1_conv1_bn', + row: 1, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block1_conv1_act', + row: 2, + details: 'ReLU', + col: 0 + }, + { + className: 'Convolution2D', + name: 'block1_conv2', + row: 3, + details: '64 3x3 filters, 1x1 strides, border mode valid', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block1_conv2_bn', + row: 4, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block1_conv2_act', + row: 5, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block2_sepconv1', + row: 6, + details: '128 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block2_sepconv1_bn', + row: 7, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block2_sepconv2_act', + row: 8, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block2_sepconv2', + row: 9, + details: '128 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block2_sepconv2_bn', + row: 10, + details: 'channel axis features', + col: 0 + }, + { + className: 'Convolution2D', + name: 'convolution2d_1', + row: 6, + details: '128 1x1 filters, 2x2 strides, border mode same', + col: 1 + }, + { + className: 'MaxPooling2D', + name: 'block2_pool', + row: 11, + details: '3x3 pool size, 2x2 strides, border mode same', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'batchnormalization_1', + row: 7, + details: 'channel axis features', + col: 1 + }, + { + className: 'Merge', + name: 'merge_1', + row: 12, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block3_sepconv1_act', + row: 13, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block3_sepconv1', + row: 14, + details: '256 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block3_sepconv1_bn', + row: 15, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block3_sepconv2_act', + row: 16, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block3_sepconv2', + row: 17, + details: '256 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block3_sepconv2_bn', + row: 18, + details: 'channel axis features', + col: 0 + }, + { + className: 'Convolution2D', + name: 'convolution2d_2', + row: 13, + details: '256 1x1 filters, 2x2 strides, border mode same', + col: 1 + }, + { + className: 'MaxPooling2D', + name: 'block3_pool', + row: 19, + details: '3x3 pool size, 2x2 strides, border mode same', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'batchnormalization_2', + row: 14, + details: 'channel axis features', + col: 1 + }, + { + className: 'Merge', + name: 'merge_2', + row: 20, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block4_sepconv1_act', + row: 21, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block4_sepconv1', + row: 22, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block4_sepconv1_bn', + row: 23, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block4_sepconv2_act', + row: 24, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block4_sepconv2', + row: 25, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block4_sepconv2_bn', + row: 26, + details: 'channel axis features', + col: 0 + }, + { + className: 'Convolution2D', + name: 'convolution2d_3', + row: 21, + details: '728 1x1 filters, 2x2 strides, border mode same', + col: 1 + }, + { + className: 'MaxPooling2D', + name: 'block4_pool', + row: 27, + details: '3x3 pool size, 2x2 strides, border mode same', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'batchnormalization_3', + row: 22, + details: 'channel axis features', + col: 1 + }, + { + className: 'Merge', + name: 'merge_3', + row: 28, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block5_sepconv1_act', + row: 29, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block5_sepconv1', + row: 30, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block5_sepconv1_bn', + row: 31, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block5_sepconv2_act', + row: 32, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block5_sepconv2', + row: 33, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block5_sepconv2_bn', + row: 34, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block5_sepconv3_act', + row: 35, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block5_sepconv3', + row: 36, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block5_sepconv3_bn', + row: 37, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_4', + row: 38, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block6_sepconv1_act', + row: 39, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block6_sepconv1', + row: 40, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block6_sepconv1_bn', + row: 41, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block6_sepconv2_act', + row: 42, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block6_sepconv2', + row: 43, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block6_sepconv2_bn', + row: 44, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block6_sepconv3_act', + row: 45, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block6_sepconv3', + row: 46, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block6_sepconv3_bn', + row: 47, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_5', + row: 48, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block7_sepconv1_act', + row: 49, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block7_sepconv1', + row: 50, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block7_sepconv1_bn', + row: 51, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block7_sepconv2_act', + row: 52, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block7_sepconv2', + row: 53, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block7_sepconv2_bn', + row: 54, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block7_sepconv3_act', + row: 55, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block7_sepconv3', + row: 56, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block7_sepconv3_bn', + row: 57, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_6', + row: 58, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block8_sepconv1_act', + row: 59, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block8_sepconv1', + row: 60, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block8_sepconv1_bn', + row: 61, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block8_sepconv2_act', + row: 62, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block8_sepconv2', + row: 63, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block8_sepconv2_bn', + row: 64, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block8_sepconv3_act', + row: 65, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block8_sepconv3', + row: 66, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block8_sepconv3_bn', + row: 67, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_7', + row: 68, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block9_sepconv1_act', + row: 69, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block9_sepconv1', + row: 70, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block9_sepconv1_bn', + row: 71, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block9_sepconv2_act', + row: 72, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block9_sepconv2', + row: 73, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block9_sepconv2_bn', + row: 74, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block9_sepconv3_act', + row: 75, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block9_sepconv3', + row: 76, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block9_sepconv3_bn', + row: 77, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_8', + row: 78, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block10_sepconv1_act', + row: 79, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block10_sepconv1', + row: 80, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block10_sepconv1_bn', + row: 81, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block10_sepconv2_act', + row: 82, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block10_sepconv2', + row: 83, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block10_sepconv2_bn', + row: 84, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block10_sepconv3_act', + row: 85, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block10_sepconv3', + row: 86, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block10_sepconv3_bn', + row: 87, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_9', + row: 88, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block11_sepconv1_act', + row: 89, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block11_sepconv1', + row: 90, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block11_sepconv1_bn', + row: 91, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block11_sepconv2_act', + row: 92, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block11_sepconv2', + row: 93, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block11_sepconv2_bn', + row: 94, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block11_sepconv3_act', + row: 95, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block11_sepconv3', + row: 96, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block11_sepconv3_bn', + row: 97, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_10', + row: 98, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block12_sepconv1_act', + row: 99, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block12_sepconv1', + row: 100, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block12_sepconv1_bn', + row: 101, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block12_sepconv2_act', + row: 102, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block12_sepconv2', + row: 103, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block12_sepconv2_bn', + row: 104, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block12_sepconv3_act', + row: 105, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block12_sepconv3', + row: 106, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block12_sepconv3_bn', + row: 107, + details: 'channel axis features', + col: 0 + }, + { + className: 'Merge', + name: 'merge_11', + row: 108, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'Activation', + name: 'block13_sepconv1_act', + row: 109, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block13_sepconv1', + row: 110, + details: '728 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block13_sepconv1_bn', + row: 111, + details: 'channel axis features', + col: 0 + }, + { + className: 'Activation', + name: 'block13_sepconv2_act', + row: 112, + details: 'ReLU', + col: 0 + }, + { + className: 'SeparableConvolution2D', + name: 'block13_sepconv2', + row: 113, + details: '1024 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'block13_sepconv2_bn', + row: 114, + details: 'channel axis features', + col: 0 + }, + { + className: 'Convolution2D', + name: 'convolution2d_4', + row: 109, + details: '1024 1x1 filters, 2x2 strides, border mode same', + col: 1 + }, + { + className: 'MaxPooling2D', + name: 'block13_pool', + row: 115, + details: '3x3 pool size, 2x2 strides, border mode same', + col: 0 + }, + { + className: 'BatchNormalization', + name: 'batchnormalization_4', + row: 110, + details: 'channel axis features', + col: 1 + }, + { + className: 'Merge', + name: 'merge_12', + row: 116, + details: 'sum along channel axes', + col: 1 + }, + { + className: 'SeparableConvolution2D', + name: 'block14_sepconv1', + row: 117, + details: '1536 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 1 + }, + { + className: 'BatchNormalization', + name: 'block14_sepconv1_bn', + row: 118, + details: 'channel axis features', + col: 1 + }, + { + className: 'Activation', + name: 'block14_sepconv1_act', + row: 119, + details: 'ReLU', + col: 1 + }, + { + className: 'SeparableConvolution2D', + name: 'block14_sepconv2', + row: 120, + details: '2048 3x3 filters, 1x1 strides, border mode same, depth multiplier 1', + col: 1 + }, + { + className: 'BatchNormalization', + name: 'block14_sepconv2_bn', + row: 121, + details: 'channel axis features', + col: 1 + }, + { + className: 'Activation', + name: 'block14_sepconv2_act', + row: 122, + details: 'ReLU', + col: 1 + }, + { + className: 'GlobalAveragePooling2D', + name: 'avg_pool', + row: 123, + details: '', + col: 1 + }, + { + className: 'Dense', + name: 'predictions', + row: 124, + details: 'output dimensions 1000, softmax activation', + col: 1 + } +] + +export const ARCHITECTURE_CONNECTIONS = [ + { + from: 'convolution2d_1', + to: 'predictions' + }, + + // blocks + + { + from: 'block1_conv1', + to: 'block2_pool' + }, + { + from: 'block3_sepconv1_act', + to: 'block3_pool' + }, + { + from: 'block4_sepconv1_act', + to: 'block4_pool' + }, + { + from: 'block5_sepconv1_act', + to: 'block5_sepconv3_bn' + }, + { + from: 'block6_sepconv1_act', + to: 'block6_sepconv3_bn' + }, + { + from: 'block7_sepconv1_act', + to: 'block7_sepconv3_bn' + }, + { + from: 'block8_sepconv1_act', + to: 'block8_sepconv3_bn' + }, + { + from: 'block9_sepconv1_act', + to: 'block9_sepconv3_bn' + }, + { + from: 'block10_sepconv1_act', + to: 'block10_sepconv3_bn' + }, + { + from: 'block11_sepconv1_act', + to: 'block11_sepconv3_bn' + }, + { + from: 'block12_sepconv1_act', + to: 'block12_sepconv3_bn' + }, + { + from: 'block13_sepconv1_act', + to: 'block13_pool' + }, + + // connections + + { + from: 'block1_conv2_act', + to: 'convolution2d_1', + corner: 'top-right' + }, + { + from: 'block2_pool', + to: 'merge_1', + corner: 'top-right' + }, + { + from: 'block3_pool', + to: 'merge_2', + corner: 'top-right' + }, + { + from: 'block4_pool', + to: 'merge_3', + corner: 'top-right' + }, + { + from: 'block5_sepconv3_bn', + to: 'merge_4', + corner: 'top-right' + }, + { + from: 'block6_sepconv3_bn', + to: 'merge_5', + corner: 'top-right' + }, + { + from: 'block7_sepconv3_bn', + to: 'merge_6', + corner: 'top-right' + }, + { + from: 'block8_sepconv3_bn', + to: 'merge_7', + corner: 'top-right' + }, + { + from: 'block9_sepconv3_bn', + to: 'merge_8', + corner: 'top-right' + }, + { + from: 'block10_sepconv3_bn', + to: 'merge_9', + corner: 'top-right' + }, + { + from: 'block11_sepconv3_bn', + to: 'merge_10', + corner: 'top-right' + }, + { + from: 'block12_sepconv3_bn', + to: 'merge_11', + corner: 'top-right' + }, + { + from: 'block13_pool', + to: 'merge_12', + corner: 'top-right' + }, + { + from: 'merge_1', + to: 'block3_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_2', + to: 'block4_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_3', + to: 'block5_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_4', + to: 'block6_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_5', + to: 'block7_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_6', + to: 'block8_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_7', + to: 'block9_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_8', + to: 'block10_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_9', + to: 'block11_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_10', + to: 'block12_sepconv1_act', + corner: 'top-left' + }, + { + from: 'merge_11', + to: 'block13_sepconv1_act', + corner: 'top-left' + } +] diff --git a/demos/src/xception/xception.css b/demos/src/xception/xception.css new file mode 100644 index 0000000..b3265f3 --- /dev/null +++ b/demos/src/xception/xception.css @@ -0,0 +1,216 @@ +@import '../variables.css'; + +.demo.xception { + .top-container { + margin: 10px; + position: relative; + display: flex; + + .input-container { + .input-label { + font-family: $font-3; + font-size: 16px; + color: $color-2; + text-align: left; + user-select: none; + cursor: default; + } + + .image-url { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-start; + position: relative; + + span { + margin: 0 10px; + font-family: $font-3; + font-size: 16px; + color: $color-2; + } + } + } + + .controls { + width: 250px; + margin-left: 40px; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + & > .mdl-switch { + margin-bottom: 5px; + + .mdl-switch__label { + font-size: 12px !important; + } + } + } + } + + .columns.input-output { + max-width: 800px; + margin: 0 auto; + + .column { + display: flex; + align-items: center; + justify-content: center; + } + + .column.input-column { + position: relative; + + .loading-indicator { + position: absolute; + top: 0; + left: -10px; + display: flex; + flex-direction: column; + align-self: flex-start; + + .mdl-spinner { + margin: 20px; + align-self: center; + } + + .error { + color: $color-err; + font-size: 14px; + font-family: $font-1; + margin: 20px; + } + } + + .canvas-container { + display: inline-flex; + justify-content: flex-end; + + canvas { + background: white; + } + } + } + + .column.output-column { + .output { + width: 370px; + height: 160px; + display: flex; + flex-direction: column; + align-items: flex-start; + justify-content: center; + + .output-class { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + padding: 6px 0; + + .output-label { + text-align: right; + width: 200px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font-family: $font-2; + font-size: 18px; + color: $color-2; + padding: 0 6px; + border-right: 2px solid $color-1-lighter; + } + + .output-bar { + height: 8px; + transition: width 0.2s ease-out; + } + + .output-value { + text-align: left; + margin-left: 5px; + font-family: $font-2; + font-size: 14px; + color: $color-2; + } + } + + .output-class.predicted { + .output-label { + color: $color-1; + border-left-color: $color-1; + } + + .output-value { + color: $color-1; + } + } + } + } + } + + .architecture-container { + min-width: 800px; + max-width: 1200px; + margin: 0 auto; + position: relative; + + .layers-row { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + margin-bottom: 5px; + position: relative; + z-index: 1; + + .layer-column { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + padding: 5px; + + .layer { + display: inline-block; + background: white; + border: 2px solid white; + border-radius: 5px; + padding: 2px 10px 0px; + margin: 3px; + + .layer-class-name { + color: $color-1; + font-size: 14px; + font-weight: bold; + } + + .layer-details { + color: #999999; + font-size: 12px; + font-weight: bold; + } + } + + .layer.has-result { + border-color: $color-1; + } + } + } + + .architecture-connections { + position: absolute; + top: 0; + left: 0; + z-index: 0; + + path { + stroke-width: 4px; + stroke: #AAAAAA; + fill: none; + } + } + } +} diff --git a/demos/src/xception/xception.js b/demos/src/xception/xception.js new file mode 100644 index 0000000..3975561 --- /dev/null +++ b/demos/src/xception/xception.js @@ -0,0 +1,226 @@ +/* global Vue, loadImage */ +import './xception.css' + +import ndarray from 'ndarray' +import ops from 'ndarray-ops' +import filter from 'lodash/filter' +import * as utils from '../utils' +import { IMAGE_URLS } from '../image-urls' +import { ARCHITECTURE_DIAGRAM, ARCHITECTURE_CONNECTIONS } from './xception-arch' + +const MODEL_FILEPATHS_DEV = { + model: '/demos/data/xception/xception.json', + weights: '/demos/data/xception/xception_weights.buf', + metadata: '/demos/data/xception/xception_metadata.json' +} +const MODEL_FILEPATHS_PROD = { + model: 'demos/data/xception/xception.json', + weights: 'https://transcranial.github.io/keras-js-demos-data/xception/xception_weights.buf', + metadata: 'demos/data/xception/xception_metadata.json' +} +const MODEL_CONFIG = { + filepaths: (process.env.NODE_ENV === 'production') ? MODEL_FILEPATHS_PROD : MODEL_FILEPATHS_DEV +} + +/** + * + * VUE COMPONENT + * + */ +export const Xception = Vue.extend({ + props: ['hasWebgl'], + + template: require('raw!./xception.template.html'), + + data: function () { + return { + showInfoPanel: true, + useGpu: this.hasWebgl, + model: new KerasJS.Model(Object.assign({ gpu: this.hasWebgl, layerCallPauses: true }, MODEL_CONFIG)), + modelLoading: true, + modelRunning: false, + imageURLInput: null, + imageURLSelect: null, + imageURLSelectList: IMAGE_URLS, + imageLoading: false, + imageLoadingError: false, + output: null, + architectureDiagram: ARCHITECTURE_DIAGRAM, + architectureConnections: ARCHITECTURE_CONNECTIONS, + architectureDiagramPaths: [], + showComputationFlow: true + } + }, + + computed: { + loadingProgress: function () { + return this.model.getLoadingProgress() + }, + architectureDiagramRows: function () { + let rows = [] + for (let row = 0; row < 125; row++) { + let cols = [] + for (let col = 0; col < 2; col++) { + cols.push(filter(this.architectureDiagram, { row, col })) + } + rows.push(cols) + } + return rows + }, + layersWithResults: function () { + // store as computed property for reactivity + return this.model.layersWithResults + }, + outputClasses: function () { + if (!this.output) { + let empty = [] + for (let i = 0; i < 5; i++) { + empty.push({ name: '-', probability: 0 }) + } + return empty + } + return utils.imagenetClassesTopK(this.output, 5) + } + }, + + ready: function () { + this.model.ready().then(() => { + this.modelLoading = false + + this.architectureDiagramPaths = [] + setTimeout(() => { + this.architectureConnections.forEach(conn => { + const containerElem = document.getElementsByClassName('architecture-container')[0] + const fromElem = document.getElementById(conn.from) + const toElem = document.getElementById(conn.to) + const containerElemCoords = containerElem.getBoundingClientRect() + const fromElemCoords = fromElem.getBoundingClientRect() + const toElemCoords = toElem.getBoundingClientRect() + const xContainer = containerElemCoords.left + const yContainer = containerElemCoords.top + const xFrom = fromElemCoords.left + fromElemCoords.width / 2 - xContainer + const yFrom = fromElemCoords.top + fromElemCoords.height / 2 - yContainer + const xTo = toElemCoords.left + toElemCoords.width / 2 - xContainer + const yTo = toElemCoords.top + toElemCoords.height / 2 - yContainer + + let path = `M${xFrom},${yFrom} L${xTo},${yTo}` + if (conn.corner === 'top-right') { + path = `M${xFrom},${yFrom} L${xTo - 10},${yFrom} Q${xTo},${yFrom} ${xTo},${yFrom + 10} L${xTo},${yTo}` + } else if (conn.corner === 'bottom-left') { + path = `M${xFrom},${yFrom} L${xFrom},${yTo - 10} Q${xFrom},${yTo} ${xFrom + 10},${yTo} L${xTo},${yTo}` + } else if (conn.corner === 'top-left') { + path = `M${xFrom},${yFrom} L${xTo + 10},${yFrom} Q${xTo},${yFrom} ${xTo},${yFrom + 10} L${xTo},${yTo}` + } else if (conn.corner === 'bottom-right') { + path = `M${xFrom},${yFrom} L${xFrom},${yFrom + 20} Q${xFrom},${yFrom + 30} ${xFrom - 10},${yFrom + 30} L${xTo + 10},${yFrom + 30} Q${xTo},${yFrom + 30} ${xTo},${yFrom + 40} L${xTo},${yTo}` + } + + this.architectureDiagramPaths.push(path) + }) + }, 1000) + }) + }, + + methods: { + + closeInfoPanel: function () { + this.showInfoPanel = false + }, + + toggleGpu: function () { + this.model.toggleGpu(!this.useGpu) + }, + + toggleComputationFlow: function () { + this.model.layerCallPauses = !this.showComputationFlow + }, + + imageURLInputChanged: function (e) { + this.imageURLSelect = null + this.loadImageToCanvas(this.imageURLInput) + }, + + imageURLSelectChanged: function (e) { + this.imageURLInput = this.imageURLSelect + this.loadImageToCanvas(this.imageURLSelect) + }, + + loadImageToCanvas: function (url) { + if (!url) { + this.clearAll() + return + } + + this.imageLoading = true + loadImage( + url, + img => { + if (img.type === 'error') { + this.imageLoadingError = true + this.imageLoading = false + } else { + // load image data onto input canvas + const ctx = document.getElementById('input-canvas').getContext('2d') + ctx.drawImage(img, 0, 0) + this.imageLoadingError = false + this.imageLoading = false + this.modelRunning = true + // model predict + this.$nextTick(function () { + setTimeout(() => { + this.runModel() + }, 200) + }) + } + }, + { + maxWidth: 299, + maxHeight: 299, + cover: true, + crop: true, + canvas: true, + crossOrigin: 'Anonymous' + } + ) + }, + + runModel: function () { + const ctx = document.getElementById('input-canvas').getContext('2d') + const imageData = ctx.getImageData(0, 0, ctx.canvas.width, ctx.canvas.height) + const { data, width, height } = imageData + + // data processing + // see https://github.com/fchollet/keras/blob/master/keras/applications/imagenet_utils.py + // and https://github.com/fchollet/keras/blob/master/keras/applications/xception.py + let dataTensor = ndarray(new Float32Array(data), [width, height, 4]) + let dataProcessedTensor = ndarray(new Float32Array(width * height * 3), [width, height, 3]) + ops.divseq(dataTensor, 255) + ops.subseq(dataTensor, 0.5) + ops.mulseq(dataTensor, 2) + ops.assign(dataProcessedTensor.pick(null, null, 0), dataTensor.pick(null, null, 0)) + ops.assign(dataProcessedTensor.pick(null, null, 1), dataTensor.pick(null, null, 1)) + ops.assign(dataProcessedTensor.pick(null, null, 2), dataTensor.pick(null, null, 2)) + + const inputData = { + 'input_1': dataProcessedTensor.data + } + this.model.predict(inputData).then(outputData => { + this.output = outputData['predictions'] + this.modelRunning = false + }) + }, + + clearAll: function () { + this.modelRunning = false + this.imageURLInput = null + this.imageURLSelect = null + this.imageLoading = false + this.imageLoadingError = false + this.output = null + + this.model.layersWithResults = [] + + const ctx = document.getElementById('input-canvas').getContext('2d') + ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height) + } + } +}) diff --git a/demos/src/xception/xception.template.html b/demos/src/xception/xception.template.html new file mode 100644 index 0000000..c2462ef --- /dev/null +++ b/demos/src/xception/xception.template.html @@ -0,0 +1,97 @@ +
    +
    + Xception, trained on ImageNet + +
    +
    + Loading...{{ loadingProgress }}% +
    +
    +
    + Note that ~100 MB of weights must be loaded. We use the Keras architecture from here and pretrained weights from here. Enter any valid image URL as input to the network. You can also select from a list of prepopulated image URLs. The endpoint must have CORS enabled, to enable us to extract the numeric data from the canvas element, so not all URLs will work. Imgur and Flickr creative commons all work, and are good places to start. After running the network, the top-5 classes are displayed. Keep in mind also we are limited to the 1,000 classes of ImageNet. Keep in mind that this is image classification and not object detection, so the network is forced to output a single class through softmax. Best results are on images where the classification target spans a large portion of the image. All computation performed entirely in your browser. Toggling GPU on should offer significant speedups compared to CPU. Running the network may still take several seconds (optimizations to come). With "show computational flow" toggled, computation through the network will be shown in the architecture diagram (scroll down as computation is performed layer by layer). Turning this feature off will also speed up computation. +
    +
    +
    closeCLOSE
    +
    +
    +
    +
    +
    Enter a valid image URL or select an image from the dropdown:
    +
    + + or + +
    +
    +
    + Use GPU + Show computation flow +
    +
    +
    +
    +
    + +
    Error loading URL
    +
    +
    + +
    +
    +
    +
    +
    +
    {{ outputClasses[i].name }}
    +
    +
    {{ Math.round(100 * outputClasses[i].probability) }}%
    +
    +
    +
    +
    +
    +
    +
    +
    +
    {{ layer.className }}
    +
    {{ layer.details }}
    +
    +
    +
    + + + + + +
    +