diff --git a/demos/src/_variables.css b/demos/src/_variables.css index 71a5aa6..4547169 100644 --- a/demos/src/_variables.css +++ b/demos/src/_variables.css @@ -3,8 +3,9 @@ $color-1-light: rgba(27, 188, 155, 0.6); $color-1-lighter: rgba(27, 188, 155, 0.3); $color-2: #69707a; $color-3: #393E46; +$color-4: #EB9532; $color-err: #D24D57; $font-1: 'Fira Sans', sans-serif; -$font-2: 'Inconsolata', sans-serif; +$font-2: 'Share Tech Mono', sans-serif; $font-3: 'Nothing You Could Do', cursive; diff --git a/demos/src/index.css b/demos/src/index.css index 171e900..94da212 100644 --- a/demos/src/index.css +++ b/demos/src/index.css @@ -1,5 +1,5 @@ -@import 'https://fonts.googleapis.com/css?family=Inconsolata'; @import 'https://fonts.googleapis.com/css?family=Fira+Sans'; +@import 'https://fonts.googleapis.com/css?family=Share+Tech+Mono'; @import 'https://fonts.googleapis.com/css?family=Nothing+You+Could+Do'; @import './_variables.css'; @@ -66,7 +66,7 @@ body { .mdl-textfield__input { border-bottom-color: $color-1-light; font-family: $font-2; - font-size: 16px;; + font-size: 14px;; } .mdl-textfield__label { @@ -100,7 +100,7 @@ body { .mdl-menu { .mdl-menu__item { font-family: $font-2; - font-size: 16px;; + font-size: 14px;; color: $color-2; &:hover { diff --git a/demos/src/menu.css b/demos/src/menu.css index cec6eb9..8a7594c 100644 --- a/demos/src/menu.css +++ b/demos/src/menu.css @@ -1,7 +1,7 @@ @import './_variables.css'; .menu { - padding: 50px; + padding: 40px; margin: 20px; background: white; border-radius: 10px; diff --git a/demos/src/mnist-cnn.js b/demos/src/mnist-cnn.js index 2e346eb..23dccb0 100644 --- a/demos/src/mnist-cnn.js +++ b/demos/src/mnist-cnn.js @@ -208,8 +208,10 @@ export const MnistCnn = Vue.extend({ this.input[i / 4] = data[i + 3] / 255 } - this.output = this.model.predict({ input: this.input }).output - this.getIntermediateResults() + this.model.predict({ input: this.input }).then(outputData => { + this.output = outputData.output + this.getIntermediateResults() + }) }, 200, { leading: true, trailing: true }), getIntermediateResults: function () { diff --git a/demos/src/mnist-vae.js b/demos/src/mnist-vae.js index 3b97fb5..ecff691 100644 --- a/demos/src/mnist-vae.js +++ b/demos/src/mnist-vae.js @@ -164,10 +164,11 @@ export const MnistVae = Vue.extend({ const inputData = { 'input_4': new Float32Array(this.inputCoordinates) } - const outputData = this.model.predict(inputData) - this.output = outputData['convolution2d_8'] - this.drawOutput() - this.getIntermediateResults() + this.model.predict(inputData).then(outputData => { + this.output = outputData['convolution2d_8'] + this.drawOutput() + this.getIntermediateResults() + }) }, drawOutput: function () { diff --git a/demos/src/resnet50-arch.js b/demos/src/resnet50-arch.js index 6e7ec57..d1d1c4e 100644 --- a/demos/src/resnet50-arch.js +++ b/demos/src/resnet50-arch.js @@ -1304,3 +1304,292 @@ export const ARCHITECTURE_DIAGRAM = [ col: 1 } ] + +export const ARCHITECTURE_CONNECTIONS = [ + + // main + + { + from: 'res2a_branch1', + to: 'fc1000' + }, + + // initial + conv block 2a + + { + from: 'zeropadding2d_1', + to: 'bn2a_branch2c' + }, + + // identity block 2b + + { + from: 'res2b_branch2a', + to: 'bn2b_branch2c' + }, + + // identity block 2c + + { + from: 'res2c_branch2a', + to: 'bn2c_branch2c' + }, + + // conv block 3a + + { + from: 'res3a_branch2a', + to: 'bn3a_branch2c' + }, + + // identity block 3b + + { + from: 'res3b_branch2a', + to: 'bn3b_branch2c' + }, + + // identity block 3c + + { + from: 'res3c_branch2a', + to: 'bn3c_branch2c' + }, + + // identity block 3d + + { + from: 'res3d_branch2a', + to: 'bn3d_branch2c' + }, + + // conv block 4a + + { + from: 'res4a_branch2a', + to: 'bn4a_branch2c' + }, + + // identity block 4b + + { + from: 'res4b_branch2a', + to: 'bn4b_branch2c' + }, + + // identity block 4c + + { + from: 'res4c_branch2a', + to: 'bn4c_branch2c' + }, + + // identity block 4d + + { + from: 'res4d_branch2a', + to: 'bn4d_branch2c' + }, + + // identity block 4e + + { + from: 'res4e_branch2a', + to: 'bn4e_branch2c' + }, + + // identity block 4f + + { + from: 'res4f_branch2a', + to: 'bn4f_branch2c' + }, + + // conv block 5a + + { + from: 'res5a_branch2a', + to: 'bn5a_branch2c' + }, + + // identity block 5b + + { + from: 'res5b_branch2a', + to: 'bn5b_branch2c' + }, + + // identity block 5c + + { + from: 'res5c_branch2a', + to: 'bn5c_branch2c' + }, + + // block connections start + + { + from: 'maxpooling2d_1', + to: 'res2a_branch1', + corner: 'top-right' + }, + + { + from: 'activation_4', + to: 'res2b_branch2a', + corner: 'top-left' + }, + { + from: 'activation_7', + to: 'res2c_branch2a', + corner: 'top-left' + }, + { + from: 'activation_10', + to: 'res3a_branch2a', + corner: 'top-left' + }, + { + from: 'activation_13', + to: 'res3b_branch2a', + corner: 'top-left' + }, + { + from: 'activation_16', + to: 'res3c_branch2a', + corner: 'top-left' + }, + { + from: 'activation_19', + to: 'res3d_branch2a', + corner: 'top-left' + }, + { + from: 'activation_22', + to: 'res4a_branch2a', + corner: 'top-left' + }, + { + from: 'activation_25', + to: 'res4b_branch2a', + corner: 'top-left' + }, + { + from: 'activation_28', + to: 'res4c_branch2a', + corner: 'top-left' + }, + { + from: 'activation_31', + to: 'res4d_branch2a', + corner: 'top-left' + }, + { + from: 'activation_34', + to: 'res4e_branch2a', + corner: 'top-left' + }, + { + from: 'activation_37', + to: 'res4f_branch2a', + corner: 'top-left' + }, + { + from: 'activation_40', + to: 'res5a_branch2a', + corner: 'top-left' + }, + { + from: 'activation_43', + to: 'res5b_branch2a', + corner: 'top-left' + }, + { + from: 'activation_46', + to: 'res5c_branch2a', + corner: 'top-left' + }, + + // block connections to merge + + { + from: 'bn2a_branch2c', + to: 'merge_1', + corner: 'bottom-left' + }, + { + from: 'bn2b_branch2c', + to: 'merge_2', + corner: 'bottom-left' + }, + { + from: 'bn2c_branch2c', + to: 'merge_3', + corner: 'bottom-left' + }, + { + from: 'bn3a_branch2c', + to: 'merge_4', + corner: 'bottom-left' + }, + { + from: 'bn3b_branch2c', + to: 'merge_5', + corner: 'bottom-left' + }, + { + from: 'bn3c_branch2c', + to: 'merge_6', + corner: 'bottom-left' + }, + { + from: 'bn3d_branch2c', + to: 'merge_7', + corner: 'bottom-left' + }, + { + from: 'bn4a_branch2c', + to: 'merge_8', + corner: 'bottom-left' + }, + { + from: 'bn4b_branch2c', + to: 'merge_9', + corner: 'bottom-left' + }, + { + from: 'bn4c_branch2c', + to: 'merge_10', + corner: 'bottom-left' + }, + { + from: 'bn4d_branch2c', + to: 'merge_11', + corner: 'bottom-left' + }, + { + from: 'bn4e_branch2c', + to: 'merge_12', + corner: 'bottom-left' + }, + { + from: 'bn4f_branch2c', + to: 'merge_13', + corner: 'bottom-left' + }, + { + from: 'bn5a_branch2c', + to: 'merge_14', + corner: 'bottom-left' + }, + { + from: 'bn5b_branch2c', + to: 'merge_15', + corner: 'bottom-left' + }, + { + from: 'bn5c_branch2c', + to: 'merge_16', + corner: 'bottom-left' + } +] diff --git a/demos/src/resnet50.css b/demos/src/resnet50.css index 0775f25..ebb94cd 100644 --- a/demos/src/resnet50.css +++ b/demos/src/resnet50.css @@ -133,9 +133,12 @@ .architecture-container { max-width: 800px; margin: 0 auto; + position: relative; .layers-row { margin-bottom: 5px; + position: relative; + z-index: 1; .layer { display: inline-block; @@ -156,5 +159,18 @@ } } } + + .architecture-connections { + position: absolute; + top: 0; + left: 0; + z-index: 0; + + path { + stroke-width: 4px; + stroke: #AAAAAA; + fill: none; + } + } } } diff --git a/demos/src/resnet50.js b/demos/src/resnet50.js index 158cf3a..3f596e0 100644 --- a/demos/src/resnet50.js +++ b/demos/src/resnet50.js @@ -5,7 +5,7 @@ import ndarray from 'ndarray' import ops from 'ndarray-ops' import find from 'lodash/find' import * as utils from './utils' -import { ARCHITECTURE_DIAGRAM } from './resnet50-arch' +import { ARCHITECTURE_DIAGRAM, ARCHITECTURE_CONNECTIONS } from './resnet50-arch' const MODEL_FILEPATHS_DEV = { model: '/demos/data/resnet50/resnet50.json', @@ -51,6 +51,8 @@ export const ResNet50 = Vue.extend({ imageLoadingError: false, output: null, architectureDiagram: ARCHITECTURE_DIAGRAM, + architectureConnections: ARCHITECTURE_CONNECTIONS, + architectureDiagramPaths: [], useGpu: this.hasWebgl } }, @@ -70,6 +72,9 @@ export const ResNet50 = Vue.extend({ } return rows }, + layersWithResults: function () { + return this.model.layersWithResults + }, outputClasses: function () { if (!this.output) return [] return utils.imagenetClassesTopK(this.output, 5) @@ -80,6 +85,35 @@ export const ResNet50 = Vue.extend({ this.model.ready().then(() => { this.modelLoading = false }) + + this.architectureDiagramPaths = [] + setTimeout(() => { + this.architectureConnections.forEach(conn => { + const containerElem = document.getElementsByClassName('architecture-container')[0] + const fromElem = document.getElementById(conn.from) + const toElem = document.getElementById(conn.to) + const containerElemCoords = containerElem.getBoundingClientRect() + const fromElemCoords = fromElem.getBoundingClientRect() + const toElemCoords = toElem.getBoundingClientRect() + const xContainer = containerElemCoords.left + const yContainer = containerElemCoords.top + const xFrom = fromElemCoords.left + fromElemCoords.width / 2 - xContainer + const yFrom = fromElemCoords.top + fromElemCoords.height / 2 - yContainer + const xTo = toElemCoords.left + toElemCoords.width / 2 - xContainer + const yTo = toElemCoords.top + toElemCoords.height / 2 - yContainer + + let path = `M${xFrom},${yFrom} L${xTo},${yTo}` + if (conn.corner === 'top-right') { + path = `M${xFrom},${yFrom} L${xTo - 10},${yFrom} Q${xTo},${yFrom} ${xTo},${yFrom + 10} L${xTo},${yTo}` + } else if (conn.corner === 'bottom-left') { + path = `M${xFrom},${yFrom} L${xFrom},${yTo - 10} Q${xFrom},${yTo} ${xFrom + 10},${yTo} L${xTo},${yTo}` + } else if (conn.corner === 'top-left') { + path = `M${xFrom},${yFrom} L${xTo + 10},${yFrom} Q${xTo},${yFrom} ${xTo},${yFrom + 10} L${xTo},${yTo}` + } + + this.architectureDiagramPaths.push(path) + }) + }, 1000) }, methods: { @@ -151,9 +185,10 @@ export const ResNet50 = Vue.extend({ const inputData = { 'input_1': dataProcessedTensor.data } - const outputData = this.model.predict(inputData) - this.output = outputData['fc1000'] - this.modelRunning = false + this.model.predict(inputData).then(outputData => { + this.output = outputData['fc1000'] + this.modelRunning = false + }) } } }) diff --git a/demos/src/resnet50.template.html b/demos/src/resnet50.template.html index 4542a38..4367ff0 100644 --- a/demos/src/resnet50.template.html +++ b/demos/src/resnet50.template.html @@ -7,6 +7,7 @@ Loading...{{ loadingProgress }}%
+ {{ layersWithResults }}
Enter a valid image URL or select an image from the dropdown:
{{ entry.name }}
@@ -64,11 +65,20 @@
-
+
{{ layer.className }}
{{ layer.details }}
+ + + + +
diff --git a/package.json b/package.json index 798c29a..1e6fcc6 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ }, "homepage": "https://github.com/transcranial/keras-js#readme", "dependencies": { + "bluebird": "^3.4.6", "cwise": "^1.0.9", "lodash": "^4.16.4", "ndarray": "^1.0.18", @@ -50,6 +51,7 @@ "babel-core": "^6.17.0", "babel-eslint": "^7.0.0", "babel-loader": "^6.2.5", + "babel-plugin-transform-async-to-module-method": "^6.16.0", "babel-plugin-transform-class-properties": "^6.16.0", "babel-plugin-transform-object-rest-spread": "^6.16.0", "babel-polyfill": "^6.16.0", @@ -71,7 +73,6 @@ "testGlobals", "TEST_DATA", "weblas", - "GPU", "performance" ] }, @@ -81,7 +82,14 @@ ], "plugins": [ "transform-class-properties", - "transform-object-rest-spread" + "transform-object-rest-spread", + [ + "transform-async-to-module-method", + { + "module": "bluebird", + "method": "coroutine" + } + ] ] } } diff --git a/src/Model.js b/src/Model.js index f66302b..b8e79c2 100644 --- a/src/Model.js +++ b/src/Model.js @@ -1,4 +1,5 @@ /* global XMLHttpRequest */ +import Promise from 'bluebird' import toPairs from 'lodash/toPairs' import mapKeys from 'lodash/mapKeys' import camelCase from 'lodash/camelCase' @@ -77,6 +78,9 @@ export default class Model { // map of model layers this.modelLayersMap = new Map() + // array of model layer names with result + this.layersWithResults = [] + // directed acyclic graph of model network this.modelDAG = {} @@ -132,6 +136,9 @@ export default class Model { /** * Makes XHR request + * @async + * @param {string} type - type of requested data, one of `model`, `weights`, or `metadata`. + * @param {Object} [headers] - any XHR headers to be passed along with request * @returns {Promise} */ dataRequest (type, headers = {}) { @@ -162,6 +169,7 @@ export default class Model { /** * Loading progress calculated from all the XHRs combined. + * @returns {number} progress */ getLoadingProgress () { const progressValues = values(this.xhrProgress) @@ -277,9 +285,14 @@ export default class Model { } /** - * Generator function for recursively traversing the DAG + * Async function for recursively traversing the DAG + * Graph object is stored in `this.modelDAG`, keyed by layer name. + * Layers are retrieved from Map object `this.modelLayersMap`. + * @async + * @param {[]string} nodes - array of layer names + * @returns {Promise.} */ - * traverseDAG (nodes) { + async traverseDAG (nodes) { if (nodes.length === 0) { // Stopping criterion: // an output node will have 0 outbound nodes. @@ -318,19 +331,26 @@ export default class Model { } currentLayer.hasResult = true currentLayer.visited = true + this.layersWithResults.push(currentLayer.name) + await Promise.delay(0) } - yield * this.traverseDAG(outbound) + await this.traverseDAG(outbound) } else { for (let node of nodes) { - yield * this.traverseDAG([node]) + await this.traverseDAG([node]) } } } /** - * Predict API + * Predict + * @async + * @param {Object} inputData - object where the keys are the named inputs of the model, + * and values the TypedArray numeric data + * @returns {Promise.} - outputData object where the keys are the named outputs + * of the model, and values the TypedArray numeric data */ - predict (inputData) { + async predict (inputData) { this.isRunning = true const inputNames = keys(this.inputTensors) @@ -348,6 +368,7 @@ export default class Model { layer.hasResult = false layer.visited = false } + this.layersWithResults = [] // load data to input tensors inputNames.forEach(inputName => { @@ -359,8 +380,7 @@ export default class Model { }) // start traversing DAG at input - let traversing = this.traverseDAG(inputNames) - while (!traversing.next().done) {} + await this.traverseDAG(inputNames) // outputs are layers with no outbound nodes const modelClass = this.data.model.class_name