From e612abb172da9944301e3e9b4826ea8427bcc570 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Tue, 19 Jun 2018 18:55:06 +0100 Subject: [PATCH] WIP: Implement DDPG --- demo/webapp/level2.html | 8 +- demo/webapp/public/js/DDPG/ddpg.js | 40 ++++ .../public/js/{level2.js => DDPG/index.js} | 3 + demo/webapp/public/js/DDPG/memory.js | 77 ++++++++ demo/webapp/public/js/DDPG/models.js | 185 ++++++++++++++++++ demo/webapp/public/js/DDPG/noise.js | 41 ++++ 6 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 demo/webapp/public/js/DDPG/ddpg.js rename demo/webapp/public/js/{level2.js => DDPG/index.js} (89%) create mode 100644 demo/webapp/public/js/DDPG/memory.js create mode 100644 demo/webapp/public/js/DDPG/models.js create mode 100644 demo/webapp/public/js/DDPG/noise.js diff --git a/demo/webapp/level2.html b/demo/webapp/level2.html index 78bf318..3d30368 100644 --- a/demo/webapp/level2.html +++ b/demo/webapp/level2.html @@ -37,11 +37,17 @@

+ - + + + + + + diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js new file mode 100644 index 0000000..1fe09dc --- /dev/null +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -0,0 +1,40 @@ +// This class is called from js/DDPG/index.js +class DDPG { + + constructor(){ + // Default Config + this.config = { + "stateSize": 25, + "nbActions": 2, + "layerNom": true, + "normalizeObservations": true, + "seed": 0, + "criticL2Reg": 0.01, + "batchSize": 64, + "actorLr": 0.0001, + "criticLr": 0.001, + "gamma": 0.99, + "rewardScale": 1, + "nbEpochs": 500, + "nbEpochsCycle": 800, + "nbTrainSteps": 50, + "nbRolloutStep": 100 + }; + // Inputs + const obsInput = tf.input({shape: [this.config.stateSize]}); + const actionInput = tf.input({shape: [this.config.nbActions]}); + + // From js/DDPG/noise.js + this.paramNoise = new AdaptiveParamNoiseSpec(); + // Buffer replay + // The baseline use 1e6 but this size should be enough + this.memory = new Memory(1000); + // Actor and Critic are from js/DDPG/models.js + this.actor = new Actor( + this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + this.critic = new Critic( + this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + this.actor.buildModel(obsInput); + this.critic.buildModel(obsInput, actionInput); + } +}; \ No newline at end of file diff --git a/demo/webapp/public/js/level2.js b/demo/webapp/public/js/DDPG/index.js similarity index 89% rename from demo/webapp/public/js/level2.js rename to demo/webapp/public/js/DDPG/index.js index 5b1b5b7..30ebda4 100644 --- a/demo/webapp/public/js/level2.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -1,5 +1,8 @@ let levelUrl = metacar.level.level2; +// js/DDPG/ddpg.js +var ddpg = new DDPG(); + var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); diff --git a/demo/webapp/public/js/DDPG/memory.js b/demo/webapp/public/js/DDPG/memory.js new file mode 100644 index 0000000..6b13d58 --- /dev/null +++ b/demo/webapp/public/js/DDPG/memory.js @@ -0,0 +1,77 @@ + +class Memory { + + /** + * @param maxlen (number) Buffer limit + */ + constructor(maxlen){ + this.maxlen = maxlen; + this.length = 0; + this.start = 0; + + this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); + } + + /** + * @param idx (number) + */ + getItem(idx){ + if (idx < 0 || idx >= this.length){ + console.error("Memory.getItem: idx not in range."); + } + return this.data[(this.start + idx) % this.maxlen] + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + getBatch(batchSize){ + const arrLength = this.obs0List.length; + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals1': [], + }; + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * arrLength); + batch.obs0.push(this.obs0List[id]); + batch.obs1.push(this.obs1List[id]); + batch.rewards.push(this.rewardsList[id]); + batch.actions.push(this.actionsList[id]); + batch.terminals1.push(this.terminals1List[id]); + } + return batch + } + + /** + * @param obs0 [] + * @param action (number) + * @param reward (number) + * @param obs1 [] + * @param terminal1 (boolean) + */ + append(obs0, action, reward, obs1, terminal1){ + if (this.length < this.maxlen){ + this.length += 1; + } + else if (this.length == this.maxlen) { + this.start = (this.start + 1) % this.maxlen; + } + else { + console.error("Memory.append: This should never be printed"); + } + this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; + this.obs1List[(this.start + this.length - 1) % this.maxlen] = action; + this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; + this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1; + this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; + } +} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js new file mode 100644 index 0000000..5a97256 --- /dev/null +++ b/demo/webapp/public/js/DDPG/models.js @@ -0,0 +1,185 @@ + +class Actor{ + + /** + * @param stateSize(number) + * @param nbActions (number) + * @param layerNorm (boolean) + * @param seed (number) + */ + constructor(stateSize, nbActions, layerNorm, seed) { + this.stateSize = stateSize; + this.nbActions = nbActions; + this.layerNorm = layerNorm; + this.seed = seed; + } + + /** + * + * @param obs tf.input + */ + buildModel(obs){ + this.firstLayerBatchNorm = null; + this.secondLayerBatchNorm = null; + + this.relu = tf.layers.thresholdedReLU(); + + // First layer with BatchNormalization + this.firstLayer = tf.layers.dense({ + inputShape: this.stateSize, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.firstLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Second layer with BatchNormalization + this.secondLayer = tf.layers.dense({ + inputShape: 64, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.secondLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Ouput layer + this.outputLayer = tf.layers.dense({ + inputShape: 64, + units: this.nbActions, + kernelInitializer: tf.initializers.randomUniform({ + minval: 0.003, maxval: 0.003, seed: this.seed}), + activation: 'tanh', + useBias: true, + biasInitializer: "zeros" + }); + + // Actor prediction + const predict = () => { + return tf.tidy(() => { + let l1 = this.firstLayer.apply(obs); + if (this.firstLayerBatchNorm){ + l1 = this.firstLayerBatchNorm.apply(l1); + } + //l1 = this.relu.apply(l1); + let l2 = this.secondLayer.apply(l1); + if (this.secondLayerBatchNorm){ + l2 = this.secondLayerBatchNorm.apply(l2); + } + //l2 = this.relu.apply(l2); + return this.outputLayer.apply(l2); + }); + } + const output = predict(); + this.model = tf.model({inputs: this.obs, outputs: output}); + } + +}; + +class Critic { + + /** + * @param stateSize(number) + * @param nbActions (number) + * @param layerNorm (boolean) + * @param seed (number) + */ + constructor(stateSize, nbActions, layerNorm, seed) { + this.stateSize = stateSize; + this.nbActions = nbActions; + this.layerNorm = layerNorm; + } + + /** + * + * @param obs tf.input + * @param action tf.input + */ + buildModel(obs, action){ + this.firstLayerBatchNorm = null; + this.secondLayerBatchNorm = null; + + this.relu = tf.layers.thresholdedReLU(); + + // First layer with BatchNormalization + this.firstLayer = tf.layers.dense({ + inputShape: this.stateSize, + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.firstLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Second layer with BatchNormalization + this.secondLayer = tf.layers.dense({ + inputShape: 64 + this.nbActions, // Previous layer + action + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + if (this.layerNorm){ + // WARNING: BatchNormalization instead of layerNormalization + this.secondLayerBatchNorm = tf.layers.batchNormalization({ + scale: true, + center: true + }); + } + + // Ouput layer + this.outputLayer = tf.layers.dense({ + inputShape: 64, + units: 1, + kernelInitializer: tf.initializers.randomUniform({ + minval: 0.003, maxval: 0.003, seed: this.seed}), + activation: 'tanh', + useBias: true, + biasInitializer: "zeros" + }); + + // Actor prediction + const predict = () => { + return tf.tidy(() => { + let l1 = this.firstLayer.apply(obs); + l1 = l1.concat(action); + if (this.firstLayerBatchNorm){ + l1 = this.firstLayerBatchNorm.apply(l1); + } + //l1 = this.relu(l1); + let l2 = this.secondLayer.apply(l1); + if (this.secondLayerBatchNorm){ + l2 = this.secondLayerBatchNorm.apply(l2); + } + //l2 = this.relu.apply(l2); + return this.outputLayer.apply(l2); + }); + } + const output = predict(); + this.model = tf.model({inputs: this.obs, outputs: output}); + } + +}; diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js new file mode 100644 index 0000000..c173625 --- /dev/null +++ b/demo/webapp/public/js/DDPG/noise.js @@ -0,0 +1,41 @@ +/** + * Noise class + * The original baseline is made of three noise + * AdaptiveParamNoiseSpec, ActionNoise and NormalActionNoise + * Only AdaptiveParamNoiseSpec is implemented for now + * See "C Adapative Scaling" Page 14 in the paper. + */ + +class AdaptiveParamNoiseSpec{ + + /** + * @param conf Object + * conf.initialStddev: 0.1 default // σ + * conf.desiredActionStddev: 0.1 default // δ + * conf.adoptionCoefficient: 1.01 default // α + */ + constructor(conf){ + conf = conf || {}; + this.initialStddev = conf.initialStddev || 0.1; + this.desiredActionStddev = conf.initialStddev || 0.1; + this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; + this.currentStddev = conf.initialStddev; + } + + /** + * The distance from the Adaptive scaling + * @param distance number + */ + adapt(distance){ + // if d(π, _π_) > δ then σ = σ/α + if (distance > this.desiredActionStddev){ + // Decrease σ + this.currentStddev /= this.adoptionCoefficient; + } + else{ + // σ = σ*α + // Increase σ + this.currentStddev *= this.adoptionCoefficient; + } + } +}; \ No newline at end of file