From dac1db06ef7e1484a3841f528d848211eec9325d Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Fri, 22 Jun 2018 19:36:29 +0100 Subject: [PATCH] DDPG Implemented --- demo/webapp/level2.html | 5 + demo/webapp/public/js/DDPG/ddpg.js | 279 ++++++++++++++++++++--- demo/webapp/public/js/DDPG/ddpg_agent.js | 168 ++++++++++++++ demo/webapp/public/js/DDPG/index.js | 36 ++- demo/webapp/public/js/DDPG/memory.js | 16 +- demo/webapp/public/js/DDPG/models.js | 172 ++++++++++---- demo/webapp/public/js/DDPG/noise.js | 4 +- demo/webapp/public/js/viewer.js | 47 ++++ src/car.ts | 38 ++- src/control_motion_engine.ts | 12 +- src/level.ts | 13 +- src/metacar.ts | 16 +- src/ui_event.ts | 1 + src/world.ts | 9 + 14 files changed, 702 insertions(+), 114 deletions(-) create mode 100644 demo/webapp/public/js/DDPG/ddpg_agent.js diff --git a/demo/webapp/level2.html b/demo/webapp/level2.html index 3d30368..d0c93ee 100644 --- a/demo/webapp/level2.html +++ b/demo/webapp/level2.html @@ -37,6 +37,10 @@

+
+ + + @@ -48,6 +52,7 @@ + diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 1fe09dc..04e95f9 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -1,40 +1,251 @@ -// This class is called from js/DDPG/index.js + +function logTfMemory(){ + let mem = tf.memory(); + console.log("numBytes:" + mem.numBytes + + "\nnumBytesInGPU:" + mem.numBytesInGPU + + "\nnumDataBuffers:" + mem.numDataBuffers + + "\nnumTensors:" + mem.numTensors); +} + +// This class is called from js/DDPG/ddpg_agent.js class DDPG { - constructor(){ - // Default Config - this.config = { - "stateSize": 25, - "nbActions": 2, - "layerNom": true, - "normalizeObservations": true, - "seed": 0, - "criticL2Reg": 0.01, - "batchSize": 64, - "actorLr": 0.0001, - "criticLr": 0.001, - "gamma": 0.99, - "rewardScale": 1, - "nbEpochs": 500, - "nbEpochsCycle": 800, - "nbTrainSteps": 50, - "nbRolloutStep": 100 - }; - // Inputs - const obsInput = tf.input({shape: [this.config.stateSize]}); - const actionInput = tf.input({shape: [this.config.nbActions]}); + /** + * @param config (Object) + * @param actor (Actor class) + * @param critic (Critic class) + * @param memory (Memory class) + * @param noise (Noise class) + */ + constructor(actor, critic, memoryPos, memoryNeg, noise, config){ + this.actor = actor; + this.critic = critic; + this.memoryPos = memoryPos; + this.memoryNeg = memoryNeg; + this.noise = noise; + this.config = config; + this.tfGamma = tf.scalar(config.gamma); - // From js/DDPG/noise.js - this.paramNoise = new AdaptiveParamNoiseSpec(); - // Buffer replay - // The baseline use 1e6 but this size should be enough - this.memory = new Memory(1000); - // Actor and Critic are from js/DDPG/models.js - this.actor = new Actor( - this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); - this.critic = new Critic( - this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed); + // Inputs + let obsInput = tf.input({batchShape: [null, this.config.stateSize]}); + let actionInput = tf.input({batchShape: [null, this.config.nbActions]}); + + if (config.normalizeObservations){ + tf.layers.batchNormalization({ + scale: true, + center: true + }).apply(obsInput); + } + + // Randomly Initialize actor network μ(s) this.actor.buildModel(obsInput); + // Randomly Initialize critic network Q(s, a) this.critic.buildModel(obsInput, actionInput); + + // Define in js/DDPG/models.js + // Init target network Q' and μ' with the same weights + this.actorTarget = copyModel(this.actor, Actor); + this.criticTarget = copyModel(this.critic, Critic); + // Perturbed Actor (See parameter space noise Exploration paper) + this.perturbedActor = copyModel(this.actor, Actor); + //this.adaptivePerturbedActor = copyModel(this.actor, Actor); + + this.actorOptimiser = tf.train.adam(this.config.actorLr); + this.criticOptimiser = tf.train.adam(this.config.criticLr); + + this.criticWeights = []; + for (let w = 0; w < this.critic.model.weights.length; w++){ + this.criticWeights.push(this.critic.model.weights[w].val); + } + + this.actorWeights = []; + for (let w = 0; w < this.actor.model.weights.length; w++){ + this.actorWeights.push(this.actor.model.weights[w].val); + } + + // Return a batch from positive and negative experiences + this.memory = { + getBatch: (size) => { + let batch1 = this.memoryPos.getBatch(size/2); + let batch2 = this.memoryNeg.getBatch(size/2); + return { + 'obs0': batch1.obs0.concat(batch2.obs0), + 'obs1': batch1.obs1.concat(batch2.obs1), + 'rewards': batch1.rewards.concat(batch2.rewards), + 'actions': batch1.actions.concat(batch2.actions), + 'terminals': batch1.terminals.concat(batch2.terminals), + }; + } + }; + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); } -}; \ No newline at end of file + + + /** + * Distance Measure for DDPG + * See parameter space noise Exploration paper + * obs (Tensor2d) Observations + */ + distanceMeasure(observations) { + return tf.tidy(() => { + const pertubedPredictions = this.perturbedActor.model.predict(observations); + const predictions = this.actor.model.predict(observations); + + const distance = tf.square(pertubedPredictions.sub(predictions)).mean().sqrt(); + return distance; + }); + } + + /** + * AdaptParamNoise + */ + adaptParamNoise(){ + const batch = this.memory.getBatch(this.config.batchSize); + const tfObs0 = tf.tensor2d(batch.obs0); + const distance = this.distanceMeasure(tfObs0); + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + + const distanceV = distance.buffer().values; + this.noise.adapt(distanceV[0]); + setMetric("Distance", distanceV[0]) + + distance.dispose(); + tfObs0.dispose(); + } + + /** + * Eval the actions and the Q function + * @param states (tf.tensor2d) + * @return actions and qValues + */ + eval(observation){ + const tfActions = this.perturbedActor.model.predict(observation); + const tfQValues = this.critic.model.predict([observation, tfActions]); + return {tfActions, tfQValues}; + } + + /** + * Get the estimation of the Q value given the state + * and the action + * @param state number[] + * @param action [a, steering] + */ + getQvalue(state, a){ + const st = tf.tensor2d([state]); + const tfa = tf.tensor2d([a]); + const q = this.critic.model.predict([st, tfa]); + const v = q.buffer().values + st.dispose(); + tfa.dispose(); + q.dispose(); + return v[0]; + } + + /** + * @param observation (tf.tensor2d) + * @return (tf.tensor1d) + */ + predict(observation){ + const tfActions = this.actor.model.predict(observation); + return tfActions; + } + + /** + * @param observation (tf.tensor2d) + * @return (tf.tensor1d) + */ + perturbedPrediction(observation){ + const tfActions = this.perturbedActor.model.predict(observation); + return tfActions; + } + + /** + * Update the two target network + */ + targetUpdate(){ + // Define in js/DDPG/models.js + //assignModel(this.critic, this.criticTarget); + //assignModel(this.actor, this.actorTarget); + targetUpdate(this.criticTarget, this.critic, this.config); + targetUpdate(this.actorTarget, this.actor, this.config); + } + + trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + + const criticLoss = this.criticOptimiser.minimize(() => { + const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); + + const tfAPredictions = this.actorTarget.model.predict(tfObs1); + const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]); + + const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions)); + + const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean(); + return loss; + }, true, this.criticWeights); + + setMetric("CriticLoss", criticLoss.buffer().values[0]); + criticLoss.dispose(); + } + + trainActor(tfObs0, it){ + for (let i = 0; i < it; i++){ + const actorLoss = this.actorOptimiser.minimize(() => { + const tfAPredictions0 = this.actor.model.predict(tfObs0); + const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]); + const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1)); + return loss; + }, true, this.actorWeights); + const vLoss = actorLoss.buffer().values[0]; + setMetric("ActorLoss", actorLoss.buffer().values[0]); + actorLoss.dispose(); + } + } + + getTfBatch(){ + // Get batch + const batch = this.memory.getBatch(this.config.batchSize); + // Convert to tensors + const tfActions = tf.tensor2d(batch.actions); + const tfObs0 = tf.tensor2d(batch.obs0); + const tfObs1 = tf.tensor2d(batch.obs1); + const _tfRewards = tf.tensor1d(batch.rewards); + const _tfTerminals = tf.tensor1d(batch.terminals); + + const tfRewards = _tfRewards.expandDims(1); + const tfTerminals = _tfTerminals.expandDims(1); + + _tfRewards.dispose(); + _tfTerminals.dispose(); + + return { + tfActions, tfObs0, tfObs1, tfRewards, tfTerminals + } + } + + async optimizeCritic(){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + } + + async optimizeActor(it=1){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + this.trainActor(tfObs0, it); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + } +} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js new file mode 100644 index 0000000..add5eaf --- /dev/null +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -0,0 +1,168 @@ + +// This class is called from js/DDPG/index.js +class DDPGAgent { + + /** + * @param env (metacar.env) Set in js/DDPG/index.js + */ + constructor(env){ + this.stopTraining = false; + this.env = env; + // Default Config + this.config = { + "stateSize": 26, + "nbActions": 2, + "layerNom": true, + "normalizeObservations": true, + "seed": 0, + "criticL2Reg": 0.01, + "batchSize": 32, + "actorLr": 0.0001, + "criticLr": 0.001, + "gamma": 0.99, + "rewardScale": 1, + "nbEpochs": 500, + "nbEpochsCycle": 100, + "nbTrainSteps": 50, + "tau": 0.001, + "paramNoiseAdaptionInterval": 50, + }; + // From js/DDPG/noise.js + this.noise = new AdaptiveParamNoiseSpec(); + + // Configure components. + + // Buffer replay + // The baseline use 1e6 but this size should be enough for this problem + this.memoryPos = new Memory(5000); + this.memoryNeg = new Memory(5000); + // Actor and Critic are from js/DDPG/models.js + this.actor = new Actor(this.config); + this.critic = new Critic(this.config); + + // Seed javascript + Math.seedrandom(0); + + this.rewardsList = []; + + // DDPG + this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config); + } + + /** + * Play one step + */ + play(){ + // Get the current state + const state = agent.env.getState().linear; + // Pick an action + const tfActions = agent.ddpg.predict(tf.tensor2d([state])); + const actions = tfActions.buffer().values; + agent.env.step([actions[0], actions[1]]); + tfActions.dispose(); + } + + /** + * Get the estimation of the Q value given the state + * and the action + * @param state number[] + * @param action [a, steering] + */ + getQvalue(state, a){ + return this.ddpg.getQvalue(state, a); + } + + stop(){ + this.stopTraining = true; + } + + /** + * Step into the training environement + * @param tfPreviousStep (tf.tensor2d) Current state + * @param mPreviousStep number[] + * @return {done, state} One boolean and the new state + */ + stepTrain(tfPreviousStep, mPreviousStep){ + // Get actions + const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + // Step in the environment with theses actions + let mAcions = tfActions.buffer().values; + let mReward = this.env.step([mAcions[0], mAcions[1]]); + this.rewardsList.push(mReward); + // Get the new observations + let mState = this.env.getState().linear; + let tfState = tf.tensor2d([mState]); + let mDone = 0; + if (mReward == -1){ + mDone = 1; + } + + // Add the new tuple to the buffer + if (mReward >= 0) + this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + else + this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + + // Dispose tensor + tfPreviousStep.dispose(); + tfActions.dispose(); + return {mDone, mState, tfState} + } + + /** + * Train DDPG Agent + */ + async train(realTime){ + this.stopTraining = false; + // One epoch + for (let e=0; e < this.config.nbEpochs; e++){ + // Perform cycles. + for (let c=0; c < this.config.nbEpochsCycle; c++){ + if (c%10==0){ + logTfMemory(); + } + this.rewardsList = []; + // Perform rollouts. + // Get current observation + let mPreviousStep = this.env.getState().linear; + let tfPreviousStep = tf.tensor2d([mPreviousStep]); + let step = 0; + console.time("LoopTime"); + for (step=0; step < 800; step++){ + let rel = this.stepTrain(tfPreviousStep, mPreviousStep); + mPreviousStep = rel.mState; + tfPreviousStep = rel.tfState; + if (rel.mDone){ + break; + } + if (this.stopTraining){ + this.env.render(true); + return; + } + if (realTime) + await tf.nextFrame(); + } + console.timeEnd("LoopTime"); + this.env.reset(); + tfPreviousStep.dispose(); + // Mean is define is js/utils.js + console.log("e="+ e +", c="+c); + setMetric("Reward", mean(this.rewardsList)); + setMetric("EpisodeDuration", step); + this.ddpg.adaptParamNoise(); + await tf.nextFrame(); + } + console.time("LoopTrain"); + for (let t=0; t < 100; t++){ + this.ddpg.optimizeCritic(); + } + for (let t=0; t < 100; t++){ + this.ddpg.optimizeActor(); + } + console.timeEnd("LoopTrain"); + this.ddpg.targetUpdate(); + } + this.env.render(true); + } + +}; \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 30ebda4..5ceb838 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -1,17 +1,41 @@ let levelUrl = metacar.level.level2; -// js/DDPG/ddpg.js -var ddpg = new DDPG(); - var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); + +// js/DDPG/ddpg.js +var agent = new DDPGAgent(env); + +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]); + env.loop(() => { let state = env.getState(); - displayState("realtime_viewer", state, 200, 200); + displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - displayScores("realtime_viewer", [], reward, []); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); -env.load(); \ No newline at end of file +env.load().then(() => { + // Train agent + env.addEvent("train", () => { + agent.train(false); + }); + + env.addEvent("play", () => { + agent.play(); + }); + + env.addEvent("TrainRealTime", () => { + env.steping(false); + agent.train(true); + }); + + env.addEvent("stop", () => { + agent.stop(); + }); + + env.addEvent("reset_env"); +}); diff --git a/demo/webapp/public/js/DDPG/memory.js b/demo/webapp/public/js/DDPG/memory.js index 6b13d58..fc8db0a 100644 --- a/demo/webapp/public/js/DDPG/memory.js +++ b/demo/webapp/public/js/DDPG/memory.js @@ -32,21 +32,24 @@ class Memory { * @return batch [] */ getBatch(batchSize){ - const arrLength = this.obs0List.length; + const arrLength = this.length; const batch = { 'obs0': [], 'obs1': [], 'rewards': [], 'actions': [], - 'terminals1': [], + 'terminals': [], }; + if (batchSize > this.length){ + return batch; + } for (let b=0; b < batchSize; b++){ let id = Math.floor(Math.random() * arrLength); batch.obs0.push(this.obs0List[id]); batch.obs1.push(this.obs1List[id]); batch.rewards.push(this.rewardsList[id]); batch.actions.push(this.actionsList[id]); - batch.terminals1.push(this.terminals1List[id]); + batch.terminals.push(this.terminals1List[id]); } return batch } @@ -63,15 +66,18 @@ class Memory { this.length += 1; } else if (this.length == this.maxlen) { + //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose(); + //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose(); + //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose(); this.start = (this.start + 1) % this.maxlen; } else { console.error("Memory.append: This should never be printed"); } this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; - this.obs1List[(this.start + this.length - 1) % this.maxlen] = action; + this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1; this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; - this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1; + this.actionsList[(this.start + this.length - 1) % this.maxlen] = action; this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; } } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js index 5a97256..b47a55d 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/DDPG/models.js @@ -1,17 +1,93 @@ +/** + * Copy a model + * @param model Actor|Critic instance + * @param instance Actor|Critic + * @return Copy of the model + */ +function copyModel(model, instance){ + return tf.tidy(() => { + nModel = new instance(model.config); + // action might be not required + nModel.buildModel(model.obs, model.action); + const weights = model.model.weights; + for (let m=0; m < weights.length; m++){ + nModel.model.weights[m].val.assign(weights[m].val); + } + return nModel; + }) +} + +/** + * Usefull method to copy a model + * @param model Actor + * @param perturbedActor Actor + * @param stddev (number) + * @return Copy of the model + */ +function assignAndStd(actor, perturbedActor, stddev){ + return tf.tidy(() => { + const weights = actor.model.weights; + for (let m=0; m < weights.length; m++){ + let shape = perturbedActor.model.weights[m].val.shape; + let randomTensor = tf.randomNormal(shape, 0, stddev); + let nValue = weights[m].val.add(randomTensor); + perturbedActor.model.weights[m].val.assign(nValue); + } + }); +} + +/** + * Usefull method to copy a model + * @param model Actor + * @param perturbedActor Actor + * @return Copy of the model + */ +function assignModel(model, targetModel){ + return tf.tidy(() => { + const weights = model.model.weights; + for (let m=0; m < weights.length; m++){ + let nValue = weights[m].val; + targetModel.model.weights[m].val.assign(nValue); + } + }); +} + + +/** + * Usefull method to copy a model + * @param target Actor|Critic + * @param perturbedActor Actor|Critic + * @param config (Object) + * @return Copy of the model + */ +function targetUpdate(target, original, config){ + return tf.tidy(() => { + const originalW = original.model.weights; + const targetW = target.model.weights; + + const one = tf.scalar(1); + const tau = tf.scalar(config.tau); + + for (let m=0; m < originalW.length; m++){ + let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau))); + target.model.weights[m].val.assign(nValue); + } + }); +} + class Actor{ /** - * @param stateSize(number) - * @param nbActions (number) - * @param layerNorm (boolean) - * @param seed (number) + @param config (Object) */ - constructor(stateSize, nbActions, layerNorm, seed) { - this.stateSize = stateSize; - this.nbActions = nbActions; - this.layerNorm = layerNorm; - this.seed = seed; + constructor(config) { + this.stateSize = config.stateSize; + this.nbActions = config.nbActions; + this.layerNorm = config.layerNorm; + this.seed = config.seed; + this.config = config; + this.obs = null; } /** @@ -19,14 +95,17 @@ class Actor{ * @param obs tf.input */ buildModel(obs){ + this.obs = obs; + + this.firstLayerBatchNorm = null; this.secondLayerBatchNorm = null; - this.relu = tf.layers.thresholdedReLU(); + this.relu1 = tf.layers.activation({activation: 'relu'}); + //this.relu2 = tf.layers.activation({activation: 'relu'}); // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ - inputShape: this.stateSize, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -40,10 +119,10 @@ class Actor{ center: true }); } - + + /* // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - inputShape: 64, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -56,11 +135,10 @@ class Actor{ scale: true, center: true }); - } + }*/ // Ouput layer this.outputLayer = tf.layers.dense({ - inputShape: 64, units: this.nbActions, kernelInitializer: tf.initializers.randomUniform({ minval: 0.003, maxval: 0.003, seed: this.seed}), @@ -68,41 +146,43 @@ class Actor{ useBias: true, biasInitializer: "zeros" }); - - // Actor prediction - const predict = () => { + + this.predict = () => { return tf.tidy(() => { let l1 = this.firstLayer.apply(obs); if (this.firstLayerBatchNorm){ l1 = this.firstLayerBatchNorm.apply(l1); } - //l1 = this.relu.apply(l1); + l1 = this.relu1.apply(l1); + /* let l2 = this.secondLayer.apply(l1); if (this.secondLayerBatchNorm){ l2 = this.secondLayerBatchNorm.apply(l2); } - //l2 = this.relu.apply(l2); - return this.outputLayer.apply(l2); + l2 = this.relu2.apply(l2); + */ + + return this.outputLayer.apply(l1); }); } - const output = predict(); - this.model = tf.model({inputs: this.obs, outputs: output}); + const output = this.predict(); + this.model = tf.model({inputs: obs, outputs: output}); } - }; class Critic { /** - * @param stateSize(number) - * @param nbActions (number) - * @param layerNorm (boolean) - * @param seed (number) + * @param config (Object) */ - constructor(stateSize, nbActions, layerNorm, seed) { - this.stateSize = stateSize; - this.nbActions = nbActions; - this.layerNorm = layerNorm; + constructor(config) { + this.stateSize = config.stateSize; + this.nbActions = config.nbActions; + this.layerNorm = config.layerNorm; + this.seed = config.seed; + this.config = config; + this.obs = null; + this.action = null; } /** @@ -111,14 +191,18 @@ class Critic { * @param action tf.input */ buildModel(obs, action){ + this.obs = obs; + this.action = action; + this.firstLayerBatchNorm = null; this.secondLayerBatchNorm = null; - this.relu = tf.layers.thresholdedReLU(); + this.relu1 = tf.layers.activation({activation: 'relu'}); + this.relu2 = tf.layers.activation({activation: 'relu'}); + this.concat = tf.layers.concatenate(); // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ - inputShape: this.stateSize, units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -135,7 +219,7 @@ class Critic { // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - inputShape: 64 + this.nbActions, // Previous layer + action + //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later @@ -152,34 +236,34 @@ class Critic { // Ouput layer this.outputLayer = tf.layers.dense({ - inputShape: 64, units: 1, kernelInitializer: tf.initializers.randomUniform({ minval: 0.003, maxval: 0.003, seed: this.seed}), - activation: 'tanh', + activation: 'linear', useBias: true, biasInitializer: "zeros" }); // Actor prediction - const predict = () => { + this.predict = () => { return tf.tidy(() => { let l1 = this.firstLayer.apply(obs); - l1 = l1.concat(action); + l1 = this.concat.apply([l1, action]); if (this.firstLayerBatchNorm){ l1 = this.firstLayerBatchNorm.apply(l1); } - //l1 = this.relu(l1); + l1 = this.relu1.apply(l1); + let l2 = this.secondLayer.apply(l1); if (this.secondLayerBatchNorm){ l2 = this.secondLayerBatchNorm.apply(l2); } - //l2 = this.relu.apply(l2); + l2 = this.relu2.apply(l2); + return this.outputLayer.apply(l2); }); } - const output = predict(); - this.model = tf.model({inputs: this.obs, outputs: output}); + const output = this.predict(); + this.model = tf.model({inputs: [obs, action], outputs: output}); } - }; diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index c173625..c3a138f 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -6,7 +6,7 @@ * See "C Adapative Scaling" Page 14 in the paper. */ -class AdaptiveParamNoiseSpec{ +class AdaptiveParamNoiseSpec { /** * @param conf Object @@ -19,7 +19,7 @@ class AdaptiveParamNoiseSpec{ this.initialStddev = conf.initialStddev || 0.1; this.desiredActionStddev = conf.initialStddev || 0.1; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; - this.currentStddev = conf.initialStddev; + this.currentStddev = this.initialStddev; } /** diff --git a/demo/webapp/public/js/viewer.js b/demo/webapp/public/js/viewer.js index dba4376..6994e0a 100644 --- a/demo/webapp/public/js/viewer.js +++ b/demo/webapp/public/js/viewer.js @@ -112,4 +112,51 @@ function displayState(id, state, width, height){ yPos += ySize; } +} + +METRICS = {}; + +function initMetricsContainer(container, metrics){ + container = document.getElementById(container); + for (let m=0; m < metrics.length; m++){ + nDiv = document.createElement("div"); + nDiv.id = 'metrics_'+metrics[m]; + nDiv.style.width = "250px"; + nDiv.style.height = "250px"; + nDiv.style.display = "inline-block"; + nDiv.style.marginRight = "10px"; + container.appendChild(nDiv); + + METRICS[metrics[m]] = new CanvasJS.Chart('metrics_'+metrics[m], { + width: 250, + height: 250, + animationEnabled: false, + theme: "light2", + title:{ + text: metrics[m], + }, + axisY:{ + includeZero: false + }, + data: [{ + type: "line", + dataPoints: [{y: 0}] + }] + }); + METRICS[metrics[m]].render(); + } +} + +function setMetric(name, value){ + let chart = METRICS[name]; + let size = chart.options.data[0].dataPoints.length - 1; + + if (chart.options.data[0].dataPoints.length > 500){ + chart.options.data[0].dataPoints = chart.options.data[0].dataPoints.slice(1, size); + size = size - 1; + } + + size = chart.options.data[0].dataPoints.length - 1; + chart.options.data[0].dataPoints.push({y: value, x: chart.options.data[0].dataPoints[size].x+1}); + chart.render(); } \ No newline at end of file diff --git a/src/car.ts b/src/car.ts index 23aae93..32be0e6 100644 --- a/src/car.ts +++ b/src/car.ts @@ -11,6 +11,7 @@ import { CAR_IMG, Sprite, MAP, ROADSIZE, Container, Graphics } from "./global"; import { RoadSprite } from "./asset_manager"; +import { runInThisContext } from "vm"; var Global_carId = 0; @@ -55,6 +56,10 @@ export interface CarSprite extends PIXI.Sprite { optionalTurn?: boolean; agent?: boolean; v?: number; + a?: number; + yaw_rate?: number; + last_a?: number; + last_yaw_rate?: number; } export interface LidarChild extends PIXI.Graphics { @@ -62,6 +67,14 @@ export interface LidarChild extends PIXI.Graphics { pt?: boolean; } +export interface State { + lidar?: number[][]; + linear?: number[]; + v?: number; + a?: number; + steering?: number; +} + export class Car { public level: Level|Editor; @@ -173,6 +186,8 @@ export class Car { Method used to restore the position of the car to the original position (as set into the json file) */ + this.core.a = 0; + this.core.v = 0; this.core.mx = this.info.mx; this.core.my = this.info.my; let road = this.level.getRoad(this.core.my, this.core.mx); @@ -181,19 +196,26 @@ export class Car { } } - getState(linear:boolean = false): number[][]|number[]{ + getState(): State { /* Get the current state of the car The state is the current value of each point of the lidar. */ - if (!linear) - return this.motion.state.map(function(arr: any) { return arr.slice(); }); - else{ - let state: number[] = []; - this.motion.state.map((row: number[]) => { state = state.concat(row);}); - return state; - } + let nState: State = {}; + + let linear: number[] = []; + this.motion.state.map((row: number[]) => { linear = linear.concat(row);}); + linear.push(this.core.v); + + nState.linear = linear; + nState.lidar = this.motion.state.map(function(arr: any) { return arr.slice(); }); + + nState.v = this.core.v; + nState.a = this.core.last_a; + nState.steering = this.core.last_yaw_rate; + + return nState; } step(delta: number, action:number|number[]=null){ diff --git a/src/control_motion_engine.ts b/src/control_motion_engine.ts index 73f93cc..bb68b3b 100644 --- a/src/control_motion_engine.ts +++ b/src/control_motion_engine.ts @@ -10,6 +10,7 @@ import * as U from "./utils"; export class ControlMotionEngine extends MotionEngine { private actions: (string|number)[]; + private maxSpeed: number = 1.5; constructor(level: Level) { /* @@ -133,6 +134,9 @@ export class ControlMotionEngine extends MotionEngine { Step into the environement @delta (Float) time since the last update */ + this.car.last_a = this.car.a; + this.car.last_yaw_rate = this.car.yaw_rate; + // The car lose speed over time if (this.car.v > 0 && this.car.a == 0) this.car.v = Math.max(0, this.car.v - 0.01); @@ -140,14 +144,14 @@ export class ControlMotionEngine extends MotionEngine { this.car.v = Math.min(0, this.car.v + 0.01); if (this.car.a > 0 && this.car.v >= 0) - this.car.v = Math.min(1.5, this.car.v + this.car.a*0.01); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01); else if (this.car.a > 0 && this.car.v < 0){ - this.car.v = Math.min(1.5, this.car.v + this.car.a*0.03); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.a < 0 && this.car.v <= 0) - this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.01); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01); else if (this.car.a < 0 && this.car.v > 0){ - this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.03); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.yaw_rate == 0){ this.car.x += this.car.v * Math.cos(this.car.rotation)*delta; diff --git a/src/level.ts b/src/level.ts index d675f92..a181c8c 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,12 +110,7 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - /* - TODO: Let's the reward define in the agent class - */ - let reward = -0.1; - if (action == 0 || this.agent.core.v == 1) - reward += 0.5; + let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; } @@ -133,12 +128,16 @@ export class Level extends World { } - step(delta: number, action:number|number[]=null){ + step(delta: number, action:number|number[]=null, auto: boolean = true){ /* Process one step into the environement @delta (Float) time since the last update @action: (Integer) The action to take (can be null if no action) */ + if (auto && !this.steping){ + return; + } + // Go through all cars to move each one for (var c = 0; c < this.cars.length; c++) { if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move diff --git a/src/metacar.ts b/src/metacar.ts index 9628ff2..e3a32bc 100644 --- a/src/metacar.ts +++ b/src/metacar.ts @@ -8,7 +8,7 @@ import {UIEvent} from "./ui_event"; import * as U from "./utils"; import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine"; import { ControlMotionEngine } from "./control_motion_engine"; -import { LidarInfoI } from "./car"; +import { LidarInfoI, State } from "./car"; /** * @local Chooce whether to load a file from the computer. @@ -132,6 +132,14 @@ export class MetaCar { this.level.render(val); } + /** + * Choose wheter the environment should step automaticly + * @param val True or False + */ + public steping(val: boolean){ + this.level.setSteping(val); + } + /** * Usefull method to save/download a string as file. * @content The content of the file @@ -157,8 +165,8 @@ export class MetaCar { * The size of the state depends of the size of the Lidar. * @return The state as a 2D Array or 1D Array (linear:true) */ - public getState(linear:boolean = false): number[][]|number[]{ - return this.level.agent.getState(linear); + public getState(): State{ + return this.level.agent.getState(); } /** @@ -167,7 +175,7 @@ export class MetaCar { @return Reward value */ public step(action: number|number[]): number{ - return this.level.step(1, action); + return this.level.step(1, action, false); } /** diff --git a/src/ui_event.ts b/src/ui_event.ts index 929db62..0b2b6cd 100644 --- a/src/ui_event.ts +++ b/src/ui_event.ts @@ -61,6 +61,7 @@ export class UIEvent { // Listen the event button.addEventListener("click", () => { this.level.render(false); + this.level.setSteping(false); if (fc) fc(); }); } diff --git a/src/world.ts b/src/world.ts index ac9481e..0d76078 100644 --- a/src/world.ts +++ b/src/world.ts @@ -21,6 +21,7 @@ export class World { protected loop: any; // Loop method called for each render protected canvasId: string; // Id of the target canvas protected cars: Car[] = []; + protected steping: boolean = true; constructor(levelContent: LevelInfo, canvasId: string) { /* @@ -124,9 +125,11 @@ export class World { render(val: boolean){ if (val){ this.app.ticker.start(); + this.steping = true; } else{ this.app.ticker.stop(); + this.steping = false; } } @@ -144,4 +147,10 @@ export class World { return this.app.renderer.plugins.interaction.mouse.global; } + /** + * Stop stepping in the environment automaticly + */ + public setSteping(val: boolean): void{ + this.steping = val; + } } \ No newline at end of file