DDPG Implemented

demo/webapp/level2.html

@@ -37,6 +37,10 @@
         </p>
     </div>
 
+    <div class="body_container" id="statContainer" style="position:relative"></div>
+
+
+    <script src="https://canvasjs.com/assets/script/canvasjs.min.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
     <script src="/dist/metacar.min.js"></script>
@@ -48,6 +52,7 @@
     <script type="text/javascript" src="/public/js/DDPG/memory.js"></script>
     <script type="text/javascript" src="/public/js/DDPG/noise.js"></script>
     <script type="text/javascript" src="/public/js/DDPG/ddpg.js"></script>
+    <script type="text/javascript" src="/public/js/DDPG/ddpg_agent.js"></script>
     <script type="text/javascript" src="/public/js/DDPG/index.js"></script>
 </body>
 </html>

demo/webapp/public/js/DDPG/ddpg.js

@@ -1,40 +1,251 @@
-// This class is called from js/DDPG/index.js
+
+function logTfMemory(){
+    let mem = tf.memory();
+    console.log("numBytes:" + mem.numBytes + 
+                "\nnumBytesInGPU:" + mem.numBytesInGPU + 
+                "\nnumDataBuffers:" + mem.numDataBuffers + 
+                "\nnumTensors:" + mem.numTensors);
+}
+
+// This class is called from js/DDPG/ddpg_agent.js
 class DDPG {
 
-    constructor(){
-        // Default Config
-        this.config = {
-            "stateSize": 25,
-            "nbActions": 2,
-            "layerNom": true,
-            "normalizeObservations": true,
-            "seed": 0,
-            "criticL2Reg": 0.01,
-            "batchSize": 64,
-            "actorLr": 0.0001,
-            "criticLr": 0.001,
-            "gamma": 0.99,
-            "rewardScale": 1,
-            "nbEpochs": 500,
-            "nbEpochsCycle": 800,
-            "nbTrainSteps": 50,
-            "nbRolloutStep": 100
-        };
-        // Inputs
-        const obsInput = tf.input({shape: [this.config.stateSize]});
-        const actionInput = tf.input({shape: [this.config.nbActions]});
+    /**
+     * @param config (Object)
+     * @param actor (Actor class)
+     * @param critic (Critic class)
+     * @param memory (Memory class)
+     * @param noise (Noise class)
+     */
+    constructor(actor, critic, memoryPos, memoryNeg, noise, config){
+        this.actor = actor;
+        this.critic = critic;
+        this.memoryPos = memoryPos;
+        this.memoryNeg = memoryNeg;
+        this.noise = noise;
+        this.config = config;
+        this.tfGamma = tf.scalar(config.gamma);
 
-        // From js/DDPG/noise.js
-        this.paramNoise = new AdaptiveParamNoiseSpec();
-        // Buffer replay
-        // The baseline use 1e6 but this size should be enough
-        this.memory = new Memory(1000);
-        // Actor and Critic are from js/DDPG/models.js
-        this.actor = new Actor(
-                this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed);
-        this.critic = new Critic(
-                this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed);
+        // Inputs
+        let obsInput = tf.input({batchShape: [null, this.config.stateSize]});
+        let actionInput = tf.input({batchShape: [null, this.config.nbActions]});
+
+        if (config.normalizeObservations){
+            tf.layers.batchNormalization({
+                scale: true,
+                center: true
+            }).apply(obsInput);
+        }
+
+        // Randomly Initialize actor network μ(s)
         this.actor.buildModel(obsInput);
+        // Randomly Initialize critic network Q(s, a)
         this.critic.buildModel(obsInput, actionInput);
+
+        // Define in js/DDPG/models.js
+        // Init target network Q' and μ' with the same weights
+        this.actorTarget = copyModel(this.actor, Actor);
+        this.criticTarget = copyModel(this.critic, Critic);
+        // Perturbed Actor (See parameter space noise Exploration paper)
+        this.perturbedActor = copyModel(this.actor, Actor);
+        //this.adaptivePerturbedActor = copyModel(this.actor, Actor);
+
+        this.actorOptimiser = tf.train.adam(this.config.actorLr);
+        this.criticOptimiser = tf.train.adam(this.config.criticLr);
+
+        this.criticWeights = [];
+        for (let w = 0; w < this.critic.model.weights.length; w++){
+            this.criticWeights.push(this.critic.model.weights[w].val);
+        }
+
+        this.actorWeights = [];
+        for (let w = 0; w < this.actor.model.weights.length; w++){
+            this.actorWeights.push(this.actor.model.weights[w].val);
+        }
+
+        // Return a batch from positive and negative experiences
+        this.memory = {
+            getBatch: (size) => {
+                let batch1 = this.memoryPos.getBatch(size/2);
+                let batch2 = this.memoryNeg.getBatch(size/2);
+                return {
+                    'obs0': batch1.obs0.concat(batch2.obs0),
+                    'obs1': batch1.obs1.concat(batch2.obs1),
+                    'rewards': batch1.rewards.concat(batch2.rewards),
+                    'actions': batch1.actions.concat(batch2.actions),
+                    'terminals': batch1.terminals.concat(batch2.terminals),
+                };
+            }
+        };
+
+        assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev);
     }
-};
+
+
+    /**
+     * Distance Measure for DDPG
+     * See parameter space noise Exploration paper
+     * obs (Tensor2d) Observations
+     */
+     distanceMeasure(observations) {
+        return tf.tidy(() => {
+            const pertubedPredictions = this.perturbedActor.model.predict(observations);
+            const predictions = this.actor.model.predict(observations);
+
+            const distance = tf.square(pertubedPredictions.sub(predictions)).mean().sqrt();
+            return distance;
+        });
+    }
+
+    /**
+     * AdaptParamNoise
+     */
+    adaptParamNoise(){
+        const batch = this.memory.getBatch(this.config.batchSize);
+        const tfObs0 = tf.tensor2d(batch.obs0);
+        const distance = this.distanceMeasure(tfObs0);
+
+        assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev);
+
+        const distanceV = distance.buffer().values;
+        this.noise.adapt(distanceV[0]);
+        setMetric("Distance", distanceV[0])
+
+        distance.dispose();
+        tfObs0.dispose();
+    }
+
+    /**
+     * Eval the actions and the Q function
+     * @param states (tf.tensor2d)
+     * @return actions and qValues
+     */
+    eval(observation){
+        const tfActions = this.perturbedActor.model.predict(observation);
+        const tfQValues = this.critic.model.predict([observation, tfActions]);
+        return {tfActions, tfQValues};
+    }
+
+    /**
+     * Get the estimation of the Q value given the state
+     * and the action
+     * @param state number[]
+     * @param action [a, steering]
+     */
+    getQvalue(state, a){
+        const st = tf.tensor2d([state]);
+        const tfa = tf.tensor2d([a]);
+        const q = this.critic.model.predict([st, tfa]);
+        const v = q.buffer().values
+        st.dispose();
+        tfa.dispose();
+        q.dispose();
+        return v[0];
+    }
+
+    /**
+     * @param observation (tf.tensor2d)
+     * @return (tf.tensor1d)
+     */
+    predict(observation){
+        const tfActions = this.actor.model.predict(observation);
+        return tfActions;
+    }
+
+    /**
+     * @param observation (tf.tensor2d)
+     * @return (tf.tensor1d)
+     */
+    perturbedPrediction(observation){
+        const tfActions = this.perturbedActor.model.predict(observation);
+        return tfActions;
+    }
+
+    /**
+     * Update the two target network
+     */
+    targetUpdate(){
+        // Define in js/DDPG/models.js
+        //assignModel(this.critic, this.criticTarget);
+        //assignModel(this.actor, this.actorTarget);
+        targetUpdate(this.criticTarget, this.critic, this.config);
+        targetUpdate(this.actorTarget, this.actor, this.config);
+    }
+    
+    trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){
+
+        const criticLoss = this.criticOptimiser.minimize(() => {
+                const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
+            
+                const tfAPredictions = this.actorTarget.model.predict(tfObs1);
+                const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]);
+        
+                const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions));
+        
+                const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean();
+                return loss;
+        }, true, this.criticWeights);
+
+        setMetric("CriticLoss", criticLoss.buffer().values[0]);
+        criticLoss.dispose();
+    }
+
+    trainActor(tfObs0, it){
+        for (let i = 0; i < it; i++){
+            const actorLoss = this.actorOptimiser.minimize(() => {
+                const tfAPredictions0 = this.actor.model.predict(tfObs0); 
+                const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]);
+                const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1));
+                return loss;
+            }, true, this.actorWeights);
+            const vLoss = actorLoss.buffer().values[0];
+            setMetric("ActorLoss", actorLoss.buffer().values[0]);
+            actorLoss.dispose(); 
+        }
+    }
+
+    getTfBatch(){
+        // Get batch
+        const batch = this.memory.getBatch(this.config.batchSize);
+        // Convert to tensors
+        const tfActions = tf.tensor2d(batch.actions);
+        const tfObs0 = tf.tensor2d(batch.obs0);
+        const tfObs1 = tf.tensor2d(batch.obs1);
+        const _tfRewards = tf.tensor1d(batch.rewards);
+        const _tfTerminals = tf.tensor1d(batch.terminals);
+
+        const tfRewards = _tfRewards.expandDims(1);
+        const tfTerminals = _tfTerminals.expandDims(1);
+
+        _tfRewards.dispose();
+        _tfTerminals.dispose();
+
+        return {
+            tfActions, tfObs0, tfObs1, tfRewards, tfTerminals
+        }
+    }
+
+    async optimizeCritic(){
+        const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
+
+        this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
+
+        tfActions.dispose();
+        tfObs0.dispose();
+        tfObs1.dispose(); 
+        tfRewards.dispose();
+        tfTerminals.dispose();
+    }
+
+    async optimizeActor(it=1){
+        const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
+
+        this.trainActor(tfObs0, it);
+
+        tfActions.dispose();
+        tfObs0.dispose();
+        tfObs1.dispose(); 
+        tfRewards.dispose();
+        tfTerminals.dispose();
+    }
+}

demo/webapp/public/js/DDPG/ddpg_agent.js

@@ -0,0 +1,168 @@
+
+// This class is called from js/DDPG/index.js
+class DDPGAgent {
+
+    /**
+     * @param env (metacar.env) Set in js/DDPG/index.js
+     */
+    constructor(env){
+        this.stopTraining = false;
+        this.env = env;
+        // Default Config
+        this.config = {
+            "stateSize": 26,
+            "nbActions": 2,
+            "layerNom": true,
+            "normalizeObservations": true,
+            "seed": 0,
+            "criticL2Reg": 0.01,
+            "batchSize": 32,
+            "actorLr": 0.0001,
+            "criticLr": 0.001,
+            "gamma": 0.99,
+            "rewardScale": 1,
+            "nbEpochs": 500,
+            "nbEpochsCycle": 100,
+            "nbTrainSteps": 50,
+            "tau": 0.001,
+            "paramNoiseAdaptionInterval": 50,
+        };
+        // From js/DDPG/noise.js
+        this.noise = new AdaptiveParamNoiseSpec();
+
+        // Configure components.
+
+        // Buffer replay
+        // The baseline use 1e6 but this size should be enough for this problem
+        this.memoryPos = new Memory(5000);
+        this.memoryNeg = new Memory(5000);
+        // Actor and Critic are from js/DDPG/models.js
+        this.actor = new Actor(this.config);
+        this.critic = new Critic(this.config);
+
+        // Seed javascript
+        Math.seedrandom(0);
+
+        this.rewardsList = [];
+
+        // DDPG
+        this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config);
+    }
+
+    /**
+     * Play one step
+     */
+    play(){
+        // Get the current state
+        const state = agent.env.getState().linear;
+        // Pick an action
+        const tfActions = agent.ddpg.predict(tf.tensor2d([state]));
+        const actions = tfActions.buffer().values;
+        agent.env.step([actions[0], actions[1]]);
+        tfActions.dispose();
+    }
+
+    /**
+     * Get the estimation of the Q value given the state
+     * and the action
+     * @param state number[]
+     * @param action [a, steering]
+     */
+    getQvalue(state, a){
+        return this.ddpg.getQvalue(state, a);
+    }
+
+    stop(){
+        this.stopTraining = true;
+    }
+
+    /**
+     * Step into the training environement
+     * @param tfPreviousStep (tf.tensor2d) Current state
+     * @param mPreviousStep number[]
+     * @return {done, state} One boolean and the new state
+     */
+    stepTrain(tfPreviousStep, mPreviousStep){
+        // Get actions
+        const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
+        // Step in the environment with theses actions
+        let mAcions = tfActions.buffer().values;
+        let mReward = this.env.step([mAcions[0], mAcions[1]]);
+        this.rewardsList.push(mReward);
+        // Get the new observations
+        let mState = this.env.getState().linear;
+        let tfState = tf.tensor2d([mState]);
+        let mDone = 0;
+        if (mReward == -1){
+            mDone = 1;
+        }
+
+        // Add the new tuple to the buffer
+        if (mReward >= 0)
+            this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
+        else
+            this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
+
+        // Dispose tensor
+        tfPreviousStep.dispose();
+        tfActions.dispose();
+        return {mDone, mState, tfState}
+    }
+
+    /**
+     * Train DDPG Agent
+     */
+    async train(realTime){
+        this.stopTraining = false;
+        // One epoch
+        for (let e=0; e < this.config.nbEpochs; e++){
+            // Perform cycles.
+            for (let c=0; c < this.config.nbEpochsCycle; c++){
+                if (c%10==0){
+                    logTfMemory();
+                }
+                this.rewardsList = [];
+                // Perform rollouts.
+                // Get current observation
+                let mPreviousStep = this.env.getState().linear;
+                let tfPreviousStep = tf.tensor2d([mPreviousStep]);
+                let step = 0;
+                console.time("LoopTime");
+                for (step=0; step < 800; step++){
+                    let rel = this.stepTrain(tfPreviousStep, mPreviousStep);
+                    mPreviousStep = rel.mState;
+                    tfPreviousStep = rel.tfState;
+                    if (rel.mDone){
+                        break;
+                    }
+                    if (this.stopTraining){
+                        this.env.render(true);
+                        return;
+                    }
+                    if (realTime)
+                        await tf.nextFrame();
+                }
+                console.timeEnd("LoopTime");
+                this.env.reset();
+                tfPreviousStep.dispose();
+                // Mean is define is js/utils.js                
+                console.log("e="+ e +", c="+c);
+                setMetric("Reward", mean(this.rewardsList));
+                setMetric("EpisodeDuration", step);
+                this.ddpg.adaptParamNoise();
+                await tf.nextFrame();
+            }
+            console.time("LoopTrain");
+            for (let t=0; t < 100; t++){
+                this.ddpg.optimizeCritic();
+            }
+            for (let t=0; t < 100; t++){
+                this.ddpg.optimizeActor();
+            }
+            console.timeEnd("LoopTrain");
+            this.ddpg.targetUpdate();
+        }
+        this.env.render(true);
+    }
+
+};

demo/webapp/public/js/DDPG/index.js

@@ -1,17 +1,41 @@
 let levelUrl = metacar.level.level2;
 
-// js/DDPG/ddpg.js
-var ddpg = new DDPG();
-
 var env = new metacar.env("canvas", levelUrl);
 
 env.setAgentMotion(metacar.motion.ControlMotion, {});
 
+
+// js/DDPG/ddpg.js
+var agent = new DDPGAgent(env);
+
+initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]);
+
 env.loop(() => {
     let state = env.getState();
-    displayState("realtime_viewer", state, 200, 200);
+    displayState("realtime_viewer", state.lidar, 200, 200);
     let reward = env.getLastReward();
-    displayScores("realtime_viewer", [], reward, []);
+    const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
+    displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]);
 });
 
-env.load();
+env.load().then(() => {
+    // Train agent
+    env.addEvent("train", () => {
+        agent.train(false);
+    });
+
+    env.addEvent("play", () => {
+        agent.play();
+    });
+
+    env.addEvent("TrainRealTime", () => {
+        env.steping(false);
+        agent.train(true);
+    });
+
+    env.addEvent("stop", () => {
+        agent.stop();
+    });
+
+    env.addEvent("reset_env"); 
+});

demo/webapp/public/js/DDPG/memory.js

@@ -32,21 +32,24 @@ class Memory {
      * @return batch []
      */
     getBatch(batchSize){
-        const arrLength = this.obs0List.length;
+        const arrLength = this.length;
         const batch =  {
             'obs0': [],
             'obs1': [],
             'rewards': [],
             'actions': [],
-            'terminals1': [],
+            'terminals': [],
         };
+        if (batchSize > this.length){
+            return batch;
+        }
         for (let b=0; b < batchSize; b++){
             let id = Math.floor(Math.random() * arrLength);
             batch.obs0.push(this.obs0List[id]);
             batch.obs1.push(this.obs1List[id]);
             batch.rewards.push(this.rewardsList[id]);
             batch.actions.push(this.actionsList[id]);
-            batch.terminals1.push(this.terminals1List[id]);
+            batch.terminals.push(this.terminals1List[id]);
         }
         return batch
     }
@@ -63,15 +66,18 @@ class Memory {
             this.length += 1;
         }
         else if (this.length == this.maxlen) {
+            //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
+            //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
+            //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
             this.start = (this.start + 1) % this.maxlen;
         }
         else {
             console.error("Memory.append: This should never be printed");
         }
         this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
-        this.obs1List[(this.start + this.length - 1) % this.maxlen] = action;
+        this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
         this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
-        this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1;
+        this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
         this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
     }
 }

demo/webapp/public/js/DDPG/models.js

@@ -1,17 +1,93 @@
+/**
+ * Copy a model
+ * @param model Actor|Critic instance
+ * @param instance Actor|Critic
+ * @return Copy of the model
+ */
+function copyModel(model, instance){
+    return tf.tidy(() => {
+        nModel = new instance(model.config);
+        // action might be not required
+        nModel.buildModel(model.obs, model.action);
+        const weights = model.model.weights;
+        for (let m=0; m < weights.length; m++){
+            nModel.model.weights[m].val.assign(weights[m].val);
+        }
+        return nModel;
+    })
+}
+
+/**
+ * Usefull method to copy a model
+ * @param model Actor
+ * @param perturbedActor Actor
+ * @param stddev (number)
+ * @return Copy of the model
+ */
+function assignAndStd(actor, perturbedActor, stddev){
+    return tf.tidy(() => {
+        const weights = actor.model.weights;
+        for (let m=0; m < weights.length; m++){
+            let shape = perturbedActor.model.weights[m].val.shape;
+            let randomTensor = tf.randomNormal(shape, 0, stddev);
+            let nValue = weights[m].val.add(randomTensor);
+            perturbedActor.model.weights[m].val.assign(nValue);
+        }
+    });
+}
+
+/**
+ * Usefull method to copy a model
+ * @param model Actor
+ * @param perturbedActor Actor
+ * @return Copy of the model
+ */
+function assignModel(model, targetModel){
+    return tf.tidy(() => {
+        const weights = model.model.weights;
+        for (let m=0; m < weights.length; m++){
+            let nValue = weights[m].val;
+            targetModel.model.weights[m].val.assign(nValue);
+        }
+    });
+}
+
+
+/**
+ * Usefull method to copy a model
+ * @param target Actor|Critic
+ * @param perturbedActor Actor|Critic
+ * @param config (Object)
+ * @return Copy of the model
+ */
+function targetUpdate(target, original, config){
+    return tf.tidy(() => {
+        const originalW = original.model.weights;
+        const targetW = target.model.weights;
+    
+        const one = tf.scalar(1);
+        const tau = tf.scalar(config.tau);
+    
+        for (let m=0; m < originalW.length; m++){
+            let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau)));
+            target.model.weights[m].val.assign(nValue);
+        }
+    });
+}
+
 
 class Actor{
 
     /**
-     * @param stateSize(number)
-     * @param nbActions (number)
-     * @param layerNorm (boolean)
-     * @param seed (number)
+        @param config (Object)
      */
-    constructor(stateSize, nbActions, layerNorm, seed) {
-        this.stateSize = stateSize;
-        this.nbActions = nbActions;
-        this.layerNorm = layerNorm;
-        this.seed = seed;
+    constructor(config) {
+        this.stateSize = config.stateSize;
+        this.nbActions = config.nbActions;
+        this.layerNorm = config.layerNorm;
+        this.seed = config.seed;
+        this.config = config;
+        this.obs = null;
     }
 
     /**
@@ -19,14 +95,17 @@ class Actor{
      * @param obs tf.input
      */
     buildModel(obs){
+        this.obs = obs;
+
+
         this.firstLayerBatchNorm = null;    
         this.secondLayerBatchNorm = null;
 
-        this.relu = tf.layers.thresholdedReLU();
+        this.relu1 = tf.layers.activation({activation: 'relu'});
+        //this.relu2 = tf.layers.activation({activation: 'relu'});
 
         // First layer with BatchNormalization
         this.firstLayer = tf.layers.dense({
-            inputShape: this.stateSize,
             units: 64,
             kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
             activation: 'linear', // relu is add later
@@ -40,10 +119,10 @@ class Actor{
                 center: true
             });
         }
-
+        
+        /*
         // Second layer with BatchNormalization
         this.secondLayer = tf.layers.dense({
-            inputShape: 64,
             units: 64,
             kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
             activation: 'linear', // relu is add later
@@ -56,11 +135,10 @@ class Actor{
                 scale: true,
                 center: true
             });
-        }
+        }*/
 
         // Ouput layer
         this.outputLayer = tf.layers.dense({
-            inputShape: 64,
             units: this.nbActions,
             kernelInitializer: tf.initializers.randomUniform({
                 minval: 0.003, maxval: 0.003, seed: this.seed}),
@@ -68,41 +146,43 @@ class Actor{
             useBias: true,
             biasInitializer: "zeros"
         });
-        
-        // Actor prediction
-        const predict = () => {
+
+        this.predict = () => {
             return tf.tidy(() => {
                 let l1 = this.firstLayer.apply(obs);
                 if (this.firstLayerBatchNorm){
                     l1 = this.firstLayerBatchNorm.apply(l1);
                 }
-                //l1 = this.relu.apply(l1);
+                l1 = this.relu1.apply(l1);
+                /*
                 let l2 = this.secondLayer.apply(l1);
                 if (this.secondLayerBatchNorm){
                     l2 = this.secondLayerBatchNorm.apply(l2);
                 }
-                //l2 = this.relu.apply(l2);
-                return this.outputLayer.apply(l2);
+                l2 = this.relu2.apply(l2);
+                */
+                
+                return this.outputLayer.apply(l1);
             });
         }
-        const output = predict();
-        this.model = tf.model({inputs: this.obs, outputs: output});
+        const output = this.predict();
+        this.model = tf.model({inputs: obs, outputs: output});
     }
-
 };
 
 class Critic {
 
     /**
-     * @param stateSize(number)
-     * @param nbActions (number)
-     * @param layerNorm (boolean)
-     * @param seed (number)
+     * @param config (Object)
      */
-    constructor(stateSize, nbActions, layerNorm, seed) {
-        this.stateSize = stateSize;
-        this.nbActions = nbActions;
-        this.layerNorm = layerNorm;
+    constructor(config) {
+        this.stateSize = config.stateSize;
+        this.nbActions = config.nbActions;
+        this.layerNorm = config.layerNorm;
+        this.seed = config.seed;
+        this.config = config;
+        this.obs = null;
+        this.action = null;
     }
 
     /**
@@ -111,14 +191,18 @@ class Critic {
      * @param action tf.input
      */
     buildModel(obs, action){
+        this.obs = obs;
+        this.action = action;
+
         this.firstLayerBatchNorm = null;    
         this.secondLayerBatchNorm = null;
 
-        this.relu = tf.layers.thresholdedReLU();
+        this.relu1 = tf.layers.activation({activation: 'relu'});
+        this.relu2 = tf.layers.activation({activation: 'relu'});
+        this.concat = tf.layers.concatenate();
 
         // First layer with BatchNormalization
         this.firstLayer = tf.layers.dense({
-            inputShape: this.stateSize,
             units: 64,
             kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
             activation: 'linear', // relu is add later
@@ -135,7 +219,7 @@ class Critic {
 
         // Second layer with BatchNormalization
         this.secondLayer = tf.layers.dense({
-            inputShape: 64 + this.nbActions, // Previous layer + action
+            //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action
             units: 64,
             kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
             activation: 'linear', // relu is add later
@@ -152,34 +236,34 @@ class Critic {
 
         // Ouput layer
         this.outputLayer = tf.layers.dense({
-            inputShape: 64,
             units: 1,
             kernelInitializer: tf.initializers.randomUniform({
                 minval: 0.003, maxval: 0.003, seed: this.seed}),
-            activation: 'tanh',
+            activation: 'linear',
             useBias: true,
             biasInitializer: "zeros"
         });
         
         // Actor prediction
-        const predict = () => {
+        this.predict = () => {
             return tf.tidy(() => {
                 let l1 = this.firstLayer.apply(obs);
-                l1 = l1.concat(action);
+                l1 = this.concat.apply([l1, action]);
                 if (this.firstLayerBatchNorm){
                     l1 = this.firstLayerBatchNorm.apply(l1);
                 }
-                //l1 = this.relu(l1);
+                l1 = this.relu1.apply(l1);
+
                 let l2 = this.secondLayer.apply(l1);
                 if (this.secondLayerBatchNorm){
                     l2 = this.secondLayerBatchNorm.apply(l2);
                 }
-                //l2 = this.relu.apply(l2);
+                l2 = this.relu2.apply(l2);
+
                 return this.outputLayer.apply(l2);
             });
         }
-        const output = predict();
-        this.model = tf.model({inputs: this.obs, outputs: output});
+        const output = this.predict();
+        this.model = tf.model({inputs: [obs, action], outputs: output});
     }
-
 };

demo/webapp/public/js/DDPG/noise.js

@@ -6,7 +6,7 @@
  * See "C Adapative Scaling" Page 14 in the paper.
  */
 
-class AdaptiveParamNoiseSpec{
+class AdaptiveParamNoiseSpec {
 
     /**
      * @param conf Object
@@ -19,7 +19,7 @@ class AdaptiveParamNoiseSpec{
         this.initialStddev = conf.initialStddev || 0.1;
         this.desiredActionStddev = conf.initialStddev || 0.1;
         this.adoptionCoefficient = conf.adoptionCoefficient || 1.01;
-        this.currentStddev = conf.initialStddev;
+        this.currentStddev = this.initialStddev;
     }
 
     /**

demo/webapp/public/js/viewer.js

@@ -112,4 +112,51 @@ function displayState(id, state, width, height){
         yPos += ySize;
     }
 
+}
+
+METRICS = {};
+
+function initMetricsContainer(container, metrics){
+    container = document.getElementById(container);
+    for (let m=0; m < metrics.length; m++){
+        nDiv = document.createElement("div");
+        nDiv.id = 'metrics_'+metrics[m];
+        nDiv.style.width = "250px";
+        nDiv.style.height = "250px";
+        nDiv.style.display = "inline-block";
+        nDiv.style.marginRight = "10px";
+        container.appendChild(nDiv);
+
+        METRICS[metrics[m]] = new CanvasJS.Chart('metrics_'+metrics[m], {
+            width: 250,
+            height: 250,
+            animationEnabled: false,
+            theme: "light2",
+            title:{
+                text: metrics[m],
+            },
+            axisY:{
+                includeZero: false
+            },
+            data: [{        
+                type: "line",       
+                dataPoints: [{y: 0}]
+            }]
+        });
+        METRICS[metrics[m]].render();
+    }
+}
+
+function setMetric(name, value){
+    let chart = METRICS[name];
+    let size = chart.options.data[0].dataPoints.length - 1;
+
+    if (chart.options.data[0].dataPoints.length > 500){
+        chart.options.data[0].dataPoints = chart.options.data[0].dataPoints.slice(1, size);
+        size = size - 1;
+    }
+    
+    size = chart.options.data[0].dataPoints.length - 1;
+    chart.options.data[0].dataPoints.push({y: value, x: chart.options.data[0].dataPoints[size].x+1});
+    chart.render();
 }

src/car.ts

@@ -11,6 +11,7 @@ import {
     CAR_IMG, Sprite, MAP, ROADSIZE, Container, Graphics
 } from "./global";
 import { RoadSprite } from "./asset_manager";
+import { runInThisContext } from "vm";
 
 var Global_carId = 0;
 
@@ -55,6 +56,10 @@ export interface CarSprite extends PIXI.Sprite {
     optionalTurn?: boolean;
     agent?: boolean;
     v?: number;
+    a?: number;
+    yaw_rate?: number;
+    last_a?: number;
+    last_yaw_rate?: number;
 }
 
 export interface LidarChild extends PIXI.Graphics {
@@ -62,6 +67,14 @@ export interface LidarChild extends PIXI.Graphics {
     pt?: boolean;
 }
 
+export interface State {
+    lidar?: number[][];
+    linear?: number[];
+    v?: number;
+    a?: number;
+    steering?: number;
+}
+
 export class Car {
 
     public level: Level|Editor;
@@ -173,6 +186,8 @@ export class Car {
             Method used to restore the position of the car
             to the original position (as set into the json file)
         */
+        this.core.a = 0;
+        this.core.v = 0;
         this.core.mx = this.info.mx;
         this.core.my = this.info.my;
         let road = this.level.getRoad(this.core.my, this.core.mx);
@@ -181,19 +196,26 @@ export class Car {
         }
     }
 
-    getState(linear:boolean = false): number[][]|number[]{
+    getState(): State {
         /*
             Get the current state of the car
             The state is the current value of each point
             of the lidar.
         */
-        if (!linear)
-            return this.motion.state.map(function(arr: any) { return arr.slice(); });
-        else{
-            let state: number[] = [];
-            this.motion.state.map((row: number[]) => { state = state.concat(row);});
-            return state;
-        }
+        let nState: State = {};
+
+        let linear: number[] = [];
+        this.motion.state.map((row: number[]) => { linear = linear.concat(row);});
+        linear.push(this.core.v);
+
+        nState.linear = linear;
+        nState.lidar =  this.motion.state.map(function(arr: any) { return arr.slice(); });
+
+        nState.v = this.core.v;
+        nState.a = this.core.last_a;
+        nState.steering = this.core.last_yaw_rate;
+
+        return nState;
     }
 
     step(delta: number, action:number|number[]=null){

src/control_motion_engine.ts

@@ -10,6 +10,7 @@ import * as U from "./utils";
 export class ControlMotionEngine extends MotionEngine {
 
     private actions: (string|number)[];
+    private maxSpeed: number = 1.5;
 
     constructor(level: Level) {
         /*
@@ -133,6 +134,9 @@ export class ControlMotionEngine extends MotionEngine {
             Step into the environement
             @delta (Float) time since the last update
         */
+       this.car.last_a = this.car.a;
+       this.car.last_yaw_rate = this.car.yaw_rate;
+
         // The car lose speed over time
         if (this.car.v > 0 && this.car.a == 0)
             this.car.v = Math.max(0, this.car.v - 0.01);
@@ -140,14 +144,14 @@ export class ControlMotionEngine extends MotionEngine {
             this.car.v = Math.min(0, this.car.v + 0.01);
 
         if (this.car.a > 0 && this.car.v >= 0)
-            this.car.v = Math.min(1.5, this.car.v + this.car.a*0.01);
+            this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01);
         else if (this.car.a > 0 && this.car.v < 0){
-            this.car.v = Math.min(1.5, this.car.v + this.car.a*0.03);
+            this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03);
         }
         if (this.car.a < 0 && this.car.v <= 0)
-            this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.01);
+            this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01);
         else if (this.car.a < 0 && this.car.v > 0){
-            this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.03);
+            this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03);
         }
         if (this.car.yaw_rate == 0){
             this.car.x += this.car.v * Math.cos(this.car.rotation)*delta;

src/level.ts

@@ -110,12 +110,7 @@ export class Level extends World {
     }
 
     setReward(agent_col: any, on_road: any, action: any){
-        /*
-            TODO: Let's the reward define in the agent class
-        */
-        let reward = -0.1;
-        if (action == 0 || this.agent.core.v == 1)
-            reward += 0.5;
+        let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed;
         if (agent_col.length > 0){
             reward = -1;
         }
@@ -133,12 +128,16 @@ export class Level extends World {
     }
 
 
-    step(delta: number, action:number|number[]=null){
+    step(delta: number, action:number|number[]=null, auto: boolean = true){
         /*
             Process one step into the environement
             @delta (Float) time since the last update
             @action: (Integer) The action to take (can be null if no action)
         */
+        if (auto && !this.steping){
+            return;
+        }
+
         // Go through all cars to move each one
         for (var c = 0; c < this.cars.length; c++) {
             if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move

src/metacar.ts

@@ -8,7 +8,7 @@ import {UIEvent} from "./ui_event";
 import * as U from "./utils";
 import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine";
 import { ControlMotionEngine } from "./control_motion_engine";
-import { LidarInfoI } from "./car";
+import { LidarInfoI, State } from "./car";
 
 /**
  * @local Chooce whether to load a file from the computer.
@@ -132,6 +132,14 @@ export class MetaCar {
        this.level.render(val);
     }
 
+    /**
+     * Choose wheter the environment should step automaticly
+     * @param val True or False 
+     */
+    public steping(val: boolean){
+        this.level.setSteping(val);
+    }
+
     /**
      * Usefull method to save/download a string as file.
      * @content The content of the file
@@ -157,8 +165,8 @@ export class MetaCar {
      * The size of the state depends of the size of the Lidar.
      * @return The state as a 2D Array or 1D Array (linear:true)
     */
-    public getState(linear:boolean = false): number[][]|number[]{
-        return this.level.agent.getState(linear);
+    public getState(): State{
+        return this.level.agent.getState();
     }
 
     /** 
@@ -167,7 +175,7 @@ export class MetaCar {
         @return Reward value
     */
     public step(action: number|number[]): number{
-        return this.level.step(1, action);
+        return this.level.step(1, action, false);
     }
 
     /**

src/ui_event.ts

@@ -61,6 +61,7 @@ export class UIEvent {
         // Listen the event
         button.addEventListener("click", () => {
             this.level.render(false);
+            this.level.setSteping(false);
             if (fc) fc();
         });
     }

src/world.ts

@@ -21,6 +21,7 @@ export class World {
     protected loop: any; // Loop method called for each render
     protected canvasId: string; // Id of the target canvas
     protected cars: Car[] = [];
+    protected steping: boolean = true;
 
     constructor(levelContent: LevelInfo, canvasId: string) {
         /*
@@ -124,9 +125,11 @@ export class World {
     render(val: boolean){
         if (val){
             this.app.ticker.start();
+            this.steping = true;
         }
         else{
             this.app.ticker.stop();
+            this.steping = false;
         }
     }
 
@@ -144,4 +147,10 @@ export class World {
         return this.app.renderer.plugins.interaction.mouse.global;        
     }
 
+    /**
+     * Stop stepping in the environment automaticly
+     */
+    public setSteping(val: boolean): void{
+        this.steping = val;
+    }
 }