From 9342208dc77ec98f0bc607fe98764cab4efa7bf4 Mon Sep 17 00:00:00 2001 From: Thibault Neveu Date: Mon, 25 Jun 2018 09:38:11 +0100 Subject: [PATCH] Steering Angle: works --- demo/webapp/public/js/DDPG/ddpg.js | 188 ++++++++--- demo/webapp/public/js/DDPG/ddpg_agent.js | 110 +++++-- demo/webapp/public/js/DDPG/index.js | 43 ++- demo/webapp/public/js/DDPG/models.js | 135 +++----- demo/webapp/public/js/DDPG/noise.js | 4 +- demo/webapp/public/js/editor.js | 2 +- src/asset_manager.ts | 7 +- src/car.ts | 2 - src/control_motion_engine.ts | 6 +- src/embedded/level/level_2.ts | 403 +++++++++++++++++------ src/level.ts | 3 +- 11 files changed, 619 insertions(+), 284 deletions(-) diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 04e95f9..3655528 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -17,11 +17,10 @@ class DDPG { * @param memory (Memory class) * @param noise (Noise class) */ - constructor(actor, critic, memoryPos, memoryNeg, noise, config){ + constructor(actor, critic, memory, noise, config){ this.actor = actor; this.critic = critic; - this.memoryPos = memoryPos; - this.memoryNeg = memoryNeg; + this.memory = memory; this.noise = noise; this.config = config; this.tfGamma = tf.scalar(config.gamma); @@ -42,6 +41,7 @@ class DDPG { // Randomly Initialize critic network Q(s, a) this.critic.buildModel(obsInput, actionInput); + // Define in js/DDPG/models.js // Init target network Q' and μ' with the same weights this.actorTarget = copyModel(this.actor, Actor); @@ -50,35 +50,38 @@ class DDPG { this.perturbedActor = copyModel(this.actor, Actor); //this.adaptivePerturbedActor = copyModel(this.actor, Actor); + this.criticWithActor = (tfState) => { + return tf.tidy(() => { + const tfAct = this.actor.predict(tfState); + return this.critic.predict(tfState, tfAct); + }); + }; + + this.criticTargetWithActorTarget = (tfState) => { + return tf.tidy(() => { + const tfAct = this.actorTarget.predict(tfState); + return this.criticTarget.predict(tfState, tfAct); + }); + }; + + this.actorOptimiser = tf.train.adam(this.config.actorLr); this.criticOptimiser = tf.train.adam(this.config.criticLr); this.criticWeights = []; - for (let w = 0; w < this.critic.model.weights.length; w++){ - this.criticWeights.push(this.critic.model.weights[w].val); + for (let w = 0; w < this.critic.model.trainableWeights.length; w++){ + this.criticWeights.push(this.critic.model.trainableWeights[w].val); } this.actorWeights = []; - for (let w = 0; w < this.actor.model.weights.length; w++){ - this.actorWeights.push(this.actor.model.weights[w].val); + for (let w = 0; w < this.actor.model.trainableWeights.length; w++){ + this.actorWeights.push(this.actor.model.trainableWeights[w].val); } - // Return a batch from positive and negative experiences - this.memory = { - getBatch: (size) => { - let batch1 = this.memoryPos.getBatch(size/2); - let batch2 = this.memoryNeg.getBatch(size/2); - return { - 'obs0': batch1.obs0.concat(batch2.obs0), - 'obs1': batch1.obs1.concat(batch2.obs1), - 'rewards': batch1.rewards.concat(batch2.rewards), - 'actions': batch1.actions.concat(batch2.actions), - 'terminals': batch1.terminals.concat(batch2.terminals), - }; - } - }; + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); - assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + this.trainActorCt = 0; + this.trainCriticCt = 0; } @@ -102,17 +105,22 @@ class DDPG { */ adaptParamNoise(){ const batch = this.memory.getBatch(this.config.batchSize); - const tfObs0 = tf.tensor2d(batch.obs0); - const distance = this.distanceMeasure(tfObs0); + let distanceV = null; - assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev); + if (batch.obs0.length > 0){ + const tfObs0 = tf.tensor2d(batch.obs0); + const distance = this.distanceMeasure(tfObs0); + + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); + + distanceV = distance.buffer().values; + this.noise.adapt(distanceV[0]); - const distanceV = distance.buffer().values; - this.noise.adapt(distanceV[0]); - setMetric("Distance", distanceV[0]) + distance.dispose(); + tfObs0.dispose(); + } - distance.dispose(); - tfObs0.dispose(); + return distanceV; } /** @@ -176,32 +184,59 @@ class DDPG { const criticLoss = this.criticOptimiser.minimize(() => { const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); - - const tfAPredictions = this.actorTarget.model.predict(tfObs1); - const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]); + const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1); + + const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1)); - const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions)); - - const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean(); - return loss; + return tf.sub(tfQTargets, tfQPredictions0).square().mean(); }, true, this.criticWeights); - setMetric("CriticLoss", criticLoss.buffer().values[0]); + const loss = criticLoss.buffer().values[0]; criticLoss.dispose(); + + //if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){ + targetUpdate(this.criticTarget, this.critic, this.config); + //console.log("Update"); + // Saniity Check + //} + this.trainCriticCt += 1; + + return loss; } - trainActor(tfObs0, it){ - for (let i = 0; i < it; i++){ - const actorLoss = this.actorOptimiser.minimize(() => { - const tfAPredictions0 = this.actor.model.predict(tfObs0); - const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]); - const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1)); - return loss; - }, true, this.actorWeights); - const vLoss = actorLoss.buffer().values[0]; - setMetric("ActorLoss", actorLoss.buffer().values[0]); - actorLoss.dispose(); + trainActor(tfObs0){ + + const actorLoss = this.actorOptimiser.minimize(() => { + const tfQPredictions0 = this.criticWithActor(tfObs0); + return tf.mean(tfQPredictions0).mul(tf.scalar(-1.)) + }, true, this.actorWeights); + + /* + const sanityTfLoss = tf.tidy(() => { + const tfQPredictions0 = this.criticTargetWithActor(tfObs0); + const mn = tf.mean(tfQPredictions0); + const loss = mn.mul(tf.scalar(-1)); + return loss; + }); + + const sanityLoss = sanityTfLoss.buffer().values[0]; + + + if (sanityLoss == loss){ + console.warn("Sanity check failed. The optimisation have no effet here."); } + */ + + //if (this.trainActorCt % 200 == 0 && this.trainActorCt != 0){ + targetUpdate(this.actorTarget, this.actor, this.config); + //} + //this.trainActorCt += 1; + + const loss = actorLoss.buffer().values[0]; + actorLoss.dispose(); + //sanityTfLoss.dispose(); + + return loss; } getTfBatch(){ @@ -225,27 +260,74 @@ class DDPG { } } - async optimizeCritic(){ + optimizeCritic(){ const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); tfActions.dispose(); tfObs0.dispose(); tfObs1.dispose(); tfRewards.dispose(); tfTerminals.dispose(); + + return loss; } - async optimizeActor(it=1){ + optimizeActor(it=1){ const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - this.trainActor(tfObs0, it); + const loss = this.trainActor(tfObs0, it); tfActions.dispose(); tfObs0.dispose(); tfObs1.dispose(); tfRewards.dispose(); tfTerminals.dispose(); + + return loss; } + + optimizeCriticActor(){ + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const lossA = this.trainActor(tfObs0); + + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + + return {lossC, lossA}; + } + + trainRecord(){ + + let lossValues = []; + + for (let i=0; i < 32; i++){ + + const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + + const actorLoss = this.actorOptimiser.minimize(() => { + const tfAPredictions0 = this.actor.model.predict(tfObs0); + const loss = tfActions.sub(tfAPredictions0).square().mean(); + return loss; + }, true, this.actorWeights); + + const loss = actorLoss.buffer().values[0]; + lossValues.push(loss); + + actorLoss.dispose(); + tfActions.dispose(); + tfObs0.dispose(); + tfObs1.dispose(); + tfRewards.dispose(); + tfTerminals.dispose(); + } + console.log("Mean loss", mean(lossValues)); + } + } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index add5eaf..c6a3e51 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -10,21 +10,23 @@ class DDPGAgent { this.env = env; // Default Config this.config = { - "stateSize": 26, - "nbActions": 2, - "layerNom": true, + "stateSize": 17, + "nbActions": 1, + "layerNorm": false, "normalizeObservations": true, "seed": 0, "criticL2Reg": 0.01, - "batchSize": 32, + "batchSize": 64, "actorLr": 0.0001, "criticLr": 0.001, + "memorySize": 15000, "gamma": 0.99, + "noiseDecay": 0.95, "rewardScale": 1, "nbEpochs": 500, - "nbEpochsCycle": 100, + "nbEpochsCycle": 20, "nbTrainSteps": 50, - "tau": 0.001, + "tau": 0.01, "paramNoiseAdaptionInterval": 50, }; // From js/DDPG/noise.js @@ -34,8 +36,7 @@ class DDPGAgent { // Buffer replay // The baseline use 1e6 but this size should be enough for this problem - this.memoryPos = new Memory(5000); - this.memoryNeg = new Memory(5000); + this.memory = new Memory(this.config.memorySize); // Actor and Critic are from js/DDPG/models.js this.actor = new Actor(this.config); this.critic = new Critic(this.config); @@ -44,9 +45,10 @@ class DDPGAgent { Math.seedrandom(0); this.rewardsList = []; + this.epiDuration = []; // DDPG - this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config); + this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } /** @@ -54,11 +56,11 @@ class DDPGAgent { */ play(){ // Get the current state - const state = agent.env.getState().linear; + const state = this.env.getState().linear; // Pick an action - const tfActions = agent.ddpg.predict(tf.tensor2d([state])); + const tfActions = this.ddpg.predict(tf.tensor2d([state])); const actions = tfActions.buffer().values; - agent.env.step([actions[0], actions[1]]); + agent.env.step([1., actions[0]]); tfActions.dispose(); } @@ -85,48 +87,66 @@ class DDPGAgent { stepTrain(tfPreviousStep, mPreviousStep){ // Get actions const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + //const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep); + //const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed); + //const noisyActions = TruetfActions.add(rdNormal); + //const tfActions = tf.clipByValue(noisyActions, -1, 1); + // Step in the environment with theses actions let mAcions = tfActions.buffer().values; - let mReward = this.env.step([mAcions[0], mAcions[1]]); + let mReward = this.env.step([1., mAcions[0]]); this.rewardsList.push(mReward); // Get the new observations let mState = this.env.getState().linear; let tfState = tf.tensor2d([mState]); let mDone = 0; + if (mReward == -1){ mDone = 1; } // Add the new tuple to the buffer - if (mReward >= 0) - this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); - else - this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); + this.ddpg.memory.append(mPreviousStep, [mAcions[0]], mReward, mState, mDone); // Dispose tensor tfPreviousStep.dispose(); + //TruetfActions.dispose(); + //noisyActions.dispose(); tfActions.dispose(); + //rdNormal.dispose(); + //rd.dispose(); + //trueTfActions.dispose(); return {mDone, mState, tfState} } + initTrainParam(){ + this.stopTraining = false; + this.noisyActions = 2.0; + } + /** * Train DDPG Agent */ async train(realTime){ - this.stopTraining = false; + this.initTrainParam(); // One epoch for (let e=0; e < this.config.nbEpochs; e++){ // Perform cycles. + this.rewardsList = []; + this.stepList = []; + this.distanceList = []; for (let c=0; c < this.config.nbEpochsCycle; c++){ + if (c%10==0){ logTfMemory(); } - this.rewardsList = []; + // Perform rollouts. // Get current observation let mPreviousStep = this.env.getState().linear; let tfPreviousStep = tf.tensor2d([mPreviousStep]); let step = 0; + console.time("LoopTime"); for (step=0; step < 800; step++){ let rel = this.stepTrain(tfPreviousStep, mPreviousStep); @@ -139,30 +159,48 @@ class DDPGAgent { this.env.render(true); return; } - if (realTime) + if (realTime && step % 10 == 0) await tf.nextFrame(); } + this.stepList.push(step); console.timeEnd("LoopTime"); - this.env.reset(); + let distance = this.ddpg.adaptParamNoise(); + this.distanceList.push(distance[0]); + + this.env.randomRoadPosition(); tfPreviousStep.dispose(); - // Mean is define is js/utils.js console.log("e="+ e +", c="+c); - setMetric("Reward", mean(this.rewardsList)); - setMetric("EpisodeDuration", step); - this.ddpg.adaptParamNoise(); + + //this.ddpg.targetUpdate(); await tf.nextFrame(); } - console.time("LoopTrain"); - for (let t=0; t < 100; t++){ - this.ddpg.optimizeCritic(); - } - for (let t=0; t < 100; t++){ - this.ddpg.optimizeActor(); - } - console.timeEnd("LoopTrain"); - this.ddpg.targetUpdate(); + if (this.ddpg.memory.length == this.config.memorySize){ + this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); + this.ddpg.noise.desiredActionStddev = Math.min(0.5, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); + let lossValuesCritic = []; + let lossValuesActor = []; + console.time("Training"); + for (let t=0; t < 100; t++){ + let lossC = this.ddpg.optimizeCritic(); + lossValuesCritic.push(lossC); + } + for (let t=0; t < 100; t++){ + let lossA = this.ddpg.optimizeActor(); + lossValuesActor.push(lossA); + } + console.timeEnd("Training"); + console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev); + setMetric("CriticLoss", mean(lossValuesCritic)); + setMetric("ActorLoss", mean(lossValuesActor)); + } + setMetric("Reward", mean(this.rewardsList)); + setMetric("EpisodeDuration", mean(this.stepList)); + setMetric("Distance", mean(this.distanceList)); + await tf.nextFrame(); + } + + + this.env.render(true); } - this.env.render(true); - } }; \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 5ceb838..2810b85 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -3,8 +3,11 @@ let levelUrl = metacar.level.level2; var env = new metacar.env("canvas", levelUrl); env.setAgentMotion(metacar.motion.ControlMotion, {}); +env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1}) +RECORD = false; + // js/DDPG/ddpg.js var agent = new DDPGAgent(env); @@ -12,13 +15,21 @@ initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "Epi env.loop(() => { let state = env.getState(); + + if (RECORD){ + agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1); + } + displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + const qValue = agent.getQvalue(state.linear, [state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); + env.load().then(() => { + + // Train agent env.addEvent("train", () => { agent.train(false); @@ -28,6 +39,22 @@ env.load().then(() => { agent.play(); }); + env.addEvent("record", () => { + RECORD = true; + }); + + env.addEvent("randomPos", () => { + env.randomRoadPosition(); + }); + + env.addEvent("stopRecord", () => { + RECORD = false; + }); + + env.addEvent("trainOnRecord", () => { + agent.ddpg.trainRecord(); + }); + env.addEvent("TrainRealTime", () => { env.steping(false); agent.train(true); @@ -38,4 +65,18 @@ env.load().then(() => { }); env.addEvent("reset_env"); + + env.addEvent("load", (content) => { + content = JSON.parse(content); + + for (let c=0; c < content.obs0.length; c++){ + if (content.obs0[c] != 0){ + agent.ddpg.memory.append(content.obs0[c], content.actions[c], content.rewards[c], content.obs0[c], content.terminals[c]); + } + } + + console.log("dataReady"); + + }, {local: true}); }); + diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/DDPG/models.js index b47a55d..65c0edb 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/DDPG/models.js @@ -24,14 +24,14 @@ function copyModel(model, instance){ * @param stddev (number) * @return Copy of the model */ -function assignAndStd(actor, perturbedActor, stddev){ +function assignAndStd(actor, perturbedActor, stddev, seed){ return tf.tidy(() => { - const weights = actor.model.weights; + const weights = actor.model.trainableWeights; for (let m=0; m < weights.length; m++){ - let shape = perturbedActor.model.weights[m].val.shape; - let randomTensor = tf.randomNormal(shape, 0, stddev); + let shape = perturbedActor.model.trainableWeights[m].val.shape; + let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed); let nValue = weights[m].val.add(randomTensor); - perturbedActor.model.weights[m].val.assign(nValue); + perturbedActor.model.trainableWeights[m].val.assign(nValue); } }); } @@ -44,10 +44,10 @@ function assignAndStd(actor, perturbedActor, stddev){ */ function assignModel(model, targetModel){ return tf.tidy(() => { - const weights = model.model.weights; + const weights = model.model.trainableWeights; for (let m=0; m < weights.length; m++){ let nValue = weights[m].val; - targetModel.model.weights[m].val.assign(nValue); + targetModel.model.trainableWeights[m].val.assign(nValue); } }); } @@ -62,15 +62,20 @@ function assignModel(model, targetModel){ */ function targetUpdate(target, original, config){ return tf.tidy(() => { - const originalW = original.model.weights; - const targetW = target.model.weights; + const originalW = original.model.trainableWeights; + const targetW = target.model.trainableWeights; const one = tf.scalar(1); const tau = tf.scalar(config.tau); for (let m=0; m < originalW.length; m++){ + const lastValue = target.model.trainableWeights[m].val.clone(); let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau))); - target.model.weights[m].val.assign(nValue); + target.model.trainableWeights[m].val.assign(nValue); + const diff = lastValue.sub(target.model.trainableWeights[m].val).mean().buffer().values; + if (diff[0] == 0){ + console.warn("targetUpdate: Nothing have been changed!") + } } }); } @@ -97,45 +102,23 @@ class Actor{ buildModel(obs){ this.obs = obs; - - this.firstLayerBatchNorm = null; - this.secondLayerBatchNorm = null; - - this.relu1 = tf.layers.activation({activation: 'relu'}); - //this.relu2 = tf.layers.activation({activation: 'relu'}); - // First layer with BatchNormalization this.firstLayer = tf.layers.dense({ units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.firstLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } - - /* + // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ - units: 64, + units: 32, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.secondLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - }*/ // Ouput layer this.outputLayer = tf.layers.dense({ @@ -147,22 +130,15 @@ class Actor{ biasInitializer: "zeros" }); - this.predict = () => { - return tf.tidy(() => { + this.predict = (tfState) => { + return tf.tidy(() => { + if (tfState){ + obs = tfState; + } let l1 = this.firstLayer.apply(obs); - if (this.firstLayerBatchNorm){ - l1 = this.firstLayerBatchNorm.apply(l1); - } - l1 = this.relu1.apply(l1); - /* let l2 = this.secondLayer.apply(l1); - if (this.secondLayerBatchNorm){ - l2 = this.secondLayerBatchNorm.apply(l2); - } - l2 = this.relu2.apply(l2); - */ - - return this.outputLayer.apply(l1); + + return this.outputLayer.apply(l2); }); } const output = this.predict(); @@ -194,45 +170,35 @@ class Critic { this.obs = obs; this.action = action; - this.firstLayerBatchNorm = null; - this.secondLayerBatchNorm = null; - - this.relu1 = tf.layers.activation({activation: 'relu'}); - this.relu2 = tf.layers.activation({activation: 'relu'}); - this.concat = tf.layers.concatenate(); + this.add = tf.layers.add(); // First layer with BatchNormalization - this.firstLayer = tf.layers.dense({ + this.firstLayerS = tf.layers.dense({ + units: 64, + kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), + activation: 'linear', // relu is add later + useBias: true, + biasInitializer: "zeros" + }); + + // First layer with BatchNormalization + this.firstLayerA = tf.layers.dense({ units: 64, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.firstLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } // Second layer with BatchNormalization this.secondLayer = tf.layers.dense({ //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action - units: 64, + units: 32, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'linear', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - if (this.layerNorm){ - // WARNING: BatchNormalization instead of layerNormalization - this.secondLayerBatchNorm = tf.layers.batchNormalization({ - scale: true, - center: true - }); - } // Ouput layer this.outputLayer = tf.layers.dense({ @@ -245,22 +211,21 @@ class Critic { }); // Actor prediction - this.predict = () => { + this.predict = (tfState, tfActions) => { return tf.tidy(() => { - let l1 = this.firstLayer.apply(obs); - l1 = this.concat.apply([l1, action]); - if (this.firstLayerBatchNorm){ - l1 = this.firstLayerBatchNorm.apply(l1); + if (tfState && tfActions){ + obs = tfState; + action = tfActions; } - l1 = this.relu1.apply(l1); + + let l1A = this.firstLayerA.apply(action); + let l1S = this.firstLayerS.apply(obs) - let l2 = this.secondLayer.apply(l1); - if (this.secondLayerBatchNorm){ - l2 = this.secondLayerBatchNorm.apply(l2); - } - l2 = this.relu2.apply(l2); + let concat = this.add.apply([l1A, l1S]) - return this.outputLayer.apply(l2); + let l2 = this.secondLayer.apply(concat); + + return this.outputLayer.apply(l2); }); } const output = this.predict(); diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index c3a138f..9c0ebb2 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -16,8 +16,8 @@ class AdaptiveParamNoiseSpec { */ constructor(conf){ conf = conf || {}; - this.initialStddev = conf.initialStddev || 0.1; - this.desiredActionStddev = conf.initialStddev || 0.1; + this.initialStddev = conf.initialStddev || 0.3; + this.desiredActionStddev = conf.desiredActionStddev || 0.3; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; this.currentStddev = this.initialStddev; } diff --git a/demo/webapp/public/js/editor.js b/demo/webapp/public/js/editor.js index 97b0e54..3a569b3 100644 --- a/demo/webapp/public/js/editor.js +++ b/demo/webapp/public/js/editor.js @@ -16,5 +16,5 @@ editor.load().then(() => { localStorage.setItem('mylevel.json', JSON.stringify(content)); window.open("/test_editor.html"); - }, {download: false, name: "level.json"}); + }, {download: true, name: "level.json"}); }); \ No newline at end of file diff --git a/src/asset_manager.ts b/src/asset_manager.ts index 8a3c97c..c93474d 100644 --- a/src/asset_manager.ts +++ b/src/asset_manager.ts @@ -140,6 +140,9 @@ export class AssetManger { } car.line = line; + car.a = 0; + car.v = 0; + // Set the car on the road car.x = road.x; car.y = road.y; @@ -158,7 +161,6 @@ export class AssetManger { let line_side_factor_t = line == 0 ? 0:Math.PI; car.x += line_side_factor*Math.round(x_m); car.y += line_side_factor*Math.round(y_m); - // (x, y position) Relatif to the map car.mx = Math.floor(car.x / ROADSIZE); car.my = Math.floor(car.y / ROADSIZE); @@ -166,6 +168,9 @@ export class AssetManger { car.rotation = (th+line_side_factor_t); // Set the new road of the car car.checkAndsetNewRoad(road); + + + } createAsset(img: string, info: AssetInfo, textures: any, type: string){ diff --git a/src/car.ts b/src/car.ts index 32be0e6..a4a6309 100644 --- a/src/car.ts +++ b/src/car.ts @@ -186,8 +186,6 @@ export class Car { Method used to restore the position of the car to the original position (as set into the json file) */ - this.core.a = 0; - this.core.v = 0; this.core.mx = this.info.mx; this.core.my = this.info.my; let road = this.level.getRoad(this.core.my, this.core.mx); diff --git a/src/control_motion_engine.ts b/src/control_motion_engine.ts index bb68b3b..437269c 100644 --- a/src/control_motion_engine.ts +++ b/src/control_motion_engine.ts @@ -10,7 +10,7 @@ import * as U from "./utils"; export class ControlMotionEngine extends MotionEngine { private actions: (string|number)[]; - private maxSpeed: number = 1.5; + private maxSpeed: number = 2.0; constructor(level: Level) { /* @@ -144,12 +144,12 @@ export class ControlMotionEngine extends MotionEngine { this.car.v = Math.min(0, this.car.v + 0.01); if (this.car.a > 0 && this.car.v >= 0) - this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01); + this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.005); else if (this.car.a > 0 && this.car.v < 0){ this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03); } if (this.car.a < 0 && this.car.v <= 0) - this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01); + this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.005); else if (this.car.a < 0 && this.car.v > 0){ this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03); } diff --git a/src/embedded/level/level_2.ts b/src/embedded/level/level_2.ts index 49a32f8..88a98ea 100644 --- a/src/embedded/level/level_2.ts +++ b/src/embedded/level/level_2.ts @@ -1,114 +1,319 @@ export const level2: any = { - "cars": [ + "cars": [], + "road": [ { - "mx": 7, - "my": 7, - "line": 0 + "x": 300, + "y": 360 }, { - "mx": 9, - "my": 6, - "line": 0 + "x": 300, + "y": 360 }, { - "mx": 8, - "my": 6, - "line": 1 + "x": 360, + "y": 360 }, { - "mx": 10, - "my": 6, - "line": 1 + "x": 420, + "y": 360 + }, + { + "x": 240, + "y": 360 + }, + { + "x": 180, + "y": 360 + }, + { + "x": 120, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 120, + "y": 360 + }, + { + "x": 60, + "y": 360 + }, + { + "x": 300, + "y": 300 + }, + { + "x": 300, + "y": 240 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 480, + "y": 360 + }, + { + "x": 540, + "y": 360 + }, + { + "x": 300, + "y": 180 + }, + { + "x": 360, + "y": 180 + }, + { + "x": 240, + "y": 180 + }, + { + "x": 240, + "y": 180 + }, + { + "x": 180, + "y": 180 + }, + { + "x": 420, + "y": 180 + }, + { + "x": 420, + "y": 120 + }, + { + "x": 180, + "y": 120 + }, + { + "x": 300, + "y": 60 + }, + { + "x": 420, + "y": 60 + }, + { + "x": 180, + "y": 60 + }, + { + "x": 240, + "y": 60 + }, + { + "x": 360, + "y": 60 + }, + { + "x": 480, + "y": 60 + }, + { + "x": 120, + "y": 60 + }, + { + "x": 540, + "y": 60 + }, + { + "x": 540, + "y": 120 + }, + { + "x": 540, + "y": 180 + }, + { + "x": 540, + "y": 240 + }, + { + "x": 540, + "y": 300 + }, + { + "x": 60, + "y": 60 + }, + { + "x": 60, + "y": 120 + }, + { + "x": 60, + "y": 180 + }, + { + "x": 60, + "y": 240 + }, + { + "x": 60, + "y": 300 + }, + { + "x": 60, + "y": 240 + }, + { + "x": 360, + "y": 180 + }, + { + "x": 480, + "y": 360 + } + ], + "asset": [ + { + "x": 162, + "y": 264 + }, + { + "x": 407, + "y": 264 + }, + { + "x": 259, + "y": 126 + }, + { + "x": 306, + "y": 33 + }, + { + "x": 491, + "y": 148 + }, + { + "x": 391, + "y": 431 + }, + { + "x": 608, + "y": 35 + }, + { + "x": 484, + "y": 224 + }, + { + "x": 248, + "y": 297 + }, + { + "x": 376, + "y": 336 + }, + { + "x": 78, + "y": 429 + }, + { + "x": 1, + "y": 205 + }, + { + "x": 3, + "y": 106 + }, + { + "x": -46, + "y": 290 + }, + { + "x": 622, + "y": 154 + }, + { + "x": 617, + "y": 280 } ], "house": [ { - "x": 68, - "y": 317 - } - ], - "house2": [ - { - "x": 624, - "y": 84 + "x": 162, + "y": 264 }, { - "x": -15, - "y": 144 + "x": 622, + "y": 154 } ], "house3": [ { - "x": 618, - "y": 231 + "x": 407, + "y": 264 }, { - "x": 13, - "y": 441 - } - ], - "bench": [ - { - "x": 318, - "y": 216 - }, - { - "x": 422, - "y": 36 + "x": -46, + "y": 290 } ], "tree": [ { - "x": 151, - "y": 144 + "x": 259, + "y": 126 }, { - "x": 229, - "y": 175 + "x": 491, + "y": 148 }, { - "x": 326, - "y": 132 + "x": 391, + "y": 431 }, { - "x": 413, - "y": 167 + "x": 608, + "y": 35 }, { - "x": 487, - "y": 157 + "x": 484, + "y": 224 }, { - "x": 445, - "y": 123 + "x": 248, + "y": 297 }, { - "x": 511, + "x": 78, "y": 429 }, { - "x": 584, - "y": 427 + "x": 1, + "y": 205 }, { - "x": 80, - "y": 391 + "x": 3, + "y": 106 }, { - "x": 219, - "y": 409 + "x": 617, + "y": 280 + } + ], + "bench": [ + { + "x": 306, + "y": 33 }, { - "x": 207, - "y": 326 - }, - { - "x": 156, - "y": 426 - }, - { - "x": 162, - "y": 13 + "x": 376, + "y": 336 } ], "map": [ @@ -129,11 +334,11 @@ export const level2: any = { 0, "↱", "↔", + "↧", "↔", "↔", "↔", - "↔", - "↔", + "↧", "↔", "↰", 0 @@ -142,9 +347,35 @@ export const level2: any = { 0, "↕", 0, + "↕", 0, 0, 0, + "↕", + 0, + "↕", + 0 + ], + [ + 0, + "↕", + 0, + "↳", + "↔", + "↧", + "↠", + "↲", + 0, + "↕", + 0 + ], + [ + 0, + "↟", + 0, + 0, + 0, + "↕", 0, 0, 0, @@ -157,7 +388,7 @@ export const level2: any = { 0, 0, 0, - 0, + "↕", 0, 0, 0, @@ -170,10 +401,10 @@ export const level2: any = { "↔", "↔", "↔", - "↧", - "↔", + "↥", "↔", "↔", + "↠", "↲", 0 ], @@ -183,43 +414,17 @@ export const level2: any = { 0, 0, 0, - "↕", 0, 0, 0, 0, - 0 - ], - [ - 0, - 0, - 0, - 0, - 0, - "↕", - 0, - "↱", - "↔", - "↔", - "↔" - ], - [ - 0, - 0, - 0, - 0, - 0, - "↕", - 0, - "↕", - 0, 0, 0 ] ], "agent": { - "mx": 2, - "my": 4, + "mx": 4, + "my": 3, "line": 0, "motion": { "type": "BasicMotionEngine", diff --git a/src/level.ts b/src/level.ts index a181c8c..605640c 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,7 +110,8 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; + let reward = 0; + //let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; }