diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index c6a3e51..553575b 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -11,7 +11,7 @@ class DDPGAgent { // Default Config this.config = { "stateSize": 17, - "nbActions": 1, + "nbActions": 2, "layerNorm": false, "normalizeObservations": true, "seed": 0, @@ -19,9 +19,9 @@ class DDPGAgent { "batchSize": 64, "actorLr": 0.0001, "criticLr": 0.001, - "memorySize": 15000, + "memorySize": 20000, "gamma": 0.99, - "noiseDecay": 0.95, + "noiseDecay": 0.99, "rewardScale": 1, "nbEpochs": 500, "nbEpochsCycle": 20, @@ -60,7 +60,7 @@ class DDPGAgent { // Pick an action const tfActions = this.ddpg.predict(tf.tensor2d([state])); const actions = tfActions.buffer().values; - agent.env.step([1., actions[0]]); + agent.env.step([actions[0], actions[1]]); tfActions.dispose(); } @@ -94,7 +94,7 @@ class DDPGAgent { // Step in the environment with theses actions let mAcions = tfActions.buffer().values; - let mReward = this.env.step([1., mAcions[0]]); + let mReward = this.env.step([mAcions[0], mAcions[1]]); this.rewardsList.push(mReward); // Get the new observations let mState = this.env.getState().linear; @@ -106,7 +106,7 @@ class DDPGAgent { } // Add the new tuple to the buffer - this.ddpg.memory.append(mPreviousStep, [mAcions[0]], mReward, mState, mDone); + this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); // Dispose tensor tfPreviousStep.dispose(); @@ -176,7 +176,7 @@ class DDPGAgent { } if (this.ddpg.memory.length == this.config.memorySize){ this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); - this.ddpg.noise.desiredActionStddev = Math.min(0.5, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); + this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); let lossValuesCritic = []; let lossValuesActor = []; console.time("Training"); diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 2810b85..6ceed76 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -22,7 +22,7 @@ env.loop(() => { displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); - const qValue = agent.getQvalue(state.linear, [state.steering]); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/DDPG/noise.js index 9c0ebb2..08c8978 100644 --- a/demo/webapp/public/js/DDPG/noise.js +++ b/demo/webapp/public/js/DDPG/noise.js @@ -16,8 +16,8 @@ class AdaptiveParamNoiseSpec { */ constructor(conf){ conf = conf || {}; - this.initialStddev = conf.initialStddev || 0.3; - this.desiredActionStddev = conf.desiredActionStddev || 0.3; + this.initialStddev = conf.initialStddev || 0.4; + this.desiredActionStddev = conf.desiredActionStddev || 0.4; this.adoptionCoefficient = conf.adoptionCoefficient || 1.01; this.currentStddev = this.initialStddev; } diff --git a/src/level.ts b/src/level.ts index 605640c..f7f12f2 100644 --- a/src/level.ts +++ b/src/level.ts @@ -110,8 +110,7 @@ export class Level extends World { } setReward(agent_col: any, on_road: any, action: any){ - let reward = 0; - //let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed; + let reward = 0 + Math.max(0., this.agent.core.v) / this.agent.motion.maxSpeed; if (agent_col.length > 0){ reward = -1; }