diff --git a/demo/webapp/public/img/ddpg.png b/demo/webapp/public/img/ddpg.png new file mode 100644 index 0000000..130f3c9 Binary files /dev/null and b/demo/webapp/public/img/ddpg.png differ diff --git a/demo/webapp/public/js/DDPG/memory.js b/demo/webapp/public/js/DDPG/memory.js deleted file mode 100644 index fc8db0a..0000000 --- a/demo/webapp/public/js/DDPG/memory.js +++ /dev/null @@ -1,83 +0,0 @@ - -class Memory { - - /** - * @param maxlen (number) Buffer limit - */ - constructor(maxlen){ - this.maxlen = maxlen; - this.length = 0; - this.start = 0; - - this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); - this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); - this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); - this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); - this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0); - } - - /** - * @param idx (number) - */ - getItem(idx){ - if (idx < 0 || idx >= this.length){ - console.error("Memory.getItem: idx not in range."); - } - return this.data[(this.start + idx) % this.maxlen] - } - - /** - * Sample a batch - * @param batchSize (number) - * @return batch [] - */ - getBatch(batchSize){ - const arrLength = this.length; - const batch = { - 'obs0': [], - 'obs1': [], - 'rewards': [], - 'actions': [], - 'terminals': [], - }; - if (batchSize > this.length){ - return batch; - } - for (let b=0; b < batchSize; b++){ - let id = Math.floor(Math.random() * arrLength); - batch.obs0.push(this.obs0List[id]); - batch.obs1.push(this.obs1List[id]); - batch.rewards.push(this.rewardsList[id]); - batch.actions.push(this.actionsList[id]); - batch.terminals.push(this.terminals1List[id]); - } - return batch - } - - /** - * @param obs0 [] - * @param action (number) - * @param reward (number) - * @param obs1 [] - * @param terminal1 (boolean) - */ - append(obs0, action, reward, obs1, terminal1){ - if (this.length < this.maxlen){ - this.length += 1; - } - else if (this.length == this.maxlen) { - //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose(); - //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose(); - //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose(); - this.start = (this.start + 1) % this.maxlen; - } - else { - console.error("Memory.append: This should never be printed"); - } - this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0; - this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1; - this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward; - this.actionsList[(this.start + this.length - 1) % this.maxlen] = action; - this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1; - } -} \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/ddpg/ddpg.js similarity index 74% rename from demo/webapp/public/js/DDPG/ddpg.js rename to demo/webapp/public/js/ddpg/ddpg.js index 63fd1cc..c55c0a2 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/ddpg/ddpg.js @@ -29,13 +29,6 @@ class DDPG { let obsInput = tf.input({batchShape: [null, this.config.stateSize]}); let actionInput = tf.input({batchShape: [null, this.config.nbActions]}); - if (config.normalizeObservations){ - tf.layers.batchNormalization({ - scale: true, - center: true - }).apply(obsInput); - } - // Randomly Initialize actor network μ(s) this.actor.buildModel(obsInput); // Randomly Initialize critic network Q(s, a) @@ -56,7 +49,6 @@ class DDPG { return this.critic.predict(tfState, tfAct); }); }; - this.criticTargetWithActorTarget = (tfState) => { return tf.tidy(() => { const tfAct = this.actorTarget.predict(tfState); @@ -64,7 +56,6 @@ class DDPG { }); }; - this.actorOptimiser = tf.train.adam(this.config.actorLr); this.criticOptimiser = tf.train.adam(this.config.criticLr); @@ -72,23 +63,19 @@ class DDPG { for (let w = 0; w < this.critic.model.trainableWeights.length; w++){ this.criticWeights.push(this.critic.model.trainableWeights[w].val); } - this.actorWeights = []; for (let w = 0; w < this.actor.model.trainableWeights.length; w++){ this.actorWeights.push(this.actor.model.trainableWeights[w].val); } assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); - - this.trainActorCt = 0; - this.trainCriticCt = 0; } /** * Distance Measure for DDPG * See parameter space noise Exploration paper - * obs (Tensor2d) Observations + * @param observations (Tensor2d) Observations */ distanceMeasure(observations) { return tf.tidy(() => { @@ -105,6 +92,11 @@ class DDPG { */ adaptParamNoise(){ const batch = this.memory.getBatch(this.config.batchSize); + if (batch.obs0.length == 0){ + assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed); + return [0]; + } + let distanceV = null; if (batch.obs0.length > 0){ @@ -123,17 +115,6 @@ class DDPG { return distanceV; } - /** - * Eval the actions and the Q function - * @param states (tf.tensor2d) - * @return actions and qValues - */ - eval(observation){ - const tfActions = this.perturbedActor.model.predict(observation); - const tfQValues = this.critic.model.predict([observation, tfActions]); - return {tfActions, tfQValues}; - } - /** * Get the estimation of the Q value given the state * and the action @@ -174,13 +155,13 @@ class DDPG { */ targetUpdate(){ // Define in js/DDPG/models.js - //assignModel(this.critic, this.criticTarget); - //assignModel(this.actor, this.actorTarget); targetUpdate(this.criticTarget, this.critic, this.config); targetUpdate(this.actorTarget, this.actor, this.config); } - trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){ + + let costs; const criticLoss = this.criticOptimiser.minimize(() => { const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]); @@ -188,18 +169,18 @@ class DDPG { const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1)); - return tf.sub(tfQTargets, tfQPredictions0).square().mean(); + const erros = tf.sub(tfQTargets, tfQPredictions0).square(); + costs = erros.buffer().values; + + return erros.mean(); }, true, this.criticWeights); + this.memory.appendBackWithCost(batch, costs); + const loss = criticLoss.buffer().values[0]; criticLoss.dispose(); - //if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){ targetUpdate(this.criticTarget, this.critic, this.config); - //console.log("Update"); - // Saniity Check - //} - this.trainCriticCt += 1; return loss; } @@ -215,14 +196,13 @@ class DDPG { const loss = actorLoss.buffer().values[0]; actorLoss.dispose(); - //sanityTfLoss.dispose(); return loss; } getTfBatch(){ // Get batch - const batch = this.memory.getBatch(this.config.batchSize); + const batch = this.memory.popBatch(this.config.batchSize); // Convert to tensors const tfActions = tf.tensor2d(batch.actions); const tfObs0 = tf.tensor2d(batch.obs0); @@ -237,12 +217,12 @@ class DDPG { _tfTerminals.dispose(); return { - tfActions, tfObs0, tfObs1, tfRewards, tfTerminals + batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals } } optimizeCritic(){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); @@ -255,10 +235,10 @@ class DDPG { return loss; } - optimizeActor(it=1){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + optimizeActor(){ + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - const loss = this.trainActor(tfObs0, it); + const loss = this.trainActor(tfObs0); tfActions.dispose(); tfObs0.dispose(); @@ -270,9 +250,9 @@ class DDPG { } optimizeCriticActor(){ - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); + const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); + const lossC = this.trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals); const lossA = this.trainActor(tfObs0); tfActions.dispose(); @@ -283,32 +263,4 @@ class DDPG { return {lossC, lossA}; } - - trainRecord(){ - - let lossValues = []; - - for (let i=0; i < 32; i++){ - - const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch(); - - const actorLoss = this.actorOptimiser.minimize(() => { - const tfAPredictions0 = this.actor.model.predict(tfObs0); - const loss = tfActions.sub(tfAPredictions0).square().mean(); - return loss; - }, true, this.actorWeights); - - const loss = actorLoss.buffer().values[0]; - lossValues.push(loss); - - actorLoss.dispose(); - tfActions.dispose(); - tfObs0.dispose(); - tfObs1.dispose(); - tfRewards.dispose(); - tfTerminals.dispose(); - } - console.log("Mean loss", mean(lossValues)); - } - } \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/ddpg/ddpg_agent.js similarity index 66% rename from demo/webapp/public/js/DDPG/ddpg_agent.js rename to demo/webapp/public/js/ddpg/ddpg_agent.js index 2a6050c..0906222 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/ddpg/ddpg_agent.js @@ -1,36 +1,45 @@ - // This class is called from js/DDPG/index.js class DDPGAgent { /** * @param env (metacar.env) Set in js/DDPG/index.js */ - constructor(env){ + constructor(env, config){ + this.stopTraining = false; this.env = env; + + config = config || {}; // Default Config this.config = { - "stateSize": 17, - "nbActions": 2, - "layerNorm": false, - "normalizeObservations": true, - "seed": 0, - "criticL2Reg": 0.01, - "batchSize": 64, - "actorLr": 0.0001, - "criticLr": 0.001, - "memorySize": 20000, - "gamma": 0.99, - "noiseDecay": 0.99, - "rewardScale": 1, - "nbEpochs": 500, - "nbEpochsCycle": 20, - "nbTrainSteps": 50, - "tau": 0.01, - "paramNoiseAdaptionInterval": 50, + "stateSize": config.stateSize || 17, + "nbActions": config.nbActions || 2, + "seed": config.seed || 0, + "batchSize": config.batchSize || 128, + "actorLr": config.actorLr || 0.0001, + "criticLr": config .criticLr || 0.001, + "memorySize": config.memorySize || 30000, + "gamma": config.gamme || 0.99, + "noiseDecay": config.noiseDecay || 0.99, + "rewardScale": config.rewardScale || 1, + "nbEpochs": config.nbEpochs || 200, + "nbEpochsCycle": config.nbEpochsCycle || 10, + "nbTrainSteps": config.nbTrainSteps || 110, + "tau": config.tau || 0.01, + "initialStddev": config.initialStddev || 0.1, + "desiredActionStddev": config.desiredActionStddev || 0.1, + "adoptionCoefficient": config.adoptionCoefficient || 1.01, + "actorFirstLayerSize": config.actorFirstLayerSize || 64, + "actorSecondLayerSize": config.actorSecondLayerSize || 32, + "criticFirstLayerSSize": config.criticFirstLayerSSize || 64, + "criticFirstLayerASize": config.criticFirstLayerASize || 64, + "criticSecondLayerSize": config.criticSecondLayerSize || 32, + "maxStep": config.maxStep || 800, + "stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError:true, + "resetEpisode": config.resetEpisode != undefined ? config.resetEpisode:false }; // From js/DDPG/noise.js - this.noise = new AdaptiveParamNoiseSpec(); + this.noise = new AdaptiveParamNoiseSpec(this.config); // Configure components. @@ -51,20 +60,20 @@ class DDPGAgent { this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } - save(env){ + save(name){ /* Save the network */ - this.ddpg.critic.model.save('downloads://critic-model-ddpg-agent'); - this.ddpg.actor.model.save('downloads://actor-model-ddpg-agent'); + this.ddpg.critic.model.save('downloads://critic-' + name); + this.ddpg.actor.model.save('downloads://actor-'+ name); } - async restore(){ + async restore(folder, name){ /* Restore the weights of the network */ - this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/ddpg/critic-model-ddpg-agent.json'); - this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/ddpg/actor-model-ddpg-agent.json"); + this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/'+folder+'/critic-'+name+'.json'); + this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/"+folder+"/actor-"+name+".json"); } /** @@ -103,11 +112,6 @@ class DDPGAgent { stepTrain(tfPreviousStep, mPreviousStep){ // Get actions const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep); - //const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep); - //const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed); - //const noisyActions = TruetfActions.add(rdNormal); - //const tfActions = tf.clipByValue(noisyActions, -1, 1); - // Step in the environment with theses actions let mAcions = tfActions.buffer().values; let mReward = this.env.step([mAcions[0], mAcions[1]]); @@ -116,43 +120,31 @@ class DDPGAgent { let mState = this.env.getState().linear; let tfState = tf.tensor2d([mState]); let mDone = 0; - - if (mReward == -1){ + if (mReward == -1 && this.config.stopOnRewardError){ mDone = 1; } - // Add the new tuple to the buffer this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone); - - // Dispose tensor + // Dispose tensors tfPreviousStep.dispose(); - //TruetfActions.dispose(); - //noisyActions.dispose(); tfActions.dispose(); - //rdNormal.dispose(); - //rd.dispose(); - //trueTfActions.dispose(); - return {mDone, mState, tfState} - } - initTrainParam(){ - this.stopTraining = false; - this.noisyActions = 2.0; + return {mDone, mState, tfState}; } /** * Train DDPG Agent */ async train(realTime){ - this.initTrainParam(); + this.stopTraining = false; // One epoch for (let e=0; e < this.config.nbEpochs; e++){ // Perform cycles. this.rewardsList = []; this.stepList = []; this.distanceList = []; + document.getElementById("trainingProgress").innerHTML = "Progression: "+e+"/"+this.config.nbEpochs+"
"; for (let c=0; c < this.config.nbEpochsCycle; c++){ - if (c%10==0){ logTfMemory(); } @@ -164,11 +156,11 @@ class DDPGAgent { let step = 0; console.time("LoopTime"); - for (step=0; step < 800; step++){ + for (step=0; step < this.config.maxStep; step++){ let rel = this.stepTrain(tfPreviousStep, mPreviousStep); mPreviousStep = rel.mState; tfPreviousStep = rel.tfState; - if (rel.mDone){ + if (rel.mDone && this.config.stopOnRewardError){ break; } if (this.stopTraining){ @@ -183,6 +175,9 @@ class DDPGAgent { let distance = this.ddpg.adaptParamNoise(); this.distanceList.push(distance[0]); + if (this.config.resetEpisode){ + this.env.reset(); + } this.env.randomRoadPosition(); tfPreviousStep.dispose(); console.log("e="+ e +", c="+c); @@ -190,28 +185,25 @@ class DDPGAgent { //this.ddpg.targetUpdate(); await tf.nextFrame(); } - if (this.ddpg.memory.length == this.config.memorySize){ - this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay); + if (e > 5){ this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev); let lossValuesCritic = []; let lossValuesActor = []; - console.time("Training"); - for (let t=0; t < 100; t++){ - let lossC = this.ddpg.optimizeCritic(); + console.time("Training"); + for (let t=0; t < this.config.nbTrainSteps; t++){ + let {lossC, lossA} = this.ddpg.optimizeCriticActor(); lossValuesCritic.push(lossC); - } - for (let t=0; t < 100; t++){ - let lossA = this.ddpg.optimizeActor(); lossValuesActor.push(lossA); } console.timeEnd("Training"); console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev); setMetric("CriticLoss", mean(lossValuesCritic)); setMetric("ActorLoss", mean(lossValuesActor)); - } + } + setMetric("Reward", mean(this.rewardsList)); setMetric("EpisodeDuration", mean(this.stepList)); - setMetric("Distance", mean(this.distanceList)); + setMetric("NoiseDistance", mean(this.distanceList)); await tf.nextFrame(); } diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/ddpg/index.js similarity index 56% rename from demo/webapp/public/js/DDPG/index.js rename to demo/webapp/public/js/ddpg/index.js index 2eacb40..07e5927 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/ddpg/index.js @@ -2,60 +2,49 @@ let levelUrl = metacar.level.level2; var env = new metacar.env("canvas", levelUrl); + env.setAgentMotion(metacar.motion.ControlMotion, {}); -env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1}) - - -RECORD = false; +env.setAgentLidar({pts: 5, width: 3, height: 7, pos: -0.5}) // js/DDPG/ddpg.js -var agent = new DDPGAgent(env); +var agent = new DDPGAgent(env, { + stateSize: 26, + resetEpisode: true +}); -initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]); +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]); env.loop(() => { let state = env.getState(); - - if (RECORD){ - agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1); - } - displayState("realtime_viewer", state.lidar, 200, 200); let reward = env.getLastReward(); const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); }); - env.load().then(() => { - - // Train agent - env.addEvent("train", () => { - agent.train(false); + env.addEvent("train [Background]", () => { + let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model."); + if (train){ + env.render(false); + agent.train(false); + } }); - env.addEvent("play", () => { - agent.play(); - }); - - env.addEvent("record", () => { - RECORD = true; - }); - - env.addEvent("randomPos", () => { - env.randomRoadPosition(); - }); - - env.addEvent("stopRecord", () => { - RECORD = false; - }); - - env.addEvent("TrainRealTime", () => { + env.addEvent("Train [Show the training]", () => { env.steping(false); agent.train(true); }); + env.addEvent("shuffle", () => { + env.randomRoadPosition(); + }) + + env.addEvent("play", () => { + agent.play(); + }); + env.addEvent("stop", () => { agent.stop(); }); @@ -63,11 +52,11 @@ env.load().then(() => { env.addEvent("reset_env"); env.addEvent("save", () => { - agent.save(); + agent.save("model-ddpg-road"); }); env.addEvent("load", () => { - agent.restore() + agent.restore("ddpg-road", "model-ddpg-road") }); }); diff --git a/demo/webapp/public/js/ddpg/memory.js b/demo/webapp/public/js/ddpg/memory.js new file mode 100644 index 0000000..b4921fb --- /dev/null +++ b/demo/webapp/public/js/ddpg/memory.js @@ -0,0 +1,240 @@ + +class Memory { + + /** + * @param maxlen (number) Buffer limit + */ + constructor(maxlen){ + this.maxlen = maxlen; + this.buffer = []; + this.priorBuffer = []; + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + getBatch(batchSize){ + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + + if (batchSize > this.priorBuffer.length){ + console.warn("The size of the replay buffer is < to the batchSize. Return empty batch."); + return batch; + } + + for (let b=0; b < batchSize/2; b++){ + let id = Math.floor(Math.random() * this.priorBuffer.length); + batch.obs0.push(this.priorBuffer[id].obs0); + batch.obs1.push(this.priorBuffer[id].obs1); + batch.rewards.push(this.priorBuffer[id].reward); + batch.actions.push(this.priorBuffer[id].action); + batch.terminals.push(this.priorBuffer[id].terminal); + } + return batch + } + + _bufferBatch(batchSize){ + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + + for (let b=0; b < batchSize/2; b++){ + let nElem = this.buffer.pop(); + batch.obs0.push(nElem.obs0); + batch.obs1.push(nElem.obs1); + batch.rewards.push(nElem.reward); + batch.actions.push(nElem.action); + batch.terminals.push(nElem.terminal); + } + + for (let b=0; b < batchSize/2; b++){ + let id = Math.floor(Math.random() * this.buffer.length); + batch.obs0.push(this.buffer[id].obs0); + batch.obs1.push(this.buffer[id].obs1); + batch.rewards.push(this.buffer[id].reward); + batch.actions.push(this.buffer[id].action); + batch.terminals.push(this.buffer[id].terminal); + this.buffer.splice(id, 1); + } + + return batch + } + + _addRandomBufferBatch(batchSize, batch){ + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * this.buffer.length); + batch.obs0.push(this.buffer[id].obs0); + batch.obs1.push(this.buffer[id].obs1); + batch.rewards.push(this.buffer[id].reward); + batch.actions.push(this.buffer[id].action); + batch.terminals.push(this.buffer[id].terminal); + this.buffer.splice(id, 1); + } + return batch + } + + /** + * Sample a batch + * @param batchSize (number) + * @return batch [] + */ + popBatch(batchSize){ + let originalBatchSize = batchSize; + let priorBufferBatchSize; + let bufferBatchSize; + if (batchSize % 2 != 0){ + console.warn("Batch size should be a even.") + } + if (this.priorBuffer.length < batchSize/2){ + //console.log("get full batch from buffer"); + const batch = this._bufferBatch(batchSize); + console.assert(batch.obs0.length == batchSize); + return batch; + } + const batch = { + 'obs0': [], + 'obs1': [], + 'rewards': [], + 'actions': [], + 'terminals': [], + }; + if (batchSize > this.length){ + console.warn("The size of the replay buffer is < to the batchSize. Return empty batch."); + return batch; + } + + if (this.buffer.length > 0){ + //console.log("Get half of prior and other from buffer."); + batchSize = batchSize / 2; + } + else{ + //console.log("Get all from priorBuffer"); + } + + for (let b=0; b < batchSize; b++){ + let id = Math.floor(Math.random() * this.priorBuffer.length); + batch.obs0.push(this.priorBuffer[id].obs0); + batch.obs1.push(this.priorBuffer[id].obs1); + batch.rewards.push(this.priorBuffer[id].reward); + batch.actions.push(this.priorBuffer[id].action); + batch.terminals.push(this.priorBuffer[id].terminal); + this.priorBuffer.splice(id, 1); + } + + if (this.buffer.length > 0){ + this._addRandomBufferBatch(batchSize, batch); + } + console.assert(batch.obs0.length == originalBatchSize); + return batch + } + + _insert(element, array) { + if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){ + array.unshift(element); + return array; + } + array.splice(this._locationOf(element, array) + 1, 0, element); + return array; + } + + _locationOf(element, array, start, end) { + start = start || 0; + end = end || array.length; + + var pivot = parseInt(start + (end - start) / 2, 10); + + if (end-start <= 1 || array[pivot] === element) return pivot; + + if (array[pivot].cost != null && array[pivot].cost < element.cost) { + return this._locationOf(element, array, pivot, end); + } else { + return this._locationOf(element, array, start, pivot); + } + } + + /** + * @param batch (Object) from getBatch() + * @param cost (number) Cost associated with each row of the batch + */ + appendBackWithCost(batch, costs){ + for (let b=0; b < batch.obs0.length; b++){ + if (this.buffer.length == this.maxlen){ + this.buffer.shift(); + } + this._insert({ + obs0: batch.obs0[b], + action: batch.actions[b], + reward: batch.rewards[b], + obs1: batch.obs1[b], + terminal: batch.terminals[b], + cost: costs[b] + }, this.buffer); + } + console.assert(this.buffer.length <= this.maxlen); + } + + /** + * @param obs0 [] + * @param action (number) + * @param reward (number) + * @param obs1 [] + * @param terminal1 (boolean) + */ + append(obs0, action, reward, obs1, terminal){ + if (this.priorBuffer.length == this.maxlen){ + this.priorBuffer.shift(); + } + this.priorBuffer.push({ + obs0: obs0, + action: action, + reward: reward, + obs1: obs1, + terminal: terminal, + cost: null + }); + console.assert(this.priorBuffer.length <= this.maxlen); + } +} +/* +var mem = new Memory(20000); +Math.seedrandom(0); +console.assert(mem.length == 0); + +var array = []; +for (let i=1; i < 40000; i++){ + mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i); +} + +console.assert(mem.length == 20000); +console.assert(mem.list[0].obs0 == "obs0-20000"); +console.assert(mem.list[19999].obs0 == "obs0-39999"); + +let batch = mem.getBatch(32); + +console.assert(batch.obs0.length == 32); +console.assert(mem.length == 20000 - 32); + +let costs = []; +for (i=31; i >= 0; i--){ + costs.push(i); +} +mem.appendBackWithCost(batch, costs); + +console.log(mem.list); + +/* +for (let i=1; i < 64; i++){ + mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i); +} +*/ \ No newline at end of file diff --git a/demo/webapp/public/js/DDPG/models.js b/demo/webapp/public/js/ddpg/models.js similarity index 80% rename from demo/webapp/public/js/DDPG/models.js rename to demo/webapp/public/js/ddpg/models.js index 65c0edb..7823acc 100644 --- a/demo/webapp/public/js/DDPG/models.js +++ b/demo/webapp/public/js/ddpg/models.js @@ -18,45 +18,29 @@ function copyModel(model, instance){ } /** - * Usefull method to copy a model - * @param model Actor - * @param perturbedActor Actor + * Copy the value of of the model into the perturbedModel and + * add a random pertubation + * @param model Actor|Critic instance + * @param perturbedActor Actor|Critic instance * @param stddev (number) * @return Copy of the model */ -function assignAndStd(actor, perturbedActor, stddev, seed){ - return tf.tidy(() => { - const weights = actor.model.trainableWeights; - for (let m=0; m < weights.length; m++){ - let shape = perturbedActor.model.trainableWeights[m].val.shape; - let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed); - let nValue = weights[m].val.add(randomTensor); - perturbedActor.model.trainableWeights[m].val.assign(nValue); - } - }); -} - -/** - * Usefull method to copy a model - * @param model Actor - * @param perturbedActor Actor - * @return Copy of the model - */ -function assignModel(model, targetModel){ +function assignAndStd(model, perturbedModel, stddev, seed){ return tf.tidy(() => { const weights = model.model.trainableWeights; for (let m=0; m < weights.length; m++){ - let nValue = weights[m].val; - targetModel.model.trainableWeights[m].val.assign(nValue); + let shape = perturbedModel.model.trainableWeights[m].val.shape; + let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed); + let nValue = weights[m].val.add(randomTensor); + perturbedModel.model.trainableWeights[m].val.assign(nValue); } }); } - /** - * Usefull method to copy a model - * @param target Actor|Critic - * @param perturbedActor Actor|Critic + * Update the target models + * @param target Actor|Critic instance + * @param perturbedActor Actor|Critic instance * @param config (Object) * @return Copy of the model */ @@ -90,6 +74,10 @@ class Actor{ this.stateSize = config.stateSize; this.nbActions = config.nbActions; this.layerNorm = config.layerNorm; + + this.firstLayerSize = config.actorFirstLayerSize; + this.secondLayerSize = config.actorSecondLayerSize; + this.seed = config.seed; this.config = config; this.obs = null; @@ -102,24 +90,22 @@ class Actor{ buildModel(obs){ this.obs = obs; - // First layer with BatchNormalization + // First layer this.firstLayer = tf.layers.dense({ - units: 64, + units: this.firstLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'relu', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - - // Second layer with BatchNormalization + // Second layer this.secondLayer = tf.layers.dense({ - units: 32, + units: this.secondLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), - activation: 'relu', // relu is add later + activation: 'relu', useBias: true, biasInitializer: "zeros" }); - // Ouput layer this.outputLayer = tf.layers.dense({ units: this.nbActions, @@ -129,12 +115,13 @@ class Actor{ useBias: true, biasInitializer: "zeros" }); - + // Actor prediction this.predict = (tfState) => { return tf.tidy(() => { if (tfState){ obs = tfState; } + let l1 = this.firstLayer.apply(obs); let l2 = this.secondLayer.apply(l1); @@ -155,6 +142,11 @@ class Critic { this.stateSize = config.stateSize; this.nbActions = config.nbActions; this.layerNorm = config.layerNorm; + + this.firstLayerSSize = config.criticFirstLayerSSize + this.firstLayerASize = config.criticFirstLayerASize; + this.secondLayerSize = config.criticSecondLayerSize; + this.seed = config.seed; this.config = config; this.obs = null; @@ -170,30 +162,28 @@ class Critic { this.obs = obs; this.action = action; + // Used to merged the two first Layer later. this.add = tf.layers.add(); - // First layer with BatchNormalization + // First layer this.firstLayerS = tf.layers.dense({ - units: 64, + units: this.firstLayerSSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - - // First layer with BatchNormalization + // First layer this.firstLayerA = tf.layers.dense({ - units: 64, + units: this.firstLayerASize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'linear', // relu is add later useBias: true, biasInitializer: "zeros" }); - - // Second layer with BatchNormalization + // Second layer this.secondLayer = tf.layers.dense({ - //inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action - units: 32, + units: this.secondLayerSize, kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}), activation: 'relu', useBias: true, @@ -210,7 +200,7 @@ class Critic { biasInitializer: "zeros" }); - // Actor prediction + // Critic prediction this.predict = (tfState, tfActions) => { return tf.tidy(() => { if (tfState && tfActions){ @@ -220,7 +210,7 @@ class Critic { let l1A = this.firstLayerA.apply(action); let l1S = this.firstLayerS.apply(obs) - + // Merged layers let concat = this.add.apply([l1A, l1S]) let l2 = this.secondLayer.apply(concat); @@ -228,6 +218,7 @@ class Critic { return this.outputLayer.apply(l2); }); } + const output = this.predict(); this.model = tf.model({inputs: [obs, action], outputs: output}); } diff --git a/demo/webapp/public/js/DDPG/noise.js b/demo/webapp/public/js/ddpg/noise.js similarity index 100% rename from demo/webapp/public/js/DDPG/noise.js rename to demo/webapp/public/js/ddpg/noise.js diff --git a/demo/webapp/public/js/level1.js b/demo/webapp/public/js/policy_monte_carlo/index.js similarity index 96% rename from demo/webapp/public/js/level1.js rename to demo/webapp/public/js/policy_monte_carlo/index.js index c40dc16..b8b5632 100644 --- a/demo/webapp/public/js/level1.js +++ b/demo/webapp/public/js/policy_monte_carlo/index.js @@ -7,7 +7,7 @@ var env = new metacar.env("canvas", levelUrl); var agent = new PolicyAgent(env); env.loop(() => { - let state = env.getState(); + let state = env.getState().lidar; displayState("realtime_viewer", state, 200, 200); let scores = agent.getStateValues(state); let reward = env.getLastReward(); diff --git a/demo/webapp/public/js/policy_agent.js b/demo/webapp/public/js/policy_monte_carlo/policy_agent.js similarity index 96% rename from demo/webapp/public/js/policy_agent.js rename to demo/webapp/public/js/policy_monte_carlo/policy_agent.js index 08cc158..b8c8bb6 100644 --- a/demo/webapp/public/js/policy_agent.js +++ b/demo/webapp/public/js/policy_monte_carlo/policy_agent.js @@ -179,14 +179,12 @@ class PolicyAgent { */ this.valueModel = await tf.loadModel('https://metacar-project.com/public/models/policy/value-model-policy-agent.json'); this.policyModel = await tf.loadModel("https://metacar-project.com/public/models/policy/policy-model-policy-agent.json"); - //this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json'); - //this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json"); } play(){ tf.tidy(() => { // Get the current state - const st = tf.tensor2d(this.env.getState(), [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); + const st = tf.tensor2d(this.env.getState().lidar, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); // Predict the policy const softmax = this.policyModel.predict(st); // Get the action @@ -231,7 +229,8 @@ class PolicyAgent { console.time("Exploring"); for (var step = 0; step < this.nb_step; step++) { // Get the current state - const array_st = this.env.getState(true); + let array_st = this.env.getState().linear; + array_st = array_st.slice(0, array_st.length - 1); // Convert the state into a tensor //const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]); const st = tf.tensor2d([array_st]); diff --git a/demo/webapp/public/js/level0.js b/demo/webapp/public/js/q_table/index.js similarity index 92% rename from demo/webapp/public/js/level0.js rename to demo/webapp/public/js/q_table/index.js index 4e44e77..4e6f42b 100644 --- a/demo/webapp/public/js/level0.js +++ b/demo/webapp/public/js/q_table/index.js @@ -12,8 +12,8 @@ var agent = new QTableAgent(env, 2); env.loop(() => { let state = env.getState(); - displayState("realtime_viewer", state, 200, 200); - let scores = agent.getStateValues(state); + displayState("realtime_viewer", state.lidar, 200, 200); + let scores = agent.getStateValues(state.lidar); let reward = env.getLastReward(); displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]); }); diff --git a/demo/webapp/public/js/q_table_agent.js b/demo/webapp/public/js/q_table/q_table_agent.js similarity index 96% rename from demo/webapp/public/js/q_table_agent.js rename to demo/webapp/public/js/q_table/q_table_agent.js index 9a96e22..e3ebc8d 100644 --- a/demo/webapp/public/js/q_table_agent.js +++ b/demo/webapp/public/js/q_table/q_table_agent.js @@ -54,7 +54,7 @@ class QTableAgent { play(){ // Get the current state - let state = this.env.getState(); + let state = this.env.getState().lidar; state = state.toString(); // In this state in not in the Q(s, a) function if (!(state in this.Q)){ @@ -108,7 +108,7 @@ class QTableAgent { console.log("episode=", ep, "eps=", eps, "mean_reward", mean(mean_reward)); } mean_reward = []; - let st = this.env.getState().toString(); + let st = this.env.getState().lidar.toString(); let act; let gamma = 0.99; let st2; @@ -117,7 +117,7 @@ class QTableAgent { act = this.pickAction(st, eps); let reward = this.env.step(act); mean_reward.push(reward); - st2 = this.env.getState().toString(); + st2 = this.env.getState().lidar.toString(); // Pick greedy action (eps = 0) act2 = this.pickAction(st2, 0.); this.createStateIfNotExist(st2); diff --git a/demo/webapp/public/js/traffic/index.js b/demo/webapp/public/js/traffic/index.js new file mode 100644 index 0000000..58531dd --- /dev/null +++ b/demo/webapp/public/js/traffic/index.js @@ -0,0 +1,67 @@ + +let levelUrl = metacar.level.level3; + +var env = new metacar.env("canvas", levelUrl); + +env.setAgentMotion(metacar.motion.ControlMotion, {}); +env.setAgentLidar({pts: 7, width: 3, height: 7, pos: -0.5}) + +// js/DDPG/ddpg.js +var agent = new DDPGAgent(env, { + stateSize: 50, + desiredActionStddev: 0.3, + initialStddev: 0.3 +}); + +initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]); + +env.loop(() => { + let state = env.getState(); + displayState("realtime_viewer", state.lidar, 200, 200); + let reward = env.getLastReward(); + const qValue = agent.getQvalue(state.linear, [state.a, state.steering]); + displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]); +}); + +env.load().then(() => { + + env.addEvent("train [Background]", () => { + let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model."); + if (train){ + env.render(false); + agent.train(false); + } + }); + + env.addEvent("Train [Show the training]", () => { + env.steping(false); + agent.train(true); + }); + + env.addEvent("shuffle", () => { + env.randomRoadPosition(); + }) + + env.addEvent("play", () => { + agent.play(); + }); + + env.addEvent("stop", () => { + agent.stop(); + }); + + env.addEvent("reset_env"); + + env.addEvent("save", () => { + agent.save("model-ddpg-traffic"); + }); + + env.addEvent("load", () => { + agent.restore("ddpg-traffic", "model-ddpg-traffic") + }); +}); + + + + + diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json similarity index 79% rename from demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json rename to demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json index 3cc61fd..f5fb493 100644 --- a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json +++ b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.json @@ -1 +1 @@ -{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file +{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin new file mode 100644 index 0000000..8b9076d Binary files /dev/null and b/demo/webapp/public/models/ddpg-road/actor-model-ddpg-road.weights.bin differ diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json similarity index 80% rename from demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json rename to demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json index 0f8345c..55cff60 100644 --- a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json +++ b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.json @@ -1 +1 @@ -{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin new file mode 100644 index 0000000..b0d7df6 Binary files /dev/null and b/demo/webapp/public/models/ddpg-road/critic-model-ddpg-road.weights.bin differ diff --git a/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.json b/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.json new file mode 100644 index 0000000..93a21c3 --- /dev/null +++ b/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[50,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.weights.bin b/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.weights.bin new file mode 100644 index 0000000..d6491c5 Binary files /dev/null and b/demo/webapp/public/models/ddpg-traffic/actor-model-ddpg-traffic.weights.bin differ diff --git a/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json new file mode 100644 index 0000000..77f3fe7 --- /dev/null +++ b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[50,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin new file mode 100644 index 0000000..a5ea774 Binary files /dev/null and b/demo/webapp/public/models/ddpg-traffic/critic-model-ddpg-traffic.weights.bin differ diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin deleted file mode 100644 index 5ed364c..0000000 Binary files a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin and /dev/null differ diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin deleted file mode 100644 index 82a1e3c..0000000 Binary files a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin and /dev/null differ