mirror of
https://github.com/wassname/metacar.git
synced 2026-09-10 12:12:06 +08:00
Prioritized Experience Replay
This commit is contained in:
Thibault Neveu
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
@@ -1,83 +0,0 @@
|
||||
|
||||
class Memory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.length = 0;
|
||||
this.start = 0;
|
||||
|
||||
this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param idx (number)
|
||||
*/
|
||||
getItem(idx){
|
||||
if (idx < 0 || idx >= this.length){
|
||||
console.error("Memory.getItem: idx not in range.");
|
||||
}
|
||||
return this.data[(this.start + idx) % this.maxlen]
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const arrLength = this.length;
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
return batch;
|
||||
}
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * arrLength);
|
||||
batch.obs0.push(this.obs0List[id]);
|
||||
batch.obs1.push(this.obs1List[id]);
|
||||
batch.rewards.push(this.rewardsList[id]);
|
||||
batch.actions.push(this.actionsList[id]);
|
||||
batch.terminals.push(this.terminals1List[id]);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal1){
|
||||
if (this.length < this.maxlen){
|
||||
this.length += 1;
|
||||
}
|
||||
else if (this.length == this.maxlen) {
|
||||
//this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
this.start = (this.start + 1) % this.maxlen;
|
||||
}
|
||||
else {
|
||||
console.error("Memory.append: This should never be printed");
|
||||
}
|
||||
this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
|
||||
this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
|
||||
this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
|
||||
this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
|
||||
this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
|
||||
}
|
||||
}
|
||||
@@ -29,13 +29,6 @@ class DDPG {
|
||||
let obsInput = tf.input({batchShape: [null, this.config.stateSize]});
|
||||
let actionInput = tf.input({batchShape: [null, this.config.nbActions]});
|
||||
|
||||
if (config.normalizeObservations){
|
||||
tf.layers.batchNormalization({
|
||||
scale: true,
|
||||
center: true
|
||||
}).apply(obsInput);
|
||||
}
|
||||
|
||||
// Randomly Initialize actor network μ(s)
|
||||
this.actor.buildModel(obsInput);
|
||||
// Randomly Initialize critic network Q(s, a)
|
||||
@@ -56,7 +49,6 @@ class DDPG {
|
||||
return this.critic.predict(tfState, tfAct);
|
||||
});
|
||||
};
|
||||
|
||||
this.criticTargetWithActorTarget = (tfState) => {
|
||||
return tf.tidy(() => {
|
||||
const tfAct = this.actorTarget.predict(tfState);
|
||||
@@ -64,7 +56,6 @@ class DDPG {
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
this.actorOptimiser = tf.train.adam(this.config.actorLr);
|
||||
this.criticOptimiser = tf.train.adam(this.config.criticLr);
|
||||
|
||||
@@ -72,23 +63,19 @@ class DDPG {
|
||||
for (let w = 0; w < this.critic.model.trainableWeights.length; w++){
|
||||
this.criticWeights.push(this.critic.model.trainableWeights[w].val);
|
||||
}
|
||||
|
||||
this.actorWeights = [];
|
||||
for (let w = 0; w < this.actor.model.trainableWeights.length; w++){
|
||||
this.actorWeights.push(this.actor.model.trainableWeights[w].val);
|
||||
}
|
||||
|
||||
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
|
||||
|
||||
this.trainActorCt = 0;
|
||||
this.trainCriticCt = 0;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Distance Measure for DDPG
|
||||
* See parameter space noise Exploration paper
|
||||
* obs (Tensor2d) Observations
|
||||
* @param observations (Tensor2d) Observations
|
||||
*/
|
||||
distanceMeasure(observations) {
|
||||
return tf.tidy(() => {
|
||||
@@ -105,6 +92,11 @@ class DDPG {
|
||||
*/
|
||||
adaptParamNoise(){
|
||||
const batch = this.memory.getBatch(this.config.batchSize);
|
||||
if (batch.obs0.length == 0){
|
||||
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
|
||||
return [0];
|
||||
}
|
||||
|
||||
let distanceV = null;
|
||||
|
||||
if (batch.obs0.length > 0){
|
||||
@@ -123,17 +115,6 @@ class DDPG {
|
||||
return distanceV;
|
||||
}
|
||||
|
||||
/**
|
||||
* Eval the actions and the Q function
|
||||
* @param states (tf.tensor2d)
|
||||
* @return actions and qValues
|
||||
*/
|
||||
eval(observation){
|
||||
const tfActions = this.perturbedActor.model.predict(observation);
|
||||
const tfQValues = this.critic.model.predict([observation, tfActions]);
|
||||
return {tfActions, tfQValues};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the estimation of the Q value given the state
|
||||
* and the action
|
||||
@@ -174,13 +155,13 @@ class DDPG {
|
||||
*/
|
||||
targetUpdate(){
|
||||
// Define in js/DDPG/models.js
|
||||
//assignModel(this.critic, this.criticTarget);
|
||||
//assignModel(this.actor, this.actorTarget);
|
||||
targetUpdate(this.criticTarget, this.critic, this.config);
|
||||
targetUpdate(this.actorTarget, this.actor, this.config);
|
||||
}
|
||||
|
||||
trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){
|
||||
trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){
|
||||
|
||||
let costs;
|
||||
|
||||
const criticLoss = this.criticOptimiser.minimize(() => {
|
||||
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
|
||||
@@ -188,18 +169,18 @@ class DDPG {
|
||||
|
||||
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1));
|
||||
|
||||
return tf.sub(tfQTargets, tfQPredictions0).square().mean();
|
||||
const erros = tf.sub(tfQTargets, tfQPredictions0).square();
|
||||
costs = erros.buffer().values;
|
||||
|
||||
return erros.mean();
|
||||
}, true, this.criticWeights);
|
||||
|
||||
this.memory.appendBackWithCost(batch, costs);
|
||||
|
||||
const loss = criticLoss.buffer().values[0];
|
||||
criticLoss.dispose();
|
||||
|
||||
//if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){
|
||||
targetUpdate(this.criticTarget, this.critic, this.config);
|
||||
//console.log("Update");
|
||||
// Saniity Check
|
||||
//}
|
||||
this.trainCriticCt += 1;
|
||||
|
||||
return loss;
|
||||
}
|
||||
@@ -215,14 +196,13 @@ class DDPG {
|
||||
|
||||
const loss = actorLoss.buffer().values[0];
|
||||
actorLoss.dispose();
|
||||
//sanityTfLoss.dispose();
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
getTfBatch(){
|
||||
// Get batch
|
||||
const batch = this.memory.getBatch(this.config.batchSize);
|
||||
const batch = this.memory.popBatch(this.config.batchSize);
|
||||
// Convert to tensors
|
||||
const tfActions = tf.tensor2d(batch.actions);
|
||||
const tfObs0 = tf.tensor2d(batch.obs0);
|
||||
@@ -237,12 +217,12 @@ class DDPG {
|
||||
_tfTerminals.dispose();
|
||||
|
||||
return {
|
||||
tfActions, tfObs0, tfObs1, tfRewards, tfTerminals
|
||||
batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals
|
||||
}
|
||||
}
|
||||
|
||||
optimizeCritic(){
|
||||
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
|
||||
|
||||
@@ -255,10 +235,10 @@ class DDPG {
|
||||
return loss;
|
||||
}
|
||||
|
||||
optimizeActor(it=1){
|
||||
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
optimizeActor(){
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const loss = this.trainActor(tfObs0, it);
|
||||
const loss = this.trainActor(tfObs0);
|
||||
|
||||
tfActions.dispose();
|
||||
tfObs0.dispose();
|
||||
@@ -270,9 +250,9 @@ class DDPG {
|
||||
}
|
||||
|
||||
optimizeCriticActor(){
|
||||
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
|
||||
const lossC = this.trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
|
||||
const lossA = this.trainActor(tfObs0);
|
||||
|
||||
tfActions.dispose();
|
||||
@@ -283,32 +263,4 @@ class DDPG {
|
||||
|
||||
return {lossC, lossA};
|
||||
}
|
||||
|
||||
trainRecord(){
|
||||
|
||||
let lossValues = [];
|
||||
|
||||
for (let i=0; i < 32; i++){
|
||||
|
||||
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const actorLoss = this.actorOptimiser.minimize(() => {
|
||||
const tfAPredictions0 = this.actor.model.predict(tfObs0);
|
||||
const loss = tfActions.sub(tfAPredictions0).square().mean();
|
||||
return loss;
|
||||
}, true, this.actorWeights);
|
||||
|
||||
const loss = actorLoss.buffer().values[0];
|
||||
lossValues.push(loss);
|
||||
|
||||
actorLoss.dispose();
|
||||
tfActions.dispose();
|
||||
tfObs0.dispose();
|
||||
tfObs1.dispose();
|
||||
tfRewards.dispose();
|
||||
tfTerminals.dispose();
|
||||
}
|
||||
console.log("Mean loss", mean(lossValues));
|
||||
}
|
||||
|
||||
}
|
||||
+53
-61
@@ -1,36 +1,45 @@
|
||||
|
||||
// This class is called from js/DDPG/index.js
|
||||
class DDPGAgent {
|
||||
|
||||
/**
|
||||
* @param env (metacar.env) Set in js/DDPG/index.js
|
||||
*/
|
||||
constructor(env){
|
||||
constructor(env, config){
|
||||
|
||||
this.stopTraining = false;
|
||||
this.env = env;
|
||||
|
||||
config = config || {};
|
||||
// Default Config
|
||||
this.config = {
|
||||
"stateSize": 17,
|
||||
"nbActions": 2,
|
||||
"layerNorm": false,
|
||||
"normalizeObservations": true,
|
||||
"seed": 0,
|
||||
"criticL2Reg": 0.01,
|
||||
"batchSize": 64,
|
||||
"actorLr": 0.0001,
|
||||
"criticLr": 0.001,
|
||||
"memorySize": 20000,
|
||||
"gamma": 0.99,
|
||||
"noiseDecay": 0.99,
|
||||
"rewardScale": 1,
|
||||
"nbEpochs": 500,
|
||||
"nbEpochsCycle": 20,
|
||||
"nbTrainSteps": 50,
|
||||
"tau": 0.01,
|
||||
"paramNoiseAdaptionInterval": 50,
|
||||
"stateSize": config.stateSize || 17,
|
||||
"nbActions": config.nbActions || 2,
|
||||
"seed": config.seed || 0,
|
||||
"batchSize": config.batchSize || 128,
|
||||
"actorLr": config.actorLr || 0.0001,
|
||||
"criticLr": config .criticLr || 0.001,
|
||||
"memorySize": config.memorySize || 30000,
|
||||
"gamma": config.gamme || 0.99,
|
||||
"noiseDecay": config.noiseDecay || 0.99,
|
||||
"rewardScale": config.rewardScale || 1,
|
||||
"nbEpochs": config.nbEpochs || 200,
|
||||
"nbEpochsCycle": config.nbEpochsCycle || 10,
|
||||
"nbTrainSteps": config.nbTrainSteps || 110,
|
||||
"tau": config.tau || 0.01,
|
||||
"initialStddev": config.initialStddev || 0.1,
|
||||
"desiredActionStddev": config.desiredActionStddev || 0.1,
|
||||
"adoptionCoefficient": config.adoptionCoefficient || 1.01,
|
||||
"actorFirstLayerSize": config.actorFirstLayerSize || 64,
|
||||
"actorSecondLayerSize": config.actorSecondLayerSize || 32,
|
||||
"criticFirstLayerSSize": config.criticFirstLayerSSize || 64,
|
||||
"criticFirstLayerASize": config.criticFirstLayerASize || 64,
|
||||
"criticSecondLayerSize": config.criticSecondLayerSize || 32,
|
||||
"maxStep": config.maxStep || 800,
|
||||
"stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError:true,
|
||||
"resetEpisode": config.resetEpisode != undefined ? config.resetEpisode:false
|
||||
};
|
||||
// From js/DDPG/noise.js
|
||||
this.noise = new AdaptiveParamNoiseSpec();
|
||||
this.noise = new AdaptiveParamNoiseSpec(this.config);
|
||||
|
||||
// Configure components.
|
||||
|
||||
@@ -51,20 +60,20 @@ class DDPGAgent {
|
||||
this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config);
|
||||
}
|
||||
|
||||
save(env){
|
||||
save(name){
|
||||
/*
|
||||
Save the network
|
||||
*/
|
||||
this.ddpg.critic.model.save('downloads://critic-model-ddpg-agent');
|
||||
this.ddpg.actor.model.save('downloads://actor-model-ddpg-agent');
|
||||
this.ddpg.critic.model.save('downloads://critic-' + name);
|
||||
this.ddpg.actor.model.save('downloads://actor-'+ name);
|
||||
}
|
||||
|
||||
async restore(){
|
||||
async restore(folder, name){
|
||||
/*
|
||||
Restore the weights of the network
|
||||
*/
|
||||
this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/ddpg/critic-model-ddpg-agent.json');
|
||||
this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/ddpg/actor-model-ddpg-agent.json");
|
||||
this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/'+folder+'/critic-'+name+'.json');
|
||||
this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/"+folder+"/actor-"+name+".json");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,11 +112,6 @@ class DDPGAgent {
|
||||
stepTrain(tfPreviousStep, mPreviousStep){
|
||||
// Get actions
|
||||
const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
|
||||
//const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
|
||||
//const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed);
|
||||
//const noisyActions = TruetfActions.add(rdNormal);
|
||||
//const tfActions = tf.clipByValue(noisyActions, -1, 1);
|
||||
|
||||
// Step in the environment with theses actions
|
||||
let mAcions = tfActions.buffer().values;
|
||||
let mReward = this.env.step([mAcions[0], mAcions[1]]);
|
||||
@@ -116,43 +120,31 @@ class DDPGAgent {
|
||||
let mState = this.env.getState().linear;
|
||||
let tfState = tf.tensor2d([mState]);
|
||||
let mDone = 0;
|
||||
|
||||
if (mReward == -1){
|
||||
if (mReward == -1 && this.config.stopOnRewardError){
|
||||
mDone = 1;
|
||||
}
|
||||
|
||||
// Add the new tuple to the buffer
|
||||
this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
|
||||
|
||||
// Dispose tensor
|
||||
// Dispose tensors
|
||||
tfPreviousStep.dispose();
|
||||
//TruetfActions.dispose();
|
||||
//noisyActions.dispose();
|
||||
tfActions.dispose();
|
||||
//rdNormal.dispose();
|
||||
//rd.dispose();
|
||||
//trueTfActions.dispose();
|
||||
return {mDone, mState, tfState}
|
||||
}
|
||||
|
||||
initTrainParam(){
|
||||
this.stopTraining = false;
|
||||
this.noisyActions = 2.0;
|
||||
return {mDone, mState, tfState};
|
||||
}
|
||||
|
||||
/**
|
||||
* Train DDPG Agent
|
||||
*/
|
||||
async train(realTime){
|
||||
this.initTrainParam();
|
||||
this.stopTraining = false;
|
||||
// One epoch
|
||||
for (let e=0; e < this.config.nbEpochs; e++){
|
||||
// Perform cycles.
|
||||
this.rewardsList = [];
|
||||
this.stepList = [];
|
||||
this.distanceList = [];
|
||||
document.getElementById("trainingProgress").innerHTML = "Progression: "+e+"/"+this.config.nbEpochs+"<br>";
|
||||
for (let c=0; c < this.config.nbEpochsCycle; c++){
|
||||
|
||||
if (c%10==0){
|
||||
logTfMemory();
|
||||
}
|
||||
@@ -164,11 +156,11 @@ class DDPGAgent {
|
||||
let step = 0;
|
||||
|
||||
console.time("LoopTime");
|
||||
for (step=0; step < 800; step++){
|
||||
for (step=0; step < this.config.maxStep; step++){
|
||||
let rel = this.stepTrain(tfPreviousStep, mPreviousStep);
|
||||
mPreviousStep = rel.mState;
|
||||
tfPreviousStep = rel.tfState;
|
||||
if (rel.mDone){
|
||||
if (rel.mDone && this.config.stopOnRewardError){
|
||||
break;
|
||||
}
|
||||
if (this.stopTraining){
|
||||
@@ -183,6 +175,9 @@ class DDPGAgent {
|
||||
let distance = this.ddpg.adaptParamNoise();
|
||||
this.distanceList.push(distance[0]);
|
||||
|
||||
if (this.config.resetEpisode){
|
||||
this.env.reset();
|
||||
}
|
||||
this.env.randomRoadPosition();
|
||||
tfPreviousStep.dispose();
|
||||
console.log("e="+ e +", c="+c);
|
||||
@@ -190,28 +185,25 @@ class DDPGAgent {
|
||||
//this.ddpg.targetUpdate();
|
||||
await tf.nextFrame();
|
||||
}
|
||||
if (this.ddpg.memory.length == this.config.memorySize){
|
||||
this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay);
|
||||
if (e > 5){
|
||||
this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
|
||||
let lossValuesCritic = [];
|
||||
let lossValuesActor = [];
|
||||
console.time("Training");
|
||||
for (let t=0; t < 100; t++){
|
||||
let lossC = this.ddpg.optimizeCritic();
|
||||
console.time("Training");
|
||||
for (let t=0; t < this.config.nbTrainSteps; t++){
|
||||
let {lossC, lossA} = this.ddpg.optimizeCriticActor();
|
||||
lossValuesCritic.push(lossC);
|
||||
}
|
||||
for (let t=0; t < 100; t++){
|
||||
let lossA = this.ddpg.optimizeActor();
|
||||
lossValuesActor.push(lossA);
|
||||
}
|
||||
console.timeEnd("Training");
|
||||
console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev);
|
||||
setMetric("CriticLoss", mean(lossValuesCritic));
|
||||
setMetric("ActorLoss", mean(lossValuesActor));
|
||||
}
|
||||
}
|
||||
|
||||
setMetric("Reward", mean(this.rewardsList));
|
||||
setMetric("EpisodeDuration", mean(this.stepList));
|
||||
setMetric("Distance", mean(this.distanceList));
|
||||
setMetric("NoiseDistance", mean(this.distanceList));
|
||||
await tf.nextFrame();
|
||||
}
|
||||
|
||||
@@ -2,60 +2,49 @@ let levelUrl = metacar.level.level2;
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1})
|
||||
|
||||
|
||||
RECORD = false;
|
||||
env.setAgentLidar({pts: 5, width: 3, height: 7, pos: -0.5})
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var agent = new DDPGAgent(env);
|
||||
var agent = new DDPGAgent(env, {
|
||||
stateSize: 26,
|
||||
resetEpisode: true
|
||||
});
|
||||
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "Distance"]);
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
|
||||
if (RECORD){
|
||||
agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1);
|
||||
}
|
||||
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let reward = env.getLastReward();
|
||||
const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
|
||||
displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]);
|
||||
});
|
||||
|
||||
|
||||
env.load().then(() => {
|
||||
|
||||
|
||||
// Train agent
|
||||
env.addEvent("train", () => {
|
||||
agent.train(false);
|
||||
env.addEvent("train [Background]", () => {
|
||||
let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model.");
|
||||
if (train){
|
||||
env.render(false);
|
||||
agent.train(false);
|
||||
}
|
||||
});
|
||||
|
||||
env.addEvent("play", () => {
|
||||
agent.play();
|
||||
});
|
||||
|
||||
env.addEvent("record", () => {
|
||||
RECORD = true;
|
||||
});
|
||||
|
||||
env.addEvent("randomPos", () => {
|
||||
env.randomRoadPosition();
|
||||
});
|
||||
|
||||
env.addEvent("stopRecord", () => {
|
||||
RECORD = false;
|
||||
});
|
||||
|
||||
env.addEvent("TrainRealTime", () => {
|
||||
env.addEvent("Train [Show the training]", () => {
|
||||
env.steping(false);
|
||||
agent.train(true);
|
||||
});
|
||||
|
||||
env.addEvent("shuffle", () => {
|
||||
env.randomRoadPosition();
|
||||
})
|
||||
|
||||
env.addEvent("play", () => {
|
||||
agent.play();
|
||||
});
|
||||
|
||||
env.addEvent("stop", () => {
|
||||
agent.stop();
|
||||
});
|
||||
@@ -63,11 +52,11 @@ env.load().then(() => {
|
||||
env.addEvent("reset_env");
|
||||
|
||||
env.addEvent("save", () => {
|
||||
agent.save();
|
||||
agent.save("model-ddpg-road");
|
||||
});
|
||||
|
||||
env.addEvent("load", () => {
|
||||
agent.restore()
|
||||
agent.restore("ddpg-road", "model-ddpg-road")
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
|
||||
class Memory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.buffer = [];
|
||||
this.priorBuffer = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
if (batchSize > this.priorBuffer.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
_bufferBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let nElem = this.buffer.pop();
|
||||
batch.obs0.push(nElem.obs0);
|
||||
batch.obs1.push(nElem.obs1);
|
||||
batch.rewards.push(nElem.reward);
|
||||
batch.actions.push(nElem.action);
|
||||
batch.terminals.push(nElem.terminal);
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
|
||||
return batch
|
||||
}
|
||||
|
||||
_addRandomBufferBatch(batchSize, batch){
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
popBatch(batchSize){
|
||||
let originalBatchSize = batchSize;
|
||||
let priorBufferBatchSize;
|
||||
let bufferBatchSize;
|
||||
if (batchSize % 2 != 0){
|
||||
console.warn("Batch size should be a even.")
|
||||
}
|
||||
if (this.priorBuffer.length < batchSize/2){
|
||||
//console.log("get full batch from buffer");
|
||||
const batch = this._bufferBatch(batchSize);
|
||||
console.assert(batch.obs0.length == batchSize);
|
||||
return batch;
|
||||
}
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
//console.log("Get half of prior and other from buffer.");
|
||||
batchSize = batchSize / 2;
|
||||
}
|
||||
else{
|
||||
//console.log("Get all from priorBuffer");
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
this.priorBuffer.splice(id, 1);
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
this._addRandomBufferBatch(batchSize, batch);
|
||||
}
|
||||
console.assert(batch.obs0.length == originalBatchSize);
|
||||
return batch
|
||||
}
|
||||
|
||||
_insert(element, array) {
|
||||
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){
|
||||
array.unshift(element);
|
||||
return array;
|
||||
}
|
||||
array.splice(this._locationOf(element, array) + 1, 0, element);
|
||||
return array;
|
||||
}
|
||||
|
||||
_locationOf(element, array, start, end) {
|
||||
start = start || 0;
|
||||
end = end || array.length;
|
||||
|
||||
var pivot = parseInt(start + (end - start) / 2, 10);
|
||||
|
||||
if (end-start <= 1 || array[pivot] === element) return pivot;
|
||||
|
||||
if (array[pivot].cost != null && array[pivot].cost < element.cost) {
|
||||
return this._locationOf(element, array, pivot, end);
|
||||
} else {
|
||||
return this._locationOf(element, array, start, pivot);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param batch (Object) from getBatch()
|
||||
* @param cost (number) Cost associated with each row of the batch
|
||||
*/
|
||||
appendBackWithCost(batch, costs){
|
||||
for (let b=0; b < batch.obs0.length; b++){
|
||||
if (this.buffer.length == this.maxlen){
|
||||
this.buffer.shift();
|
||||
}
|
||||
this._insert({
|
||||
obs0: batch.obs0[b],
|
||||
action: batch.actions[b],
|
||||
reward: batch.rewards[b],
|
||||
obs1: batch.obs1[b],
|
||||
terminal: batch.terminals[b],
|
||||
cost: costs[b]
|
||||
}, this.buffer);
|
||||
}
|
||||
console.assert(this.buffer.length <= this.maxlen);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal){
|
||||
if (this.priorBuffer.length == this.maxlen){
|
||||
this.priorBuffer.shift();
|
||||
}
|
||||
this.priorBuffer.push({
|
||||
obs0: obs0,
|
||||
action: action,
|
||||
reward: reward,
|
||||
obs1: obs1,
|
||||
terminal: terminal,
|
||||
cost: null
|
||||
});
|
||||
console.assert(this.priorBuffer.length <= this.maxlen);
|
||||
}
|
||||
}
|
||||
/*
|
||||
var mem = new Memory(20000);
|
||||
Math.seedrandom(0);
|
||||
console.assert(mem.length == 0);
|
||||
|
||||
var array = [];
|
||||
for (let i=1; i < 40000; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
|
||||
console.assert(mem.length == 20000);
|
||||
console.assert(mem.list[0].obs0 == "obs0-20000");
|
||||
console.assert(mem.list[19999].obs0 == "obs0-39999");
|
||||
|
||||
let batch = mem.getBatch(32);
|
||||
|
||||
console.assert(batch.obs0.length == 32);
|
||||
console.assert(mem.length == 20000 - 32);
|
||||
|
||||
let costs = [];
|
||||
for (i=31; i >= 0; i--){
|
||||
costs.push(i);
|
||||
}
|
||||
mem.appendBackWithCost(batch, costs);
|
||||
|
||||
console.log(mem.list);
|
||||
|
||||
/*
|
||||
for (let i=1; i < 64; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
*/
|
||||
@@ -18,45 +18,29 @@ function copyModel(model, instance){
|
||||
}
|
||||
|
||||
/**
|
||||
* Usefull method to copy a model
|
||||
* @param model Actor
|
||||
* @param perturbedActor Actor
|
||||
* Copy the value of of the model into the perturbedModel and
|
||||
* add a random pertubation
|
||||
* @param model Actor|Critic instance
|
||||
* @param perturbedActor Actor|Critic instance
|
||||
* @param stddev (number)
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function assignAndStd(actor, perturbedActor, stddev, seed){
|
||||
return tf.tidy(() => {
|
||||
const weights = actor.model.trainableWeights;
|
||||
for (let m=0; m < weights.length; m++){
|
||||
let shape = perturbedActor.model.trainableWeights[m].val.shape;
|
||||
let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed);
|
||||
let nValue = weights[m].val.add(randomTensor);
|
||||
perturbedActor.model.trainableWeights[m].val.assign(nValue);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Usefull method to copy a model
|
||||
* @param model Actor
|
||||
* @param perturbedActor Actor
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function assignModel(model, targetModel){
|
||||
function assignAndStd(model, perturbedModel, stddev, seed){
|
||||
return tf.tidy(() => {
|
||||
const weights = model.model.trainableWeights;
|
||||
for (let m=0; m < weights.length; m++){
|
||||
let nValue = weights[m].val;
|
||||
targetModel.model.trainableWeights[m].val.assign(nValue);
|
||||
let shape = perturbedModel.model.trainableWeights[m].val.shape;
|
||||
let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed);
|
||||
let nValue = weights[m].val.add(randomTensor);
|
||||
perturbedModel.model.trainableWeights[m].val.assign(nValue);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Usefull method to copy a model
|
||||
* @param target Actor|Critic
|
||||
* @param perturbedActor Actor|Critic
|
||||
* Update the target models
|
||||
* @param target Actor|Critic instance
|
||||
* @param perturbedActor Actor|Critic instance
|
||||
* @param config (Object)
|
||||
* @return Copy of the model
|
||||
*/
|
||||
@@ -90,6 +74,10 @@ class Actor{
|
||||
this.stateSize = config.stateSize;
|
||||
this.nbActions = config.nbActions;
|
||||
this.layerNorm = config.layerNorm;
|
||||
|
||||
this.firstLayerSize = config.actorFirstLayerSize;
|
||||
this.secondLayerSize = config.actorSecondLayerSize;
|
||||
|
||||
this.seed = config.seed;
|
||||
this.config = config;
|
||||
this.obs = null;
|
||||
@@ -102,24 +90,22 @@ class Actor{
|
||||
buildModel(obs){
|
||||
this.obs = obs;
|
||||
|
||||
// First layer with BatchNormalization
|
||||
// First layer
|
||||
this.firstLayer = tf.layers.dense({
|
||||
units: 64,
|
||||
units: this.firstLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu', // relu is add later
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Second layer with BatchNormalization
|
||||
// Second layer
|
||||
this.secondLayer = tf.layers.dense({
|
||||
units: 32,
|
||||
units: this.secondLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu', // relu is add later
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Ouput layer
|
||||
this.outputLayer = tf.layers.dense({
|
||||
units: this.nbActions,
|
||||
@@ -129,12 +115,13 @@ class Actor{
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Actor prediction
|
||||
this.predict = (tfState) => {
|
||||
return tf.tidy(() => {
|
||||
if (tfState){
|
||||
obs = tfState;
|
||||
}
|
||||
|
||||
let l1 = this.firstLayer.apply(obs);
|
||||
let l2 = this.secondLayer.apply(l1);
|
||||
|
||||
@@ -155,6 +142,11 @@ class Critic {
|
||||
this.stateSize = config.stateSize;
|
||||
this.nbActions = config.nbActions;
|
||||
this.layerNorm = config.layerNorm;
|
||||
|
||||
this.firstLayerSSize = config.criticFirstLayerSSize
|
||||
this.firstLayerASize = config.criticFirstLayerASize;
|
||||
this.secondLayerSize = config.criticSecondLayerSize;
|
||||
|
||||
this.seed = config.seed;
|
||||
this.config = config;
|
||||
this.obs = null;
|
||||
@@ -170,30 +162,28 @@ class Critic {
|
||||
this.obs = obs;
|
||||
this.action = action;
|
||||
|
||||
// Used to merged the two first Layer later.
|
||||
this.add = tf.layers.add();
|
||||
|
||||
// First layer with BatchNormalization
|
||||
// First layer
|
||||
this.firstLayerS = tf.layers.dense({
|
||||
units: 64,
|
||||
units: this.firstLayerSSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// First layer with BatchNormalization
|
||||
// First layer
|
||||
this.firstLayerA = tf.layers.dense({
|
||||
units: 64,
|
||||
units: this.firstLayerASize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Second layer with BatchNormalization
|
||||
// Second layer
|
||||
this.secondLayer = tf.layers.dense({
|
||||
//inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action
|
||||
units: 32,
|
||||
units: this.secondLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
@@ -210,7 +200,7 @@ class Critic {
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Actor prediction
|
||||
// Critic prediction
|
||||
this.predict = (tfState, tfActions) => {
|
||||
return tf.tidy(() => {
|
||||
if (tfState && tfActions){
|
||||
@@ -220,7 +210,7 @@ class Critic {
|
||||
|
||||
let l1A = this.firstLayerA.apply(action);
|
||||
let l1S = this.firstLayerS.apply(obs)
|
||||
|
||||
// Merged layers
|
||||
let concat = this.add.apply([l1A, l1S])
|
||||
|
||||
let l2 = this.secondLayer.apply(concat);
|
||||
@@ -228,6 +218,7 @@ class Critic {
|
||||
return this.outputLayer.apply(l2);
|
||||
});
|
||||
}
|
||||
|
||||
const output = this.predict();
|
||||
this.model = tf.model({inputs: [obs, action], outputs: output});
|
||||
}
|
||||
@@ -7,7 +7,7 @@ var env = new metacar.env("canvas", levelUrl);
|
||||
var agent = new PolicyAgent(env);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
let state = env.getState().lidar;
|
||||
displayState("realtime_viewer", state, 200, 200);
|
||||
let scores = agent.getStateValues(state);
|
||||
let reward = env.getLastReward();
|
||||
+3
-4
@@ -179,14 +179,12 @@ class PolicyAgent {
|
||||
*/
|
||||
this.valueModel = await tf.loadModel('https://metacar-project.com/public/models/policy/value-model-policy-agent.json');
|
||||
this.policyModel = await tf.loadModel("https://metacar-project.com/public/models/policy/policy-model-policy-agent.json");
|
||||
//this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json');
|
||||
//this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json");
|
||||
}
|
||||
|
||||
play(){
|
||||
tf.tidy(() => {
|
||||
// Get the current state
|
||||
const st = tf.tensor2d(this.env.getState(), [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
const st = tf.tensor2d(this.env.getState().lidar, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
// Predict the policy
|
||||
const softmax = this.policyModel.predict(st);
|
||||
// Get the action
|
||||
@@ -231,7 +229,8 @@ class PolicyAgent {
|
||||
console.time("Exploring");
|
||||
for (var step = 0; step < this.nb_step; step++) {
|
||||
// Get the current state
|
||||
const array_st = this.env.getState(true);
|
||||
let array_st = this.env.getState().linear;
|
||||
array_st = array_st.slice(0, array_st.length - 1);
|
||||
// Convert the state into a tensor
|
||||
//const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
const st = tf.tensor2d([array_st]);
|
||||
@@ -12,8 +12,8 @@ var agent = new QTableAgent(env, 2);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state, 200, 200);
|
||||
let scores = agent.getStateValues(state);
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let scores = agent.getStateValues(state.lidar);
|
||||
let reward = env.getLastReward();
|
||||
displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]);
|
||||
});
|
||||
+3
-3
@@ -54,7 +54,7 @@ class QTableAgent {
|
||||
|
||||
play(){
|
||||
// Get the current state
|
||||
let state = this.env.getState();
|
||||
let state = this.env.getState().lidar;
|
||||
state = state.toString();
|
||||
// In this state in not in the Q(s, a) function
|
||||
if (!(state in this.Q)){
|
||||
@@ -108,7 +108,7 @@ class QTableAgent {
|
||||
console.log("episode=", ep, "eps=", eps, "mean_reward", mean(mean_reward));
|
||||
}
|
||||
mean_reward = [];
|
||||
let st = this.env.getState().toString();
|
||||
let st = this.env.getState().lidar.toString();
|
||||
let act;
|
||||
let gamma = 0.99;
|
||||
let st2;
|
||||
@@ -117,7 +117,7 @@ class QTableAgent {
|
||||
act = this.pickAction(st, eps);
|
||||
let reward = this.env.step(act);
|
||||
mean_reward.push(reward);
|
||||
st2 = this.env.getState().toString();
|
||||
st2 = this.env.getState().lidar.toString();
|
||||
// Pick greedy action (eps = 0)
|
||||
act2 = this.pickAction(st2, 0.);
|
||||
this.createStateIfNotExist(st2);
|
||||
@@ -0,0 +1,67 @@
|
||||
|
||||
let levelUrl = metacar.level.level3;
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
env.setAgentLidar({pts: 7, width: 3, height: 7, pos: -0.5})
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var agent = new DDPGAgent(env, {
|
||||
stateSize: 50,
|
||||
desiredActionStddev: 0.3,
|
||||
initialStddev: 0.3
|
||||
});
|
||||
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let reward = env.getLastReward();
|
||||
const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
|
||||
displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]);
|
||||
});
|
||||
|
||||
env.load().then(() => {
|
||||
|
||||
env.addEvent("train [Background]", () => {
|
||||
let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model.");
|
||||
if (train){
|
||||
env.render(false);
|
||||
agent.train(false);
|
||||
}
|
||||
});
|
||||
|
||||
env.addEvent("Train [Show the training]", () => {
|
||||
env.steping(false);
|
||||
agent.train(true);
|
||||
});
|
||||
|
||||
env.addEvent("shuffle", () => {
|
||||
env.randomRoadPosition();
|
||||
})
|
||||
|
||||
env.addEvent("play", () => {
|
||||
agent.play();
|
||||
});
|
||||
|
||||
env.addEvent("stop", () => {
|
||||
agent.stop();
|
||||
});
|
||||
|
||||
env.addEvent("reset_env");
|
||||
|
||||
env.addEvent("save", () => {
|
||||
agent.save("model-ddpg-traffic");
|
||||
});
|
||||
|
||||
env.addEvent("load", () => {
|
||||
agent.restore("ddpg-traffic", "model-ddpg-traffic")
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
+1
-1
@@ -1 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[50,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[50,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user