wassname/metacar
1
0
Fork 0
mirror of https://github.com/wassname/metacar.git synced 2026-09-09 11:26:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Steering Angle: works

Browse Source
This commit is contained in:
Thibault Neveu
2018-06-25 09:38:11 +01:00
parent dac1db06ef
commit 9342208dc7
11 changed files with 619 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
demo/webapp/public/js/DDPG/ddpg.js
+135 -53
View File
@@ -17,11 +17,10 @@ class DDPG {
* @param memory (Memory class)
* @param noise (Noise class)
*/
constructor(actor, critic, memoryPos, memoryNeg, noise, config){
constructor(actor, critic, memory, noise, config){
this.actor = actor;
this.critic = critic;
this.memoryPos = memoryPos;
this.memoryNeg = memoryNeg;
this.memory = memory;
this.noise = noise;
this.config = config;
this.tfGamma = tf.scalar(config.gamma);
@@ -42,6 +41,7 @@ class DDPG {
// Randomly Initialize critic network Q(s, a)
this.critic.buildModel(obsInput, actionInput);
// Define in js/DDPG/models.js
// Init target network Q' and μ' with the same weights
this.actorTarget = copyModel(this.actor, Actor);
@@ -50,35 +50,38 @@ class DDPG {
this.perturbedActor = copyModel(this.actor, Actor);
//this.adaptivePerturbedActor = copyModel(this.actor, Actor);
this.criticWithActor = (tfState) => {
return tf.tidy(() => {
const tfAct = this.actor.predict(tfState);
return this.critic.predict(tfState, tfAct);
});
};
this.criticTargetWithActorTarget = (tfState) => {
return tf.tidy(() => {
const tfAct = this.actorTarget.predict(tfState);
return this.criticTarget.predict(tfState, tfAct);
});
};
this.actorOptimiser = tf.train.adam(this.config.actorLr);
this.criticOptimiser = tf.train.adam(this.config.criticLr);
this.criticWeights = [];
for (let w = 0; w < this.critic.model.weights.length; w++){
this.criticWeights.push(this.critic.model.weights[w].val);
for (let w = 0; w < this.critic.model.trainableWeights.length; w++){
this.criticWeights.push(this.critic.model.trainableWeights[w].val);
}
this.actorWeights = [];
for (let w = 0; w < this.actor.model.weights.length; w++){
this.actorWeights.push(this.actor.model.weights[w].val);
for (let w = 0; w < this.actor.model.trainableWeights.length; w++){
this.actorWeights.push(this.actor.model.trainableWeights[w].val);
}
// Return a batch from positive and negative experiences
this.memory = {
getBatch: (size) => {
let batch1 = this.memoryPos.getBatch(size/2);
let batch2 = this.memoryNeg.getBatch(size/2);
return {
'obs0': batch1.obs0.concat(batch2.obs0),
'obs1': batch1.obs1.concat(batch2.obs1),
'rewards': batch1.rewards.concat(batch2.rewards),
'actions': batch1.actions.concat(batch2.actions),
'terminals': batch1.terminals.concat(batch2.terminals),
};
}
};
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev);
this.trainActorCt = 0;
this.trainCriticCt = 0;
}
@@ -102,17 +105,22 @@ class DDPG {
*/
adaptParamNoise(){
const batch = this.memory.getBatch(this.config.batchSize);
const tfObs0 = tf.tensor2d(batch.obs0);
const distance = this.distanceMeasure(tfObs0);
let distanceV = null;
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev);
if (batch.obs0.length > 0){
const tfObs0 = tf.tensor2d(batch.obs0);
const distance = this.distanceMeasure(tfObs0);
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
distanceV = distance.buffer().values;
this.noise.adapt(distanceV[0]);
const distanceV = distance.buffer().values;
this.noise.adapt(distanceV[0]);
setMetric("Distance", distanceV[0])
distance.dispose();
tfObs0.dispose();
}
distance.dispose();
tfObs0.dispose();
return distanceV;
}
/**
@@ -176,32 +184,59 @@ class DDPG {
const criticLoss = this.criticOptimiser.minimize(() => {
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
const tfAPredictions = this.actorTarget.model.predict(tfObs1);
const tfQPredictions = this.criticTarget.model.predict([tfObs1, tfAPredictions]);
const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1);
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1));
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions));
const loss = tf.sub(tfQTargets, tfQPredictions0).square().mean();
return loss;
return tf.sub(tfQTargets, tfQPredictions0).square().mean();
}, true, this.criticWeights);
setMetric("CriticLoss", criticLoss.buffer().values[0]);
const loss = criticLoss.buffer().values[0];
criticLoss.dispose();
//if (this.trainCriticCt % 200 == 0 && this.trainCriticCt != 0){
targetUpdate(this.criticTarget, this.critic, this.config);
//console.log("Update");
// Saniity Check
//}
this.trainCriticCt += 1;
return loss;
}
trainActor(tfObs0, it){
for (let i = 0; i < it; i++){
const actorLoss = this.actorOptimiser.minimize(() => {
const tfAPredictions0 = this.actor.model.predict(tfObs0);
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfAPredictions0]);
const loss = tf.mean(tfQPredictions0).mul(tf.scalar(-1));
return loss;
}, true, this.actorWeights);
const vLoss = actorLoss.buffer().values[0];
setMetric("ActorLoss", actorLoss.buffer().values[0]);
actorLoss.dispose();
trainActor(tfObs0){
const actorLoss = this.actorOptimiser.minimize(() => {
const tfQPredictions0 = this.criticWithActor(tfObs0);
return tf.mean(tfQPredictions0).mul(tf.scalar(-1.))
}, true, this.actorWeights);
/*
const sanityTfLoss = tf.tidy(() => {
const tfQPredictions0 = this.criticTargetWithActor(tfObs0);
const mn = tf.mean(tfQPredictions0);
const loss = mn.mul(tf.scalar(-1));
return loss;
});
const sanityLoss = sanityTfLoss.buffer().values[0];
if (sanityLoss == loss){
console.warn("Sanity check failed. The optimisation have no effet here.");
}
*/
//if (this.trainActorCt % 200 == 0 && this.trainActorCt != 0){
targetUpdate(this.actorTarget, this.actor, this.config);
//}
//this.trainActorCt += 1;
const loss = actorLoss.buffer().values[0];
actorLoss.dispose();
//sanityTfLoss.dispose();
return loss;
}
getTfBatch(){
@@ -225,27 +260,74 @@ class DDPG {
}
}
async optimizeCritic(){
optimizeCritic(){
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return loss;
}
async optimizeActor(it=1){
optimizeActor(it=1){
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
this.trainActor(tfObs0, it);
const loss = this.trainActor(tfObs0, it);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return loss;
}
optimizeCriticActor(){
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
const lossC = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
const lossA = this.trainActor(tfObs0);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return {lossC, lossA};
}
trainRecord(){
let lossValues = [];
for (let i=0; i < 32; i++){
const {tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
const actorLoss = this.actorOptimiser.minimize(() => {
const tfAPredictions0 = this.actor.model.predict(tfObs0);
const loss = tfActions.sub(tfAPredictions0).square().mean();
return loss;
}, true, this.actorWeights);
const loss = actorLoss.buffer().values[0];
lossValues.push(loss);
actorLoss.dispose();
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
}
console.log("Mean loss", mean(lossValues));
}
}
demo/webapp/public/js/DDPG/ddpg_agent.js
+74 -36
View File
@@ -10,21 +10,23 @@ class DDPGAgent {
this.env = env;
// Default Config
this.config = {
"stateSize": 26,
"nbActions": 2,
"layerNom": true,
"stateSize": 17,
"nbActions": 1,
"layerNorm": false,
"normalizeObservations": true,
"seed": 0,
"criticL2Reg": 0.01,
"batchSize": 32,
"batchSize": 64,
"actorLr": 0.0001,
"criticLr": 0.001,
"memorySize": 15000,
"gamma": 0.99,
"noiseDecay": 0.95,
"rewardScale": 1,
"nbEpochs": 500,
"nbEpochsCycle": 100,
"nbEpochsCycle": 20,
"nbTrainSteps": 50,
"tau": 0.001,
"tau": 0.01,
"paramNoiseAdaptionInterval": 50,
};
// From js/DDPG/noise.js
@@ -34,8 +36,7 @@ class DDPGAgent {
// Buffer replay
// The baseline use 1e6 but this size should be enough for this problem
this.memoryPos = new Memory(5000);
this.memoryNeg = new Memory(5000);
this.memory = new Memory(this.config.memorySize);
// Actor and Critic are from js/DDPG/models.js
this.actor = new Actor(this.config);
this.critic = new Critic(this.config);
@@ -44,9 +45,10 @@ class DDPGAgent {
Math.seedrandom(0);
this.rewardsList = [];
this.epiDuration = [];
// DDPG
this.ddpg = new DDPG(this.actor, this.critic, this.memoryPos, this.memoryNeg, this.noise, this.config);
this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config);
}
/**
@@ -54,11 +56,11 @@ class DDPGAgent {
*/
play(){
// Get the current state
const state = agent.env.getState().linear;
const state = this.env.getState().linear;
// Pick an action
const tfActions = agent.ddpg.predict(tf.tensor2d([state]));
const tfActions = this.ddpg.predict(tf.tensor2d([state]));
const actions = tfActions.buffer().values;
agent.env.step([actions[0], actions[1]]);
agent.env.step([1., actions[0]]);
tfActions.dispose();
}
@@ -85,48 +87,66 @@ class DDPGAgent {
stepTrain(tfPreviousStep, mPreviousStep){
// Get actions
const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
//const TruetfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
//const rdNormal = tf.randomNormal(TruetfActions.shape, 0, this.noisyActions, "float32", this.config.seed);
//const noisyActions = TruetfActions.add(rdNormal);
//const tfActions = tf.clipByValue(noisyActions, -1, 1);
// Step in the environment with theses actions
let mAcions = tfActions.buffer().values;
let mReward = this.env.step([mAcions[0], mAcions[1]]);
let mReward = this.env.step([1., mAcions[0]]);
this.rewardsList.push(mReward);
// Get the new observations
let mState = this.env.getState().linear;
let tfState = tf.tensor2d([mState]);
let mDone = 0;
if (mReward == -1){
mDone = 1;
}
// Add the new tuple to the buffer
if (mReward >= 0)
this.ddpg.memoryPos.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
else
this.ddpg.memoryNeg.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
this.ddpg.memory.append(mPreviousStep, [mAcions[0]], mReward, mState, mDone);
// Dispose tensor
tfPreviousStep.dispose();
//TruetfActions.dispose();
//noisyActions.dispose();
tfActions.dispose();
//rdNormal.dispose();
//rd.dispose();
//trueTfActions.dispose();
return {mDone, mState, tfState}
}
initTrainParam(){
this.stopTraining = false;
this.noisyActions = 2.0;
}
/**
* Train DDPG Agent
*/
async train(realTime){
this.stopTraining = false;
this.initTrainParam();
// One epoch
for (let e=0; e < this.config.nbEpochs; e++){
// Perform cycles.
this.rewardsList = [];
this.stepList = [];
this.distanceList = [];
for (let c=0; c < this.config.nbEpochsCycle; c++){
if (c%10==0){
logTfMemory();
}
this.rewardsList = [];
// Perform rollouts.
// Get current observation
let mPreviousStep = this.env.getState().linear;
let tfPreviousStep = tf.tensor2d([mPreviousStep]);
let step = 0;
console.time("LoopTime");
for (step=0; step < 800; step++){
let rel = this.stepTrain(tfPreviousStep, mPreviousStep);
@@ -139,30 +159,48 @@ class DDPGAgent {
this.env.render(true);
return;
}
if (realTime)
if (realTime && step % 10 == 0)
await tf.nextFrame();
}
this.stepList.push(step);
console.timeEnd("LoopTime");
this.env.reset();
let distance = this.ddpg.adaptParamNoise();
this.distanceList.push(distance[0]);
this.env.randomRoadPosition();
tfPreviousStep.dispose();
// Mean is define is js/utils.js
console.log("e="+ e +", c="+c);
setMetric("Reward", mean(this.rewardsList));
setMetric("EpisodeDuration", step);
this.ddpg.adaptParamNoise();
//this.ddpg.targetUpdate();
await tf.nextFrame();
}
console.time("LoopTrain");
for (let t=0; t < 100; t++){
this.ddpg.optimizeCritic();
}
for (let t=0; t < 100; t++){
this.ddpg.optimizeActor();
}
console.timeEnd("LoopTrain");
this.ddpg.targetUpdate();
if (this.ddpg.memory.length == this.config.memorySize){
this.noisyActions = Math.max(0.1, this.noisyActions * this.config.noiseDecay);
this.ddpg.noise.desiredActionStddev = Math.min(0.5, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
let lossValuesCritic = [];
let lossValuesActor = [];
console.time("Training");
for (let t=0; t < 100; t++){
let lossC = this.ddpg.optimizeCritic();
lossValuesCritic.push(lossC);
}
for (let t=0; t < 100; t++){
let lossA = this.ddpg.optimizeActor();
lossValuesActor.push(lossA);
}
console.timeEnd("Training");
console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev);
setMetric("CriticLoss", mean(lossValuesCritic));
setMetric("ActorLoss", mean(lossValuesActor));
}
setMetric("Reward", mean(this.rewardsList));
setMetric("EpisodeDuration", mean(this.stepList));
setMetric("Distance", mean(this.distanceList));
await tf.nextFrame();
}
this.env.render(true);
}
this.env.render(true);
}
};
demo/webapp/public/js/DDPG/index.js
+42 -1
View File
@@ -3,8 +3,11 @@ let levelUrl = metacar.level.level2;
var env = new metacar.env("canvas", levelUrl);
env.setAgentMotion(metacar.motion.ControlMotion, {});
env.setAgentLidar({pts: 4, width: 2, height: 9, pos: 1})
RECORD = false;
// js/DDPG/ddpg.js
var agent = new DDPGAgent(env);
@@ -12,13 +15,21 @@ initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "Epi
env.loop(() => {
let state = env.getState();
if (RECORD){
agent.ddpg.memory.append(state.linear, [state.a, state.steering], 1, [1, 1, 1], 1);
}
displayState("realtime_viewer", state.lidar, 200, 200);
let reward = env.getLastReward();
const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
const qValue = agent.getQvalue(state.linear, [state.steering]);
displayScores("realtime_viewer", [qValue], reward, ["Q(a, s)"]);
});
env.load().then(() => {
// Train agent
env.addEvent("train", () => {
agent.train(false);
@@ -28,6 +39,22 @@ env.load().then(() => {
agent.play();
});
env.addEvent("record", () => {
RECORD = true;
});
env.addEvent("randomPos", () => {
env.randomRoadPosition();
});
env.addEvent("stopRecord", () => {
RECORD = false;
});
env.addEvent("trainOnRecord", () => {
agent.ddpg.trainRecord();
});
env.addEvent("TrainRealTime", () => {
env.steping(false);
agent.train(true);
@@ -38,4 +65,18 @@ env.load().then(() => {
});
env.addEvent("reset_env");
env.addEvent("load", (content) => {
content = JSON.parse(content);
for (let c=0; c < content.obs0.length; c++){
if (content.obs0[c] != 0){
agent.ddpg.memory.append(content.obs0[c], content.actions[c], content.rewards[c], content.obs0[c], content.terminals[c]);
}
}
console.log("dataReady");
}, {local: true});
});
demo/webapp/public/js/DDPG/models.js
+50 -85
View File
@@ -24,14 +24,14 @@ function copyModel(model, instance){
* @param stddev (number)
* @return Copy of the model
*/
function assignAndStd(actor, perturbedActor, stddev){
function assignAndStd(actor, perturbedActor, stddev, seed){
return tf.tidy(() => {
const weights = actor.model.weights;
const weights = actor.model.trainableWeights;
for (let m=0; m < weights.length; m++){
let shape = perturbedActor.model.weights[m].val.shape;
let randomTensor = tf.randomNormal(shape, 0, stddev);
let shape = perturbedActor.model.trainableWeights[m].val.shape;
let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed);
let nValue = weights[m].val.add(randomTensor);
perturbedActor.model.weights[m].val.assign(nValue);
perturbedActor.model.trainableWeights[m].val.assign(nValue);
}
});
}
@@ -44,10 +44,10 @@ function assignAndStd(actor, perturbedActor, stddev){
*/
function assignModel(model, targetModel){
return tf.tidy(() => {
const weights = model.model.weights;
const weights = model.model.trainableWeights;
for (let m=0; m < weights.length; m++){
let nValue = weights[m].val;
targetModel.model.weights[m].val.assign(nValue);
targetModel.model.trainableWeights[m].val.assign(nValue);
}
});
}
@@ -62,15 +62,20 @@ function assignModel(model, targetModel){
*/
function targetUpdate(target, original, config){
return tf.tidy(() => {
const originalW = original.model.weights;
const targetW = target.model.weights;
const originalW = original.model.trainableWeights;
const targetW = target.model.trainableWeights;
const one = tf.scalar(1);
const tau = tf.scalar(config.tau);
for (let m=0; m < originalW.length; m++){
const lastValue = target.model.trainableWeights[m].val.clone();
let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau)));
target.model.weights[m].val.assign(nValue);
target.model.trainableWeights[m].val.assign(nValue);
const diff = lastValue.sub(target.model.trainableWeights[m].val).mean().buffer().values;
if (diff[0] == 0){
console.warn("targetUpdate: Nothing have been changed!")
}
}
});
}
@@ -97,45 +102,23 @@ class Actor{
buildModel(obs){
this.obs = obs;
this.firstLayerBatchNorm = null;
this.secondLayerBatchNorm = null;
this.relu1 = tf.layers.activation({activation: 'relu'});
//this.relu2 = tf.layers.activation({activation: 'relu'});
// First layer with BatchNormalization
this.firstLayer = tf.layers.dense({
units: 64,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
activation: 'linear', // relu is add later
activation: 'relu', // relu is add later
useBias: true,
biasInitializer: "zeros"
});
if (this.layerNorm){
// WARNING: BatchNormalization instead of layerNormalization
this.firstLayerBatchNorm = tf.layers.batchNormalization({
scale: true,
center: true
});
}
/*
// Second layer with BatchNormalization
this.secondLayer = tf.layers.dense({
units: 64,
units: 32,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
activation: 'linear', // relu is add later
activation: 'relu', // relu is add later
useBias: true,
biasInitializer: "zeros"
});
if (this.layerNorm){
// WARNING: BatchNormalization instead of layerNormalization
this.secondLayerBatchNorm = tf.layers.batchNormalization({
scale: true,
center: true
});
}*/
// Ouput layer
this.outputLayer = tf.layers.dense({
@@ -147,22 +130,15 @@ class Actor{
biasInitializer: "zeros"
});
this.predict = () => {
return tf.tidy(() => {
this.predict = (tfState) => {
return tf.tidy(() => {
if (tfState){
obs = tfState;
}
let l1 = this.firstLayer.apply(obs);
if (this.firstLayerBatchNorm){
l1 = this.firstLayerBatchNorm.apply(l1);
}
l1 = this.relu1.apply(l1);
/*
let l2 = this.secondLayer.apply(l1);
if (this.secondLayerBatchNorm){
l2 = this.secondLayerBatchNorm.apply(l2);
}
l2 = this.relu2.apply(l2);
*/
return this.outputLayer.apply(l1);
return this.outputLayer.apply(l2);
});
}
const output = this.predict();
@@ -194,45 +170,35 @@ class Critic {
this.obs = obs;
this.action = action;
this.firstLayerBatchNorm = null;
this.secondLayerBatchNorm = null;
this.relu1 = tf.layers.activation({activation: 'relu'});
this.relu2 = tf.layers.activation({activation: 'relu'});
this.concat = tf.layers.concatenate();
this.add = tf.layers.add();
// First layer with BatchNormalization
this.firstLayer = tf.layers.dense({
this.firstLayerS = tf.layers.dense({
units: 64,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
activation: 'linear', // relu is add later
useBias: true,
biasInitializer: "zeros"
});
// First layer with BatchNormalization
this.firstLayerA = tf.layers.dense({
units: 64,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
activation: 'linear', // relu is add later
useBias: true,
biasInitializer: "zeros"
});
if (this.layerNorm){
// WARNING: BatchNormalization instead of layerNormalization
this.firstLayerBatchNorm = tf.layers.batchNormalization({
scale: true,
center: true
});
}
// Second layer with BatchNormalization
this.secondLayer = tf.layers.dense({
//inputShape: [this.config.batchSize, 64 + this.nbActions], // Previous layer + action
units: 64,
units: 32,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
activation: 'linear', // relu is add later
activation: 'relu',
useBias: true,
biasInitializer: "zeros"
});
if (this.layerNorm){
// WARNING: BatchNormalization instead of layerNormalization
this.secondLayerBatchNorm = tf.layers.batchNormalization({
scale: true,
center: true
});
}
// Ouput layer
this.outputLayer = tf.layers.dense({
@@ -245,22 +211,21 @@ class Critic {
});
// Actor prediction
this.predict = () => {
this.predict = (tfState, tfActions) => {
return tf.tidy(() => {
let l1 = this.firstLayer.apply(obs);
l1 = this.concat.apply([l1, action]);
if (this.firstLayerBatchNorm){
l1 = this.firstLayerBatchNorm.apply(l1);
if (tfState && tfActions){
obs = tfState;
action = tfActions;
}
l1 = this.relu1.apply(l1);
let l1A = this.firstLayerA.apply(action);
let l1S = this.firstLayerS.apply(obs)
let l2 = this.secondLayer.apply(l1);
if (this.secondLayerBatchNorm){
l2 = this.secondLayerBatchNorm.apply(l2);
}
l2 = this.relu2.apply(l2);
let concat = this.add.apply([l1A, l1S])
return this.outputLayer.apply(l2);
let l2 = this.secondLayer.apply(concat);
return this.outputLayer.apply(l2);
});
}
const output = this.predict();
demo/webapp/public/js/DDPG/noise.js
+2 -2
View File
@@ -16,8 +16,8 @@ class AdaptiveParamNoiseSpec {
*/
constructor(conf){
conf = conf || {};
this.initialStddev = conf.initialStddev || 0.1;
this.desiredActionStddev = conf.initialStddev || 0.1;
this.initialStddev = conf.initialStddev || 0.3;
this.desiredActionStddev = conf.desiredActionStddev || 0.3;
this.adoptionCoefficient = conf.adoptionCoefficient || 1.01;
this.currentStddev = this.initialStddev;
}
demo/webapp/public/js/editor.js
+1 -1
View File
@@ -16,5 +16,5 @@ editor.load().then(() => {
localStorage.setItem('mylevel.json', JSON.stringify(content));
window.open("/test_editor.html");
}, {download: false, name: "level.json"});
}, {download: true, name: "level.json"});
});
src/asset_manager.ts
+6 -1
View File
@@ -140,6 +140,9 @@ export class AssetManger {
}
car.line = line;
car.a = 0;
car.v = 0;
// Set the car on the road
car.x = road.x;
car.y = road.y;
@@ -158,7 +161,6 @@ export class AssetManger {
let line_side_factor_t = line == 0 ? 0:Math.PI;
car.x += line_side_factor*Math.round(x_m);
car.y += line_side_factor*Math.round(y_m);
// (x, y position) Relatif to the map
car.mx = Math.floor(car.x / ROADSIZE);
car.my = Math.floor(car.y / ROADSIZE);
@@ -166,6 +168,9 @@ export class AssetManger {
car.rotation = (th+line_side_factor_t);
// Set the new road of the car
car.checkAndsetNewRoad(road);
}
createAsset(img: string, info: AssetInfo, textures: any, type: string){
src/car.ts
-2
View File
@@ -186,8 +186,6 @@ export class Car {
Method used to restore the position of the car
to the original position (as set into the json file)
*/
this.core.a = 0;
this.core.v = 0;
this.core.mx = this.info.mx;
this.core.my = this.info.my;
let road = this.level.getRoad(this.core.my, this.core.mx);
src/control_motion_engine.ts
+3 -3
View File
@@ -10,7 +10,7 @@ import * as U from "./utils";
export class ControlMotionEngine extends MotionEngine {
private actions: (string|number)[];
private maxSpeed: number = 1.5;
private maxSpeed: number = 2.0;
constructor(level: Level) {
/*
@@ -144,12 +144,12 @@ export class ControlMotionEngine extends MotionEngine {
this.car.v = Math.min(0, this.car.v + 0.01);
if (this.car.a > 0 && this.car.v >= 0)
this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.01);
this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.005);
else if (this.car.a > 0 && this.car.v < 0){
this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03);
}
if (this.car.a < 0 && this.car.v <= 0)
this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.01);
this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.005);
else if (this.car.a < 0 && this.car.v > 0){
this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03);
}
src/embedded/level/level_2.ts
+304 -99
View File
@@ -1,114 +1,319 @@
export const level2: any = {
"cars": [
"cars": [],
"road": [
{
"mx": 7,
"my": 7,
"line": 0
"x": 300,
"y": 360
},
{
"mx": 9,
"my": 6,
"line": 0
"x": 300,
"y": 360
},
{
"mx": 8,
"my": 6,
"line": 1
"x": 360,
"y": 360
},
{
"mx": 10,
"my": 6,
"line": 1
"x": 420,
"y": 360
},
{
"x": 240,
"y": 360
},
{
"x": 180,
"y": 360
},
{
"x": 120,
"y": 360
},
{
"x": 480,
"y": 360
},
{
"x": 480,
"y": 360
},
{
"x": 120,
"y": 360
},
{
"x": 60,
"y": 360
},
{
"x": 300,
"y": 300
},
{
"x": 300,
"y": 240
},
{
"x": 480,
"y": 360
},
{
"x": 480,
"y": 360
},
{
"x": 540,
"y": 360
},
{
"x": 300,
"y": 180
},
{
"x": 360,
"y": 180
},
{
"x": 240,
"y": 180
},
{
"x": 240,
"y": 180
},
{
"x": 180,
"y": 180
},
{
"x": 420,
"y": 180
},
{
"x": 420,
"y": 120
},
{
"x": 180,
"y": 120
},
{
"x": 300,
"y": 60
},
{
"x": 420,
"y": 60
},
{
"x": 180,
"y": 60
},
{
"x": 240,
"y": 60
},
{
"x": 360,
"y": 60
},
{
"x": 480,
"y": 60
},
{
"x": 120,
"y": 60
},
{
"x": 540,
"y": 60
},
{
"x": 540,
"y": 120
},
{
"x": 540,
"y": 180
},
{
"x": 540,
"y": 240
},
{
"x": 540,
"y": 300
},
{
"x": 60,
"y": 60
},
{
"x": 60,
"y": 120
},
{
"x": 60,
"y": 180
},
{
"x": 60,
"y": 240
},
{
"x": 60,
"y": 300
},
{
"x": 60,
"y": 240
},
{
"x": 360,
"y": 180
},
{
"x": 480,
"y": 360
}
],
"asset": [
{
"x": 162,
"y": 264
},
{
"x": 407,
"y": 264
},
{
"x": 259,
"y": 126
},
{
"x": 306,
"y": 33
},
{
"x": 491,
"y": 148
},
{
"x": 391,
"y": 431
},
{
"x": 608,
"y": 35
},
{
"x": 484,
"y": 224
},
{
"x": 248,
"y": 297
},
{
"x": 376,
"y": 336
},
{
"x": 78,
"y": 429
},
{
"x": 1,
"y": 205
},
{
"x": 3,
"y": 106
},
{
"x": -46,
"y": 290
},
{
"x": 622,
"y": 154
},
{
"x": 617,
"y": 280
}
],
"house": [
{
"x": 68,
"y": 317
}
],
"house2": [
{
"x": 624,
"y": 84
"x": 162,
"y": 264
},
{
"x": -15,
"y": 144
"x": 622,
"y": 154
}
],
"house3": [
{
"x": 618,
"y": 231
"x": 407,
"y": 264
},
{
"x": 13,
"y": 441
}
],
"bench": [
{
"x": 318,
"y": 216
},
{
"x": 422,
"y": 36
"x": -46,
"y": 290
}
],
"tree": [
{
"x": 151,
"y": 144
"x": 259,
"y": 126
},
{
"x": 229,
"y": 175
"x": 491,
"y": 148
},
{
"x": 326,
"y": 132
"x": 391,
"y": 431
},
{
"x": 413,
"y": 167
"x": 608,
"y": 35
},
{
"x": 487,
"y": 157
"x": 484,
"y": 224
},
{
"x": 445,
"y": 123
"x": 248,
"y": 297
},
{
"x": 511,
"x": 78,
"y": 429
},
{
"x": 584,
"y": 427
"x": 1,
"y": 205
},
{
"x": 80,
"y": 391
"x": 3,
"y": 106
},
{
"x": 219,
"y": 409
"x": 617,
"y": 280
}
],
"bench": [
{
"x": 306,
"y": 33
},
{
"x": 207,
"y": 326
},
{
"x": 156,
"y": 426
},
{
"x": 162,
"y": 13
"x": 376,
"y": 336
}
],
"map": [
@@ -129,11 +334,11 @@ export const level2: any = {
0,
"↱",
"↔",
"↧",
"↔",
"↔",
"↔",
"↔",
"↔",
"↧",
"↔",
"↰",
0
@@ -142,9 +347,35 @@ export const level2: any = {
0,
"↕",
0,
"↕",
0,
0,
0,
"↕",
0,
"↕",
0
],
[
0,
"↕",
0,
"↳",
"↔",
"↧",
"↠",
"↲",
0,
"↕",
0
],
[
0,
"↟",
0,
0,
0,
"↕",
0,
0,
0,
@@ -157,7 +388,7 @@ export const level2: any = {
0,
0,
0,
0,
"↕",
0,
0,
0,
@@ -170,10 +401,10 @@ export const level2: any = {
"↔",
"↔",
"↔",
"↧",
"↔",
"↥",
"↔",
"↔",
"↠",
"↲",
0
],
@@ -183,43 +414,17 @@ export const level2: any = {
0,
0,
0,
"↕",
0,
0,
0,
0,
0
],
[
0,
0,
0,
0,
0,
"↕",
0,
"↱",
"↔",
"↔",
"↔"
],
[
0,
0,
0,
0,
0,
"↕",
0,
"↕",
0,
0,
0
]
],
"agent": {
"mx": 2,
"my": 4,
"mx": 4,
"my": 3,
"line": 0,
"motion": {
"type": "BasicMotionEngine",
src/level.ts
+2 -1
View File
@@ -110,7 +110,8 @@ export class Level extends World {
}
setReward(agent_col: any, on_road: any, action: any){
let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed;
let reward = 0;
//let reward = -0.8 + this.agent.core.v / this.agent.motion.maxSpeed;
if (agent_col.length > 0){
reward = -1;
}