wassname/rl_2d_walker.js
1
0
Fork 0
mirror of https://github.com/wassname/rl_2d_walker.js.git synced 2026-09-09 11:33:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
wassname
2018-12-02 16:00:19 +08:00
parent 90a5b66042
commit bfb075cc3d
16 changed files with 2638 additions and 2003 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+744 -1232
View File
File diff suppressed because it is too large Load Diff
scripts/view_log.ipynb
+1439 -363
View File
File diff suppressed because one or more lines are too long
src/js/charts.js
+30 -20
View File
@@ -1,14 +1,12 @@
var Charts = function() {
var Charts = function () {
this.__constructor.apply(this, arguments);
}
Charts.prototype.__constructor = function () {
}
Charts.prototype.collect = function (agents, n, chunkSize) {
Charts.prototype.__constructor = function () {}
Charts.prototype.collect = function (agents, n, chunkSize) {
if (n === undefined) n = 1
if (chunkSize===undefined) chunkSize=1
if (chunkSize === undefined) chunkSize = 1
var data = {}
// collect data
@@ -16,12 +14,14 @@ Charts.prototype.collect = function (agents, n, chunkSize) {
for (const key of keys) {
if (key === 'x') continue
data[key] = []
for (let i = 0; i < agents.length; i+=n) {
for (let i = 0; i < agents.length; i += n) {
const infos = agents[i].infos;
// var borderColor = "hsl("+agents[i].walkerhue+",45%,"+(100-15*agents[i].walker.health/config.walker_health)+"%)";
// build dataset
var dataset = {
label: 'Agent ' + i, data: [], fill: false,
label: 'Agent ' + i,
data: [],
fill: false,
// borderColor
}
for (const info of infos) {
@@ -33,9 +33,12 @@ Charts.prototype.collect = function (agents, n, chunkSize) {
}
// take means?
var leftOver = dataset.data%chunkSize
var leftOver = dataset.data % chunkSize
dataset.data = _.chunk(dataset.data, 4)
.map(c => ({ x: _.mean(_.map(c, 'x')), y: _.mean(_.map(c, 'y')) }))
.map(c => ({
x: _.mean(_.map(c, 'x')),
y: _.mean(_.map(c, 'y'))
}))
if (leftOver) dataset.data.pop()
data[key].push(dataset)
@@ -44,26 +47,31 @@ Charts.prototype.collect = function (agents, n, chunkSize) {
for (let i = 0; i < agents.length; i++) {
agents[i].infos = [] // empty it
}
return data
}
Charts.prototype.init = function (agents) {
var data = this.collect(agents, 1, 10)
var div = document.getElementById('charts');
// make charts
this.charts = []
for (const key in data) {
var canvas = document.createElement("canvas");
var canvas = document.createElement("canvas");
div.appendChild(canvas)
var ctx = canvas.getContext('2d');
var lineChart = new Chart(ctx, {
type: 'scatter',
data: { datasets: data[key] },
data: {
datasets: data[key]
},
options: {
title: { text: key, display: true },
title: {
text: key,
display: true
},
scales: {
xAxes: [{
type: 'linear',
@@ -71,7 +79,7 @@ Charts.prototype.init = function (agents) {
}]
}
}
});
});
this.charts.push(lineChart)
}
@@ -83,12 +91,14 @@ Charts.prototype.update = function (agents) {
for (const chart of this.charts) {
var newDatasets = data[chart.config.options.title.text]
chart.data.datasets.forEach((dataset) => {
var dat = newDatasets.filter(d=>d.label==dataset.label)[0].data
var dat = newDatasets.filter(d => d.label == dataset.label)[0].data
dataset.data.push(...dat);
if (dataset.data.length>maxLen) dataset.data.splice(0, dataset.data.length-maxLen)
if (dataset.data.length > maxLen) dataset.data.splice(0, dataset.data.length - maxLen)
});
chart.update();
}
}
module.exports ={Charts}
module.exports = {
Charts
}
src/js/config.js
+3 -4
View File
@@ -12,7 +12,6 @@ module.exports = {
max_floor_tiles: 50,
round_length: 1000,
min_body_delta: 0,
min_leg_delta: 0.0,
action_repeat: 4,
};
min_leg_delta: 0.0,
action_repeat: 4,
};
src/js/ddpg/ddpg.js
+95 -51
View File
@@ -1,12 +1,20 @@
const { tf } = require('./tf_import')
const { copyModel, Actor, Critic, assignAndStd, targetUpdate } = require('./models')
const {
tf
} = require('./tf_import')
const {
copyModel,
Actor,
Critic,
assignAndStd,
targetUpdate
} = require('./models')
function logTfMemory(){
function logTfMemory() {
let mem = tf.memory();
console.log("numBytes:" + mem.numBytes +
"\nnumBytesInGPU:" + mem.numBytesInGPU +
"\nnumDataBuffers:" + mem.numDataBuffers +
"\nnumTensors:" + mem.numTensors);
console.log("numBytes:" + mem.numBytes +
"\nnumBytesInGPU:" + mem.numBytesInGPU +
"\nnumDataBuffers:" + mem.numDataBuffers +
"\nnumTensors:" + mem.numTensors);
}
// This class is called from js/DDPG/ddpg_agent.js
@@ -19,7 +27,7 @@ class DDPG {
* @param memory (Memory class)
* @param noise (Noise class)
*/
constructor(actor, critic, memory, noise, config){
constructor(actor, critic, memory, noise, config) {
this.actor = actor;
this.critic = critic;
this.memory = memory;
@@ -28,15 +36,19 @@ class DDPG {
this.tfGamma = tf.scalar(config.gamma);
// Inputs
this.obsInput = tf.input({batchShape: [null, this.config.stateSize]});
this.actionInput = tf.input({batchShape: [null, this.config.nbActions]});
this.obsInput = tf.input({
batchShape: [null, this.config.stateSize]
});
this.actionInput = tf.input({
batchShape: [null, this.config.nbActions]
});
// Randomly Initialize actor network μ(s)
this.actor.buildModel(this.obsInput);
// Randomly Initialize critic network Q(s, a)
this.critic.buildModel(this.obsInput, this.actionInput);
// Define in js/DDPG/models.js
// Init target network Q' and μ' with the same weights
this.actorTarget = copyModel(this.actor, Actor);
@@ -48,7 +60,7 @@ class DDPG {
this.setLearningOp();
}
setLearningOp(){
setLearningOp() {
this.criticWithActor = (tfState) => {
return tf.tidy(() => {
const tfAct = this.actor.predict(tfState);
@@ -66,11 +78,11 @@ class DDPG {
this.criticOptimiser = tf.train.adam(this.config.criticLr);
this.criticWeights = [];
for (let w = 0; w < this.critic.model.trainableWeights.length; w++){
for (let w = 0; w < this.critic.model.trainableWeights.length; w++) {
this.criticWeights.push(this.critic.model.trainableWeights[w].val);
}
this.actorWeights = [];
for (let w = 0; w < this.actor.model.trainableWeights.length; w++){
for (let w = 0; w < this.actor.model.trainableWeights.length; w++) {
this.actorWeights.push(this.actor.model.trainableWeights[w].val);
}
@@ -83,7 +95,7 @@ class DDPG {
* See parameter space noise Exploration paper
* @param observations (Tensor2d) Observations
*/
distanceMeasure(observations) {
distanceMeasure(observations) {
return tf.tidy(() => {
const pertubedPredictions = this.perturbedActor.model.predict(observations);
const predictions = this.actor.model.predict(observations);
@@ -96,21 +108,21 @@ class DDPG {
/**
* AdaptParamNoise
*/
adaptParamNoise(){
adaptParamNoise() {
const batch = this.memory.getBatch(this.config.batchSize);
if (batch.obs0.length == 0){
if (batch.obs0.length == 0) {
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
return [0];
}
let distanceV = null;
if (batch.obs0.length > 0){
if (batch.obs0.length > 0) {
const tfObs0 = tf.tensor2d(batch.obs0);
const distance = this.distanceMeasure(tfObs0);
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
distanceV = distance.buffer().values;
this.noise.adapt(distanceV[0]);
@@ -127,7 +139,7 @@ class DDPG {
* @param state number[]
* @param action [a, steering]
*/
getQvalue(state, a){
getQvalue(state, a) {
const st = tf.tensor2d([state]);
const tfa = tf.tensor2d([a]);
const q = this.critic.model.predict([st, tfa]);
@@ -142,7 +154,7 @@ class DDPG {
* @param observation (tf.tensor2d)
* @return (tf.tensor1d)
*/
predict(observation){
predict(observation) {
const tfActions = this.actor.model.predict(observation);
return tfActions;
}
@@ -151,7 +163,7 @@ class DDPG {
* @param observation (tf.tensor2d)
* @return (tf.tensor1d)
*/
perturbedPrediction(observation){
perturbedPrediction(observation) {
const tfActions = this.perturbedActor.model.predict(observation);
return tfActions;
}
@@ -159,26 +171,26 @@ class DDPG {
/**
* Update the two target network
*/
targetUpdate(){
targetUpdate() {
// Define in js/DDPG/models.js
targetUpdate(this.criticTarget, this.critic, this.config);
targetUpdate(this.actorTarget, this.actor, this.config);
}
trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){
let costs;
trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals) {
let costs;
const criticLoss = this.criticOptimiser.minimize(() => {
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1);
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1));
const erros = tf.sub(tfQTargets, tfQPredictions0).square();
costs = erros.buffer().values;
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1);
return erros.mean();
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1));
const erros = tf.sub(tfQTargets, tfQPredictions0).square();
costs = erros.buffer().values;
return erros.mean();
}, true, this.criticWeights);
// For experience Replay
@@ -192,22 +204,22 @@ class DDPG {
return loss;
}
trainActor(tfObs0){
trainActor(tfObs0) {
const actorLoss = this.actorOptimiser.minimize(() => {
const tfQPredictions0 = this.criticWithActor(tfObs0);
const tfQPredictions0 = this.criticWithActor(tfObs0);
return tf.mean(tfQPredictions0).mul(tf.scalar(-1.))
}, true, this.actorWeights);
targetUpdate(this.actorTarget, this.actor, this.config);
const loss = actorLoss.buffer().values[0];
actorLoss.dispose();
actorLoss.dispose();
return loss;
}
getTfBatch(){
getTfBatch() {
// Get batch
const batch = this.memory.popBatch(this.config.batchSize);
@@ -225,52 +237,84 @@ class DDPG {
_tfTerminals.dispose();
return {
batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals
batch,
tfActions,
tfObs0,
tfObs1,
tfRewards,
tfTerminals
}
}
optimizeCritic(){
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
optimizeCritic() {
const {
batch,
tfActions,
tfObs0,
tfObs1,
tfRewards,
tfTerminals
} = this.getTfBatch();
const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return loss;
}
optimizeActor(){
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
optimizeActor() {
const {
batch,
tfActions,
tfObs0,
tfObs1,
tfRewards,
tfTerminals
} = this.getTfBatch();
const loss = this.trainActor(tfObs0);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return loss;
}
optimizeCriticActor(){
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
optimizeCriticActor() {
const {
batch,
tfActions,
tfObs0,
tfObs1,
tfRewards,
tfTerminals
} = this.getTfBatch();
const lossC = this.trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
const lossA = this.trainActor(tfObs0);
tfActions.dispose();
tfObs0.dispose();
tfObs1.dispose();
tfObs1.dispose();
tfRewards.dispose();
tfTerminals.dispose();
return {lossC, lossA};
return {
lossC,
lossA
};
}
}
module.exports = { logTfMemory, DDPG }
module.exports = {
logTfMemory,
DDPG
}
src/js/ddpg/ddpg_agent.js
+66 -43
View File
@@ -1,12 +1,24 @@
const { tf } = require('./tf_import')
const {
tf
} = require('./tf_import')
const AdaptiveParamNoiseSpec = require('./noise')
const PrioritizedMemory = require('./prioritized_memory')
const { Actor, Critic, copyFromSave, copyModel} = require('./models')
const { DDPG, logTfMemory } = require('./ddpg')
const { mean } = require('../utils')
const {
Actor,
Critic,
copyFromSave,
copyModel
} = require('./models')
const {
DDPG,
logTfMemory
} = require('./ddpg')
const {
mean
} = require('../utils')
function setMetric(name, value) {
function setMetric(name, value) {
console.debug('metric', name, value)
}
@@ -17,7 +29,7 @@ class DDPGAgent {
/**
* @param env (metacar.env) Set in js/DDPG/index.js
*/
constructor(env, config){
constructor(env, config) {
this.stopTraining = false;
this.env = env;
@@ -30,7 +42,7 @@ class DDPGAgent {
"seed": config.seed || 0,
"batchSize": config.batchSize || 128,
"actorLr": config.actorLr || 0.0001,
"criticLr": config .criticLr || 0.001,
"criticLr": config.criticLr || 0.001,
"memorySize": config.memorySize || 30000,
"gamma": config.gamme || 0.99,
"noiseDecay": config.noiseDecay || 0.99,
@@ -48,10 +60,10 @@ class DDPGAgent {
"criticFirstLayerASize": config.criticFirstLayerASize || 64,
"criticSecondLayerSize": config.criticSecondLayerSize || 32,
"maxStep": config.maxStep || 800,
"stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError:true,
"resetEpisode": config.resetEpisode != undefined ? config.resetEpisode:false,
"stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError : true,
"resetEpisode": config.resetEpisode != undefined ? config.resetEpisode : false,
"saveDuringTraining": config.saveDuringTraining || false,
"saveInterval": config.saveInterval || 20
"saveInterval": config.saveInterval || 20
};
this.epoch = 0;
// From js/DDPG/noise.js
@@ -77,7 +89,7 @@ class DDPGAgent {
this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config);
}
save(name){
save(name) {
/*
Save the network
*/
@@ -85,22 +97,22 @@ class DDPGAgent {
this.ddpg.critic.model.save('file://./outputs/critic-' + name);
this.ddpg.actor.model.save('file://./outputs/actor-' + name);
} else {
this.ddpg.actor.model.save('downloads://actor-'+ name);
this.ddpg.actor.model.save('downloads://actor-' + name);
this.ddpg.critic.model.save('downloads://critic-' + name);
}
}
async restore(folder, name){
async restore(folder, name) {
/*
Restore the weights of the network
*/
var critic, actor
var critic, actor
if (typeof WEB === "undefined") {
critic = await tf.loadModel('file://'+folder+'/critic-'+name+'.json');
actor = await tf.loadModel("file://"+folder+"/actor-"+name+".json");
critic = await tf.loadModel('file://' + folder + '/critic-' + name + '.json');
actor = await tf.loadModel("file://" + folder + "/actor-" + name + ".json");
} else {
critic = await tf.loadModel(window.location.href+folder+'/critic-'+name+'.json');
actor = await tf.loadModel(window.location.href+folder+"/actor-"+name+".json");
critic = await tf.loadModel(window.location.href + folder + '/critic-' + name + '.json');
actor = await tf.loadModel(window.location.href + folder + "/actor-" + name + ".json");
// const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
// const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
}
@@ -121,7 +133,7 @@ class DDPGAgent {
/**
* Play one step
*/
play(){
play() {
// Get the current state
const state = this.env.getState();
// Pick an action
@@ -137,11 +149,11 @@ class DDPGAgent {
* @param state number[]
* @param action [a, steering]
*/
getQvalue(state, a){
getQvalue(state, a) {
return this.ddpg.getQvalue(state, a);
}
stop(){
stop() {
this.stopTraining = true;
}
@@ -151,7 +163,7 @@ class DDPGAgent {
* @param mPreviousStep number[]
* @return {done, state} One boolean and the new state
*/
stepTrain(tfPreviousStep, mPreviousStep){
stepTrain(tfPreviousStep, mPreviousStep) {
// Get actions
const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
// Step in the environment with theses actions
@@ -161,7 +173,7 @@ class DDPGAgent {
this.infoList.push(info);
// Get the new observations
let tfState = tf.tensor2d([mState]);
if (mReward == -1 && this.config.stopOnRewardError){
if (mReward == -1 && this.config.stopOnRewardError) {
mDone = 1;
}
// Add the new tuple to the buffer
@@ -170,19 +182,26 @@ class DDPGAgent {
tfPreviousStep.dispose();
tfActions.dispose();
return {mDone, mState, tfState};
return {
mDone,
mState,
tfState
};
}
/**
* Optimize models and log states
*/
_optimize(){
_optimize() {
this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
let lossValuesCritic = [];
let lossValuesActor = [];
console.time("Training");
for (let t=0; t < this.config.nbTrainSteps; t++){
let {lossC, lossA} = this.ddpg.optimizeCriticActor();
for (let t = 0; t < this.config.nbTrainSteps; t++) {
let {
lossC,
lossA
} = this.ddpg.optimizeCriticActor();
lossValuesCritic.push(lossC);
lossValuesActor.push(lossA);
}
@@ -195,31 +214,33 @@ class DDPGAgent {
/**
* Train DDPG Agent
*/
async train(realTime){
async train(realTime) {
this.stopTraining = false;
// One epoch
for (this.epoch; this.epoch < this.config.nbEpochs; this.epoch++){
for (this.epoch; this.epoch < this.config.nbEpochs; this.epoch++) {
// Perform cycles.
this.rewardsList = [];
this.stepList = [];
this.distanceList = [];
// document.getElementById("trainingProgress").innerHTML = "Progression: "+this.epoch+"/"+this.config.nbEpochs+"<br>";
console.log("Progression: "+this.epoch+"/"+this.config.nbEpochs+" epochs")
for (let c=0; c < this.config.nbEpochsCycle; c++){
if (c%10==0){ logTfMemory(); }
console.log("Progression: " + this.epoch + "/" + this.config.nbEpochs + " epochs")
for (let c = 0; c < this.config.nbEpochsCycle; c++) {
if (c % 10 == 0) {
logTfMemory();
}
let mPreviousStep = this.env.getState();
let tfPreviousStep = tf.tensor2d([mPreviousStep]);
let step = 0;
console.time("LoopTime");
for (step=0; step < this.config.maxStep; step++){
for (step = 0; step < this.config.maxStep; step++) {
let rel = this.stepTrain(tfPreviousStep, mPreviousStep);
mPreviousStep = rel.mState;
tfPreviousStep = rel.tfState;
if (rel.mDone && this.config.stopOnRewardError){
if (rel.mDone && this.config.stopOnRewardError) {
break;
}
if (this.stopTraining){
if (this.stopTraining) {
this.env.render(true);
return;
}
@@ -231,19 +252,21 @@ class DDPGAgent {
let distance = this.ddpg.adaptParamNoise();
this.distanceList.push(distance[0]);
if (this.config.resetEpisode){
if (this.config.resetEpisode) {
this.env.reset();
}
this.env.shuffle({cars: false});
this.env.shuffle({
cars: false
});
tfPreviousStep.dispose();
console.log("e="+ this.epoch +", c="+c);
console.log("e=" + this.epoch + ", c=" + c);
await tf.nextFrame();
}
if (this.epoch > 5){
if (this.epoch > 5) {
this._optimize();
}
if (this.config.saveDuringTraining && this.epoch % this.config.saveInterval == 0 && this.epoch != 0){
if (this.config.saveDuringTraining && this.epoch % this.config.saveInterval == 0 && this.epoch != 0) {
this.save("model-ddpg-walker-epoch-" + this.epoch);
this.save("model-ddpg-walker");
}
@@ -251,7 +274,7 @@ class DDPGAgent {
let vals = this.infoList.map(info => info[name])
if (vals.length) {
let meanVal = mean(vals)
setMetric(name, mean(vals));
setMetric(name, mean(vals));
} else {
console.log('WARN: empty metric', name)
}
@@ -260,7 +283,7 @@ class DDPGAgent {
setMetric("EpisodeDuration", mean(this.stepList));
setMetric("NoiseDistance", mean(this.distanceList));
await tf.nextFrame();
}
}
this.env.render(true);
}
src/js/ddpg/memory.js
-86
View File
@@ -1,86 +0,0 @@
// class Memory {
// /**
// * @param maxlen (number) Buffer limit
// */
// constructor(maxlen){
// this.maxlen = maxlen;
// this.length = 0;
// this.start = 0;
// this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// }
// /**
// * @param idx (number)
// */
// getItem(idx){
// if (idx < 0 || idx >= this.length){
// console.error("Memory.getItem: idx not in range.");
// }
// return this.data[(this.start + idx) % this.maxlen]
// }
// /**
// * Sample a batch
// * @param batchSize (number)
// * @return batch []
// */
// getBatch(batchSize){
// const arrLength = this.length;
// const batch = {
// 'obs0': [],
// 'obs1': [],
// 'rewards': [],
// 'actions': [],
// 'terminals': [],
// };
// if (batchSize > this.length){
// return batch;
// }
// for (let b=0; b < batchSize; b++){
// let id = Math.floor(Math.random() * arrLength);
// batch.obs0.push(this.obs0List[id]);
// batch.obs1.push(this.obs1List[id]);
// batch.rewards.push(this.rewardsList[id]);
// batch.actions.push(this.actionsList[id]);
// batch.terminals.push(this.terminals1List[id]);
// }
// return batch
// }
// /**
// * @param obs0 []
// * @param action (number)
// * @param reward (number)
// * @param obs1 []
// * @param terminal1 (boolean)
// */
// append(obs0, action, reward, obs1, terminal1){
// if (this.length < this.maxlen){
// this.length += 1;
// }
// else if (this.length == this.maxlen) {
// //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
// //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
// //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
// this.start = (this.start + 1) % this.maxlen;
// }
// else {
// console.error("Memory.append: This should never be printed");
// }
// this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
// this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
// this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
// this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
// this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
// }
// }
// module.exports = Memory
src/js/ddpg/models.js
+63 -32
View File
@@ -1,4 +1,6 @@
const { tf } = require('./tf_import')
const {
tf
} = require('./tf_import')
/**
* Copy a model
@@ -6,13 +8,13 @@ const { tf } = require('./tf_import')
* @param instance Actor|Critic
* @return Copy of the model
*/
function copyFromSave(model, instance, config, obs, action){
function copyFromSave(model, instance, config, obs, action) {
return tf.tidy(() => {
nModel = new instance(config);
// action might be not required
nModel.buildModel(obs, action);
const weights = model.weights;
for (let m=0; m < weights.length; m++){
for (let m = 0; m < weights.length; m++) {
nModel.model.weights[m].val.assign(weights[m].val);
}
return nModel;
@@ -26,13 +28,13 @@ function copyFromSave(model, instance, config, obs, action){
* @param instance Actor|Critic
* @return Copy of the model
*/
function copyModel(model, instance){
function copyModel(model, instance) {
return tf.tidy(() => {
nModel = new instance(model.config);
// action might be not required
nModel.buildModel(model.obs, model.action);
const weights = model.model.weights;
for (let m=0; m < weights.length; m++){
for (let m = 0; m < weights.length; m++) {
nModel.model.weights[m].val.assign(weights[m].val);
}
return nModel;
@@ -47,10 +49,10 @@ function copyModel(model, instance){
* @param stddev (number)
* @return Copy of the model
*/
function assignAndStd(model, perturbedModel, stddev, seed){
function assignAndStd(model, perturbedModel, stddev, seed) {
return tf.tidy(() => {
const weights = model.model.trainableWeights;
for (let m=0; m < weights.length; m++){
for (let m = 0; m < weights.length; m++) {
let shape = perturbedModel.model.trainableWeights[m].val.shape;
let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed);
let nValue = weights[m].val.add(randomTensor);
@@ -66,20 +68,20 @@ function assignAndStd(model, perturbedModel, stddev, seed){
* @param config (Object)
* @return Copy of the model
*/
function targetUpdate(target, original, config){
function targetUpdate(target, original, config) {
return tf.tidy(() => {
const originalW = original.model.trainableWeights;
const targetW = target.model.trainableWeights;
const one = tf.scalar(1);
const tau = tf.scalar(config.tau);
for (let m=0; m < originalW.length; m++){
for (let m = 0; m < originalW.length; m++) {
const lastValue = target.model.trainableWeights[m].val.clone();
let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau)));
target.model.trainableWeights[m].val.assign(nValue);
const diff = lastValue.sub(target.model.trainableWeights[m].val).mean().buffer().values;
if (diff[0] == 0){
if (diff[0] == 0) {
console.warn("targetUpdate: Nothing have been changed!")
}
}
@@ -87,7 +89,7 @@ function targetUpdate(target, original, config){
}
class Actor{
class Actor {
/**
@param config (Object)
@@ -109,13 +111,15 @@ class Actor{
*
* @param obs tf.input
*/
buildModel(obs){
buildModel(obs) {
this.obs = obs;
// First layer
this.firstLayer = tf.layers.dense({
units: this.firstLayerSize,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
kernelInitializer: tf.initializers.glorotUniform({
seed: this.seed
}),
activation: 'relu',
useBias: true,
biasInitializer: "zeros"
@@ -123,7 +127,9 @@ class Actor{
// Second layer
this.secondLayer = tf.layers.dense({
units: this.secondLayerSize,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
kernelInitializer: tf.initializers.glorotUniform({
seed: this.seed
}),
activation: 'relu',
useBias: true,
biasInitializer: "zeros"
@@ -132,15 +138,18 @@ class Actor{
this.outputLayer = tf.layers.dense({
units: this.nbActions,
kernelInitializer: tf.initializers.randomUniform({
minval: 0.003, maxval: 0.003, seed: this.seed}),
minval: 0.003,
maxval: 0.003,
seed: this.seed
}),
activation: 'tanh',
useBias: true,
biasInitializer: "zeros"
});
// Actor prediction
this.predict = (tfState) => {
return tf.tidy(() => {
if (tfState){
return tf.tidy(() => {
if (tfState) {
obs = tfState;
}
@@ -151,7 +160,10 @@ class Actor{
});
}
const output = this.predict();
this.model = tf.model({inputs: obs, outputs: output});
this.model = tf.model({
inputs: obs,
outputs: output
});
}
};
@@ -180,7 +192,7 @@ class Critic {
* @param obs tf.input
* @param action tf.input
*/
buildModel(obs, action){
buildModel(obs, action) {
this.obs = obs;
this.action = action;
@@ -190,7 +202,9 @@ class Critic {
// First layer
this.firstLayerS = tf.layers.dense({
units: this.firstLayerSSize,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
kernelInitializer: tf.initializers.glorotUniform({
seed: this.seed
}),
activation: 'linear', // relu is add later
useBias: true,
biasInitializer: "zeros"
@@ -198,7 +212,9 @@ class Critic {
// First layer
this.firstLayerA = tf.layers.dense({
units: this.firstLayerASize,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
kernelInitializer: tf.initializers.glorotUniform({
seed: this.seed
}),
activation: 'linear', // relu is add later
useBias: true,
biasInitializer: "zeros"
@@ -206,7 +222,9 @@ class Critic {
// Second layer
this.secondLayer = tf.layers.dense({
units: this.secondLayerSize,
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
kernelInitializer: tf.initializers.glorotUniform({
seed: this.seed
}),
activation: 'relu',
useBias: true,
biasInitializer: "zeros"
@@ -216,34 +234,47 @@ class Critic {
this.outputLayer = tf.layers.dense({
units: 1,
kernelInitializer: tf.initializers.randomUniform({
minval: 0.003, maxval: 0.003, seed: this.seed}),
minval: 0.003,
maxval: 0.003,
seed: this.seed
}),
activation: 'linear',
useBias: true,
biasInitializer: "zeros"
});
// Critic prediction
this.predict = (tfState, tfActions) => {
return tf.tidy(() => {
if (tfState && tfActions){
if (tfState && tfActions) {
obs = tfState;
action = tfActions;
}
let l1A = this.firstLayerA.apply(action);
let l1S = this.firstLayerS.apply(obs)
// Merged layers
let concat = this.add.apply([l1A, l1S])
let l2 = this.secondLayer.apply(concat);
return this.outputLayer.apply(l2);
return this.outputLayer.apply(l2);
});
}
const output = this.predict();
this.model = tf.model({inputs: [obs, action], outputs: output});
this.model = tf.model({
inputs: [obs, action],
outputs: output
});
}
};
module.exports = {Actor, Critic, copyFromSave, copyModel, assignAndStd, targetUpdate}
module.exports = {
Actor,
Critic,
copyFromSave,
copyModel,
assignAndStd,
targetUpdate
}
src/js/ddpg/noise.js
+4 -5
View File
@@ -14,7 +14,7 @@ class AdaptiveParamNoiseSpec {
* conf.desiredActionStddev: 0.1 default // δ
* conf.adoptionCoefficient: 1.01 default // α
*/
constructor(conf){
constructor(conf) {
conf = conf || {};
this.initialStddev = conf.initialStddev || 0.4;
this.desiredActionStddev = conf.desiredActionStddev || 0.4;
@@ -26,13 +26,12 @@ class AdaptiveParamNoiseSpec {
* The distance from the Adaptive scaling
* @param distance number
*/
adapt(distance){
adapt(distance) {
// if d(π, _π_) > δ then σ = σ/α
if (distance > this.desiredActionStddev){
if (distance > this.desiredActionStddev) {
// Decrease σ
this.currentStddev /= this.adoptionCoefficient;
}
else{
} else {
// σ = σ*α
// Increase σ
this.currentStddev *= this.adoptionCoefficient;
src/js/ddpg/prioritized_memory.js
+31 -33
View File
@@ -1,10 +1,9 @@
class PrioritizedMemory {
/**
* @param maxlen (number) Buffer limit
*/
constructor(maxlen){
constructor(maxlen) {
this.maxlen = maxlen;
this.buffer = [];
this.priorBuffer = [];
@@ -15,8 +14,8 @@ class PrioritizedMemory {
* @param batchSize (number)
* @return batch []
*/
getBatch(batchSize){
const batch = {
getBatch(batchSize) {
const batch = {
'obs0': [],
'obs1': [],
'rewards': [],
@@ -24,12 +23,12 @@ class PrioritizedMemory {
'terminals': [],
};
if (batchSize > this.priorBuffer.length){
if (batchSize > this.priorBuffer.length) {
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
return batch;
}
for (let b=0; b < batchSize/2; b++){
for (let b = 0; b < batchSize / 2; b++) {
let id = Math.floor(Math.random() * this.priorBuffer.length);
batch.obs0.push(this.priorBuffer[id].obs0);
batch.obs1.push(this.priorBuffer[id].obs1);
@@ -40,8 +39,8 @@ class PrioritizedMemory {
return batch
}
_bufferBatch(batchSize){
const batch = {
_bufferBatch(batchSize) {
const batch = {
'obs0': [],
'obs1': [],
'rewards': [],
@@ -49,7 +48,7 @@ class PrioritizedMemory {
'terminals': [],
};
for (let b=0; b < batchSize/2; b++){
for (let b = 0; b < batchSize / 2; b++) {
let nElem = this.buffer.pop();
batch.obs0.push(nElem.obs0);
batch.obs1.push(nElem.obs1);
@@ -58,7 +57,7 @@ class PrioritizedMemory {
batch.terminals.push(nElem.terminal);
}
for (let b=0; b < batchSize/2; b++){
for (let b = 0; b < batchSize / 2; b++) {
let id = Math.floor(Math.random() * this.buffer.length);
batch.obs0.push(this.buffer[id].obs0);
batch.obs1.push(this.buffer[id].obs1);
@@ -71,8 +70,8 @@ class PrioritizedMemory {
return batch
}
_addRandomBufferBatch(batchSize, batch){
for (let b=0; b < batchSize; b++){
_addRandomBufferBatch(batchSize, batch) {
for (let b = 0; b < batchSize; b++) {
let id = Math.floor(Math.random() * this.buffer.length);
batch.obs0.push(this.buffer[id].obs0);
batch.obs1.push(this.buffer[id].obs1);
@@ -89,40 +88,39 @@ class PrioritizedMemory {
* @param batchSize (number)
* @return batch []
*/
popBatch(batchSize){
popBatch(batchSize) {
let originalBatchSize = batchSize;
let priorBufferBatchSize;
let bufferBatchSize;
if (batchSize % 2 != 0){
if (batchSize % 2 != 0) {
console.warn("Batch size should be a even.")
}
if (this.priorBuffer.length < batchSize/2){
if (this.priorBuffer.length < batchSize / 2) {
//console.log("get full batch from buffer");
const batch = this._bufferBatch(batchSize);
console.assert(batch.obs0.length == batchSize);
return batch;
}
const batch = {
const batch = {
'obs0': [],
'obs1': [],
'rewards': [],
'actions': [],
'terminals': [],
};
if (batchSize > this.length){
if (batchSize > this.length) {
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
return batch;
}
if (this.buffer.length > 0){
if (this.buffer.length > 0) {
//console.log("Get half of prior and other from buffer.");
batchSize = batchSize / 2;
}
else{
} else {
//console.log("Get all from priorBuffer");
}
for (let b=0; b < batchSize; b++){
for (let b = 0; b < batchSize; b++) {
let id = Math.floor(Math.random() * this.priorBuffer.length);
batch.obs0.push(this.priorBuffer[id].obs0);
batch.obs1.push(this.priorBuffer[id].obs1);
@@ -132,7 +130,7 @@ class PrioritizedMemory {
this.priorBuffer.splice(id, 1);
}
if (this.buffer.length > 0){
if (this.buffer.length > 0) {
this._addRandomBufferBatch(batchSize, batch);
}
console.assert(batch.obs0.length == originalBatchSize);
@@ -140,22 +138,22 @@ class PrioritizedMemory {
}
_insert(element, array) {
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null) {
array.unshift(element);
return array;
}
array.splice(this._locationOf(element, array) + 1, 0, element);
return array;
}
_locationOf(element, array, start, end) {
start = start || 0;
end = end || array.length;
var pivot = parseInt(start + (end - start) / 2, 10);
if (end-start <= 1 || array[pivot] === element) return pivot;
if (end - start <= 1 || array[pivot] === element) return pivot;
if (array[pivot].cost != null && array[pivot].cost < element.cost) {
return this._locationOf(element, array, pivot, end);
} else {
@@ -167,9 +165,9 @@ class PrioritizedMemory {
* @param batch (Object) from getBatch()
* @param cost (number) Cost associated with each row of the batch
*/
appendBackWithCost(batch, costs){
for (let b=0; b < batch.obs0.length; b++){
if (this.buffer.length == this.maxlen){
appendBackWithCost(batch, costs) {
for (let b = 0; b < batch.obs0.length; b++) {
if (this.buffer.length == this.maxlen) {
this.buffer.shift();
}
this._insert({
@@ -191,8 +189,8 @@ class PrioritizedMemory {
* @param obs1 []
* @param terminal1 (boolean)
*/
append(obs0, action, reward, obs1, terminal){
if (this.priorBuffer.length == this.maxlen){
append(obs0, action, reward, obs1, terminal) {
if (this.priorBuffer.length == this.maxlen) {
this.priorBuffer.shift();
}
this.priorBuffer.push({
src/js/ddpg/tf_import.js
+4 -4
View File
@@ -1,9 +1,9 @@
const tf = require('@tensorflow/tfjs')
if (typeof WEB ==="undefined"){
if (typeof WEB === "undefined") {
// Load the binding (note you may have to press enter in the terminal for some reason)
require('@tensorflow/tfjs-node-gpu');
require('@tensorflow/tfjs-node'); // seem to need this as well for save?
}
module.exports = { tf }
module.exports = {
tf
}
src/js/floor.js
+4 -4
View File
@@ -15,18 +15,18 @@ function createFloor(world, max_floor_tiles) {
new b2.Vec2(2.5, -0.16)
];
for(var k = 2; k < max_floor_tiles; k++) {
for (var k = 2; k < max_floor_tiles; k++) {
var ratio = k / max_floor_tiles;
// add uneven floor by continuing from the last point, plus some random jittering
edges.push(new b2.Vec2(
edges[edges.length - 1].x + (1 + ratio * Math.random() - ratio / 2),
edges[edges.length - 1].y + (ratio * Math.random() - ratio / 2)));
// edges.push(new b2.Vec2(edges[edges.length-1].x + 1,-0.16));
// edges.push(new b2.Vec2(edges[edges.length-1].x + 1,-0.16));
}
edges.push(new b2.Vec2(edges[edges.length-1].x, edges[edges.length-1].y + 8)); // front wall
edges.push(new b2.Vec2(edges[edges.length - 1].x, edges[edges.length - 1].y + 8)); // front wall
fix_def.shape.CreateChain(edges, edges.length);
body.CreateFixture(fix_def);
return body;
}
module.exports=createFloor
module.exports = createFloor
src/js/game.js
+80 -64
View File
@@ -1,20 +1,28 @@
const config = require('./config')
const { Charts } = require('./charts')
const { randi } = require('./utils')
const {
Charts
} = require('./charts')
const {
randi
} = require('./utils')
const b2 = require('../vendor/jsbox2d')
const createFloor = require('./floor.js')
const DDPGAgent = require('./ddpg/ddpg_agent')
const {
Walker
Walker
} = require('./walker')
if (typeof window !=="undefined")
var requestAnimFrame = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) { window.setTimeout(callback, 1000 / 60); };
if (typeof window !== "undefined")
var requestAnimFrame = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) {
window.setTimeout(callback, 1000 / 60);
};
else
var requestAnimFrame = function (callback) { window.setTimeout(callback, 1000 / 60); };
var requestAnimFrame = function (callback) {
window.setTimeout(callback, 1000 / 60);
};
chooseQoute = function () {
chooseQoute = function () {
var qoutes = [
'Play the funky music, robot',
'The origin of funkd',
@@ -29,22 +37,23 @@ chooseQoute = function () {
'Dance evolution',
'Have you tried turning it off and on again?',
'Eurovision 2050',
'Humans must learn to crawl then walk. Robots break dance then walk',
''
'Humans crawl before they walk. Robots dance before they walk',
'This is not a disco',
'Disco simulator 2100'
]
var qoute = qoutes[randi(0,qoutes.length)]
document.getElementById('page_quote').innerText = '"'+qoute+'"'
var qoute = qoutes[randi(0, qoutes.length)]
document.getElementById('page_quote').innerText = '"' + qoute + '"'
}
class HeadlessGame {
class HeadlessGame {
constructor(config) {
this.config = config
this.initWorld()
}
initWorld() {
var gravity = new b2.Vec2(0, -10)
this.world = new b2.World(gravity)
this.floor = createFloor(this.world, this.config.max_floor_tiles);
@@ -54,40 +63,40 @@ class HeadlessGame {
const stateSize = this.env.bodies.length * 10 + this.env.joints.length * 3
this.agent = new DDPGAgent(this.env, {
stateSize,
nbActions,
resetEpisode: true,
batchSize: 128,
actorLr: 0.0001,
criticLr: 0.001,
memorySize: 30000,
gamma: 0.99,
stateSize,
nbActions,
resetEpisode: true,
batchSize: 128,
actorLr: 0.0001,
criticLr: 0.001,
memorySize: 30000,
gamma: 0.99,
desiredActionStddev: 0.1,
initialStddev: 0.4,
desiredActionStddev: 0.1,
initialStddev: 0.4,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
nbEpochs: 1000,
nbEpochsCycle: 10,
nbTrainSteps: 100,
maxStep: 800,
saveDuringTraining: true,
saveInterval: 20,
nbEpochs: 1000,
nbEpochsCycle: 10,
nbTrainSteps: 100,
maxStep: 800,
saveDuringTraining: true,
saveInterval: 5,
tau: 0.008,
adoptionCoefficient: 1.01,
tau: 0.008,
adoptionCoefficient: 1.01,
});
}
}
class Game extends HeadlessGame {
class Game extends HeadlessGame {
constructor(config) {
super(config)
@@ -95,66 +104,66 @@ class Game extends HeadlessGame {
// this.agent.stop()
// this.agent.env.render(true)
this.agent.restore('../outputs', 'model-ddpg-walker/model')
setInterval(()=>this.agent.play(), 100)
setInterval(() => this.agent.play(), 100)
// drawInit();
// this.step_counter = 0;
// this.display_interval = setInterval(displayProgress, Math.round(380 * 1000 / config.draw_fps));
// this.charts_interval = setInterval(updateCharts, Math.round(380 * 1000 / config.draw_fps));
// this.running = true
// requestAnimFrame(loop)
}
displayProgress() {
displayProgress() {
var stats = {
'trainingTime': this.step_counter / config.simulation_fps,
'meanProgress': this.walkers.map(w => w.last_position).reduce((s, v) => s + v) / this.walkers.length,
'meanReward': this.walkers.map(w => w.reward).reduce((s, v) => s + v) / this.walkers.length,
'bufferSize': this.agents[0].brain.buffer.size
}
}
document.getElementById('stats-prog').innerText = JSON.stringify(stats, null, 2)
}
// resetSimulation() {
// // turn training off temporarlity to avoid NaN's
// updateIfLearning(false)
// // this.running = false
// this.world.Destroy() // this way we get rid of listeners and body parts and joints
// this.world = new b2.World(new b2.Vec2(0, -10));
// this.floor = createFloor(this.world);
// for (var k = 0; k < config.population_size; k++) {
// this.agents[k].walker = this.walkers[k] = new Walker(this.world, this.floor)
// }
// // this.running = true
// setTimeout(() => updateIfLearning(true), 1000)
// // setTimeout(() => requestAnimFrame(loop), 1000)
// }
// loop() {
// drawFrame()
// simulationStep()
// drawFrame()
// if (this.running) requestAnimFrame(loop); // start next timer
// }
updateCharts() {
updateCharts() {
var groupN = 100
var maxN = 100000
if (this.agents[0].infos.length>=groupN) {
if (!this.charts) {
if (this.agents[0].infos.length >= groupN) {
if (!this.charts) {
this.charts = new Charts()
this.charts.init(this.agents, groupN)
} else {
this.charts.update(this.agents, groupN, maxN)
}
}
}
}
}
@@ -164,15 +173,17 @@ class Game extends HeadlessGame {
function saveAs(dv, name) {
var a;
if (typeof window.downloadAnchor == 'undefined') {
a = window.downloadAnchor = document.createElement("a");
a.style = "display: none";
document.body.appendChild(a);
a = window.downloadAnchor = document.createElement("a");
a.style = "display: none";
document.body.appendChild(a);
} else {
a = window.downloadAnchor
a = window.downloadAnchor
}
var blob = new Blob([dv], { type: 'application/octet-binary' }),
tmpURL = window.URL.createObjectURL(blob);
var blob = new Blob([dv], {
type: 'application/octet-binary'
}),
tmpURL = window.URL.createObjectURL(blob);
a.href = tmpURL;
a.download = name;
@@ -204,4 +215,9 @@ function saveAs(dv, name) {
// reader.readAsArrayBuffer(input.files[0]);
// };
module.exports = {Game, chooseQoute, saveAs, HeadlessGame}
module.exports = {
Game,
chooseQoute,
saveAs,
HeadlessGame
}
src/js/renderer.js
+42 -34
View File
@@ -17,27 +17,27 @@ class Renderer {
setFps(fps) {
this.config.draw_fps = fps;
if(this.draw_interval)
if (this.draw_interval)
clearInterval(this.draw_interval);
if(fps > 0 && this.config.simulation_fps > 0) {
this.draw_interval = setInterval(this.drawFrame.bind(this), Math.round(1000/this.config.draw_fps));
if (fps > 0 && this.config.simulation_fps > 0) {
this.draw_interval = setInterval(this.drawFrame.bind(this), Math.round(1000 / this.config.draw_fps));
}
}
drawFrame() {
this.ctx.clearRect(0, 0, this.main_screen.width, this.main_screen.height);
this.ctx.save();
var minmax = this.getMinMaxDistance();
this.target_zoom = Math.min(this.config.max_zoom_factor, this.getZoom(minmax.min_x, minmax.max_x + 4, minmax.min_y + 2, minmax.max_y + 2.5));
this.zoom += 0.1*(this.target_zoom - this.zoom);
this.translate_x += 0.1*(1.5-minmax.min_x - this.translate_x);
this.translate_y += 0.3*(minmax.min_y*this.zoom + 280 - this.translate_y);
this.ctx.translate(this.translate_x*this.zoom, this.translate_y);
this.zoom += 0.1 * (this.target_zoom - this.zoom);
this.translate_x += 0.1 * (1.5 - minmax.min_x - this.translate_x);
this.translate_y += 0.3 * (minmax.min_y * this.zoom + 280 - this.translate_y);
this.ctx.translate(this.translate_x * this.zoom, this.translate_y);
this.ctx.scale(this.zoom, -this.zoom);
this.drawFloor();
for(var k = this.config.population_size - 1; k >= 0 ; k--) {
for (var k = this.config.population_size - 1; k >= 0; k--) {
this.drawWalker(this.walkers[k]);
}
this.ctx.restore();
@@ -45,21 +45,21 @@ class Renderer {
drawFloor() {
this.ctx.strokeStyle = "#444";
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = 1 / this.zoom;
this.ctx.beginPath();
var floor_fixture = this.floor.GetFixtureList();
this.ctx.moveTo(floor_fixture.m_shape.m_vertices[0].x, floor_fixture.m_shape.m_vertices[0].y);
for(var k = 1; k < floor_fixture.m_shape.m_vertices.length; k++) {
for (var k = 1; k < floor_fixture.m_shape.m_vertices.length; k++) {
this.ctx.lineTo(floor_fixture.m_shape.m_vertices[k].x, floor_fixture.m_shape.m_vertices[k].y);
}
this.ctx.stroke();
}
drawWalker (walker) {
drawWalker(walker) {
var hue = walker.hue || 240
this.ctx.strokeStyle = "hsl(" + hue + ",100%,0%)";
this.ctx.fillStyle = "hsl("+hue+",45%,"+(100-15*walker.health/this.config.walker_health)+"%)";
this.ctx.lineWidth = 1/this.zoom;
this.ctx.fillStyle = "hsl(" + hue + ",45%," + (100 - 15 * walker.health / this.config.walker_health) + "%)";
this.ctx.lineWidth = 1 / this.zoom;
// left legs and arms first
this.drawRect(walker.left_leg.lower_leg);
@@ -67,34 +67,34 @@ class Renderer {
this.drawRect(walker.left_arm.upper_arm);
this.drawRect(walker.left_arm.lower_arm);
this.ctx.lineWidth = walker.left_leg.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
this.ctx.lineWidth = walker.left_leg.frictionJoint.maxForce ? 4 / this.zoom : 1 / this.zoom;
this.drawRect(walker.left_leg.foot);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = walker.left_arm.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
this.ctx.lineWidth = 1 / this.zoom;
this.ctx.lineWidth = walker.left_arm.frictionJoint.maxForce ? 4 / this.zoom : 1 / this.zoom;
this.drawRect(walker.left_arm.hand);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = 1 / this.zoom;
// head
this.drawRect(walker.head.neck);
this.drawRect(walker.head.head);
// torso
this.drawRect(walker.torso.lower_torso);
this.drawRect(walker.torso.upper_torso);
// right legs and arms
this.drawRect(walker.right_leg.upper_leg);
this.drawRect(walker.right_leg.lower_leg);
this.drawRect(walker.right_arm.upper_arm);
this.drawRect(walker.right_arm.lower_arm);
this.ctx.lineWidth = walker.right_leg.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
this.ctx.lineWidth = walker.right_leg.frictionJoint.maxForce ? 4 / this.zoom : 1 / this.zoom;
this.drawRect(walker.right_leg.foot);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = walker.right_arm.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
this.ctx.lineWidth = 1 / this.zoom;
this.ctx.lineWidth = walker.right_arm.frictionJoint.maxForce ? 4 / this.zoom : 1 / this.zoom;
this.drawRect(walker.right_arm.hand);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = 1 / this.zoom;
}
drawRect(body) {
@@ -104,7 +104,7 @@ class Renderer {
var shape = fixture.GetShape();
var p0 = body.GetWorldPoint(shape.m_vertices[0]);
this.ctx.moveTo(p0.x, p0.y);
for(var k = 1; k < 4; k++) {
for (var k = 1; k < 4; k++) {
var p = body.GetWorldPoint(shape.m_vertices[k]);
this.ctx.lineTo(p.x, p.y);
}
@@ -117,7 +117,8 @@ class Renderer {
drawTest() {
this.ctx.strokeStyle = "#000";
this.ctx.fillStyle = "#666";
this.ctx.lineWidth = 1;1
this.ctx.lineWidth = 1;
1
this.ctx.beginPath();
this.ctx.moveTo(0, 0);
this.ctx.lineTo(0, 2);
@@ -133,8 +134,8 @@ class Renderer {
var max_x = -1;
var min_y = 9999;
var max_y = -1;
for(var k = 0; k < this.walkers.length; k++) {
if(this.walkers[k].health > 0) {
for (var k = 0; k < this.walkers.length; k++) {
if (this.walkers[k].health > 0) {
var dist = this.walkers[k].torso.upper_torso.GetPosition();
min_x = Math.min(min_x, dist.x);
max_x = Math.max(max_x, dist.x);
@@ -142,14 +143,21 @@ class Renderer {
max_y = Math.max(max_y, dist.y);
}
}
return {min_x:min_x, max_x:max_x, min_y:min_y, max_y:max_y};
return {
min_x: min_x,
max_x: max_x,
min_y: min_y,
max_y: max_y
};
}
getZoom(min_x, max_x, min_y, max_y) {
var delta_x = Math.abs(max_x - min_x);
var delta_y = Math.abs(max_y - min_y);
var zoom = Math.min(this.main_screen.width/delta_x,this.main_screen.height/delta_y);
var zoom = Math.min(this.main_screen.width / delta_x, this.main_screen.height / delta_y);
return zoom;
}
}
module.exports = {Renderer}
module.exports = {
Renderer
}
src/js/utils.js
+15 -7
View File
@@ -1,12 +1,12 @@
var randf = (low, high) => Math.random() * (high - low) + low
var randi = (low, high) => (Math.random() * (high - low) + low)//1
var randi = (low, high) => (Math.random() * (high - low) + low) //1
function deg2rad(deg) {
return deg / 180 * Math.PI
}
class MovingAverage{
class MovingAverage {
constructor(N) {
this.N = N
this.buffer = []
@@ -15,19 +15,27 @@ class MovingAverage{
if (this.buffer.length > this.N) this.buffer.splice(0, 1)
this.buffer.push(val)
}
mean() {
mean() {
return mean(this.buffer)
}
}
function mean(array){
function mean(array) {
if (array.length == 0)
return null;
var sum = array.reduce(function(a, b) { return a + b; });
return null;
var sum = array.reduce(function (a, b) {
return a + b;
});
var avg = sum / array.length;
return avg;
}
module.exports = {deg2rad, randf, randi, MovingAverage, mean}
module.exports = {
deg2rad,
randf,
randi,
MovingAverage,
mean
}
src/js/walker.js
+18 -21
View File
@@ -5,9 +5,11 @@ const {
randf,
deg2rad
} = require('./utils.js')
const {Renderer} = require('./renderer')
const {
Renderer
} = require('./renderer')
const STRENGTH = 3
const STRENGTH = 4
class Walker {
constructor(world, floor, config) {
@@ -147,22 +149,14 @@ class Walker {
if (contact.m_fixtureA.m_body.m_userData == "floor" | contact.m_fixtureB.m_body.m_userData) {
var otherFixture = contact.m_fixtureA.m_body.m_userData == "floor" ? contact.m_fixtureB : contact.m_fixtureA
if (otherFixture.m_body === self.right_leg.foot) {
setTimeout(() => {
self.right_leg.frictionJoint.maxForce = 0
}, 100)
setTimeout(() => {
self.right_leg.frictionJoint.maxTorque = 0
}, 100)
self.right_leg.frictionJoint.maxForce = 0
self.right_leg.frictionJoint.maxTorque = 0
} else if (otherFixture.m_body === self.left_leg.foot) {
self.left_leg.frictionJoint.maxForce = 0
self.left_leg.frictionJoint.maxTorque = 0
} else if (otherFixture.m_body == self.right_arm.hand) {
setTimeout(() => {
self.right_arm.frictionJoint.maxForce = 0
}, 100)
setTimeout(() => {
self.right_arm.frictionJoint.maxTorque = 0
}, 100)
self.right_arm.frictionJoint.maxForce = 0
self.right_arm.frictionJoint.maxTorque = 0
} else if (otherFixture.m_body === self.left_arm.hand) {
self.left_arm.frictionJoint.maxForce = 0
self.left_arm.frictionJoint.maxTorque = 0
@@ -559,10 +553,10 @@ class Walker {
@delta (Float) time since the last update
@action: (Integer) The action to take (can be null if no action)
*/
for (let i = 0; i < this.config.action_repeat; i++) {
if (this.step>50) this.simulationPreStep(motorSpeeds)
for (let i = 0; i < this.config.action_repeat; i++) {
if (this.step > 50) this.simulationPreStep(motorSpeeds)
this.world.Step(1 / this.config.time_step, this.config.velocity_iterations, this.config.position_iterations);
if (typeof WEB!=="undefined") this.renderer.drawFrame()
if (typeof WEB !== "undefined") this.renderer.drawFrame()
}
this.steps++
this.episodeSteps++
@@ -611,7 +605,10 @@ class Walker {
quad_joint_angle_cost,
bonus_happiness,
head_height_reward,
leg_switch_reward
leg_switch_reward,
head_height: (this.head.head.GetPosition().y - mean_foot_height),
center_x: this.torso.upper_torso.GetPosition().x,
center_y: this.torso.upper_torso.GetPosition().y
}
this.reward = Object.values(this.rewards).reduce((tot, v) => tot + v, 0) / 3
@@ -644,7 +641,7 @@ class Walker {
reset() {
/** Reset position to initial or random position TODO */
// console.log('reset not implemented')
if (this.bodies) this.destroy();
if (this.bodies) this.destroy();
this.build();
this.initGrip()
this.episodeSteps = 0
@@ -653,7 +650,7 @@ class Walker {
shuffle() {
/** Reset position to initial or random position TODO */
// this.joints.forEach(j => {
// })
@@ -664,7 +661,7 @@ class Walker {
// pos.x += randf(-1, 1)
// pos.y += randf(0, 2)
angle += Math.PI
b.SetTransform(pos, angle)
b.SetTransform(pos, angle)
// let dx = randf(-1, 1)
// let dy = randf(-1, 1)