wassname/rl_2d_walker.js
1
0
Fork 0
mirror of https://github.com/wassname/rl_2d_walker.js.git synced 2026-09-09 11:33:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

moving around, webpack

Browse Source
This commit is contained in:
wassname
2018-12-01 16:36:35 +08:00
parent d910497e90
commit 09550ad374
23 changed files with 332 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/agent.js
-93
View File
@@ -1,93 +0,0 @@
function Agent(opt, globals) {
this.walker = new Walker(globals.world, globals.floor)
this.options = opt
this.globals = globals
this.frequency = 20
this.loaded = false
this.infos = []
this.maxInfos = 2000
this.steps = 0
this.timer = 0
this.timerFrequency = 60 / this.frequency
if (this.options.dynamicallyLoaded !== true) {
this.init(globals.brains.actor.newConfiguration(), null)
}
};
Agent.prototype.init = function (actor, critic) {
var actions = this.walker.joints.length + 4
var temporal = 1
var states = this.walker.bodies.length * 10 + this.walker.joints.length * 3
var input = window.neurojs.Agent.getInputDimension(states, actions, temporal)
// Example params https://github.com/udacity/deep-reinforcement-learning/blob/master/ddpg-bipedal/ddpg_agent.py
this.brain = new window.neurojs.Agent({
actor: actor,
critic: critic,
states: states,
actions: actions,
algorithm: 'ddpg',
temporalWindow: temporal,
discount: 0.99, // time discount
rate: 3e-4, // learning rate,
theta: 1e-3, // progressive copy
// alpha: 0.1, // advantage learning
// buffer: window.neurojs.Buffers.UniformReplayBuffer,
experience: 100e3,
learningPerTick: 512,
startLearningAt: 10000,
})
this.brain.algorithm.critic.optim.regularization.l2 = 0.0001
this.brain.algorithm.actor.optim.regularization.l2 = 0.0001
// this.globals.brains.shared.add('actor', this.brain.algorithm.actor)
this.globals.brains.shared.add('critic', this.brain.algorithm.critic)
this.actions = actions
this.loaded = true
};
Agent.prototype.step = function () {
if (!this.loaded) {
return
}
this.timer++
if (this.timer % this.timerFrequency === 0) {
this.steps++
var [state, reward, done, info] = this.walker.simulationStep()
if (done) {
// TODO reset?
}
if (this.infos.length>this.maxInfos) this.infos = this.infos.slice(1)
// train
info.loss = this.brain.learn(reward)
this.action = this.brain.policy(state)
info.x = this.steps
info.time = new Date().getTime()
this.infos.push(info)
}
if (this.action) {
this.walker.simulationPreStep(this.action)
}
return this.timer % this.timerFrequency === 0
};
js/game.js
-243
View File
@@ -1,243 +0,0 @@
var requestAnimFrame = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) { window.setTimeout(callback, 1000 / 60); };
config = {
time_step: 60,
simulation_fps: 60,
draw_fps: 60,
velocity_iterations: 8,
position_iterations: 3,
max_zoom_factor: 130,
min_motor_speed: -2,
max_motor_speed: 2,
population_size: 1,
walker_health: 100,
max_floor_tiles: 50,
round_length: 1000,
min_body_delta: 0,
min_leg_delta: 0.0,
};
globals = {};
chooseQoute = function () {
var qoutes = [
'Play the funky music, robot',
'The origin of funkd',
'The chaos computer club',
'Only the humans that like to dance survived',
'Classic robot dance move - the human',
'First we dance Manhatten, then we dance the world',
'Video of subjects one hour after ingesting substance q1043',
'Red robot redemption',
'Father was a rolling robot',
'Float like a bumblebee, string like a butterfly',
'Dance evolution',
'Have you tried turning it off and on again?',
'Eurovision 2050',
'Humans must learn to crawl then walk. Robots break dance then walk',
''
]
var qoute = qoutes[Math.randi(0,qoutes.length)]
document.getElementById('page_quote').innerText = '"'+qoute+'"'
}
displayProgress = function () {
// TODO show stats
var stats = {
'trainingTime': globals.step_counter / config.simulation_fps,
'meanProgress': globals.walkers.map(w => w.last_position).reduce((s, v) => s + v) / globals.walkers.length,
'meanReward': globals.walkers.map(w => w.reward).reduce((s, v) => s + v) / globals.walkers.length,
'bufferSize': globals.agents[0].brain.buffer.size
}
document.getElementById('stats-prog').innerText = JSON.stringify(stats, null, 2)
}
gameInit = function() {
var bodyParts = 16
var joints = 14
var state = bodyParts * 10 + joints * 3
var actions = joints + 4
var input = 2 * state + 1 * actions
globals.brains = {
actor: new window.neurojs.Network.Model([
{ type: 'input', size: input },
{ type: 'fc', size: 256, activation: 'relu' },
// { type: 'noise', sigma: 0.2, delta: 0.001, theta: 0.15 },
// { type: 'fc', size: 160, activation: 'relu' },
// { type: 'noise', sigma: 0.2, delta: 0.001, theta: 0.15 },
// { type: 'fc', size: 100, activation: 'relu' },
{ type: 'fc', size: 40, activation: 'relu', dropout: 0.30 },
// delta represents the equilibrium or mean value supported by fundamentals;
// sigma the degree of volatility around it caused by shocks,
// theta the rate by which these shocks dissipate and the variable reverts towards the mean.
{ type: 'fc', size: actions, activation: 'tanh' },
{ type: 'noise', sigma: 0.3, delta: 0.1, theta: 0.15 },
{ type: 'regression' }
]),
critic: new window.neurojs.Network.Model([
{ type: 'input', size: input + actions },
{ type: 'fc', size: 256, activation: 'relu' },
// { type: 'fc', size: 256, activation: 'relu' },
{ type: 'fc', size: 40, activation: 'relu' },
{ type: 'fc', size: 1 },
{ type: 'regression' }
])
}
globals.brains.shared = new window.neurojs.Shared.ConfigPool()
// this.brains.shared.set('actor', this.brains.actor.newConfiguration())
globals.brains.shared.set('critic', globals.brains.critic.newConfiguration())
chooseQoute()
globals.world = new b2.World(new b2.Vec2(0, -10));
globals.floor = createFloor(globals.world);
[globals.agents, globals.walkers] = createPopulation();
drawInit();
globals.step_counter = 0;
globals.display_interval = setInterval(displayProgress, Math.round(380 * 1000 / config.draw_fps));
globals.charts_interval = setInterval(updateCharts, Math.round(380 * 1000 / config.draw_fps));
globals.running = true
requestAnimFrame(loop)
}
loop = function () {
drawFrame()
simulationStep()
drawFrame()
if (globals.running) requestAnimFrame(loop); // start next timer
}
resetSimulation = function () {
// turn training off temporarlity to avoid NaN's
updateIfLearning(false)
// globals.running = false
globals.world.Destroy() // this way we get rid of listeners and body parts and joints
globals.world = new b2.World(new b2.Vec2(0, -10));
globals.floor = createFloor(globals.world);
for (var k = 0; k < config.population_size; k++) {
globals.agents[k].walker = globals.walkers[k] = new Walker(globals.world, globals.floor)
}
// globals.running = true
setTimeout(() => updateIfLearning(true), 1000)
// setTimeout(() => requestAnimFrame(loop), 1000)
}
simulationStep = function () {
globals.step_counter++;
// step world
globals.world.Step(1/config.time_step, config.velocity_iterations, config.position_iterations);
populationSimulationStep();
globals.world.ClearForces();
// step agents (only after step 50 when they are on the ground)
}
updateCharts = function () {
var groupN = 100
if (globals.agents[0].infos.length>=groupN) {
if (!globals.charts) {
globals.charts = new Charts()
globals.charts.init(globals.agents, groupN)
} else {
globals.charts.update(globals.agents, groupN, 100000)
}
}
}
createPopulation = function(genomes) {
var walkers = [];
var agents = []
for(var k = 0; k < config.population_size; k++) {
var agent = new Agent({}, globals)
agents.push(agent);
walkers.push(agent.walker)
}
return [agents, walkers];
}
populationSimulationStep = function() {
for (var k = 0; k < config.population_size; k++) {
if (globals.walkers[k].steps>150)
globals.agents[k].step()
else
globals.walkers[k].simulationStep()
}
var steps = globals.agents[0].walker.steps
if ((steps!==0) && (0 == steps % config.round_length)) {
resetSimulation()
}
}
function saveAs(dv, name) {
var a;
if (typeof window.downloadAnchor == 'undefined') {
a = window.downloadAnchor = document.createElement("a");
a.style = "display: none";
document.body.appendChild(a);
} else {
a = window.downloadAnchor
}
var blob = new Blob([dv], { type: 'application/octet-binary' }),
tmpURL = window.URL.createObjectURL(blob);
a.href = tmpURL;
a.download = name;
a.click();
window.URL.revokeObjectURL(tmpURL);
a.href = "";
}
downloadBrain = function (n) {
var ts = (new Date()).toISOString().replace(':','_')
var buf = globals.agents[n].brain.export()
saveAs(new DataView(buf), 'walker_brain_'+n+'_'+ts+'.bin')
};
readBrain = function (buf) {
var input = event.target;
var reader = new FileReader();
reader.onload = function(){
var buffer = reader.result
var imported = window.neurojs.NetOnDisk.readMultiPart(buffer)
for (var i = 0; i < globals.agents.length; i++) {
globals.agents[i].brain.algorithm.actor.set(imported.actor.clone())
globals.agents[i].brain.algorithm.critic.set(imported.critic)
}
};
reader.readAsArrayBuffer(input.files[0]);
};
updateIfLearning = function (value) {
for (var i = 0; i < globals.agents.length; i++) {
globals.agents[i].brain.learning = value
}
};
package.json
+3 -1
View File
@@ -9,7 +9,9 @@
"jsdom": "^13.0.0",
"phaser": "^3.15.1"
},
"devDependencies": {},
"devDependencies": {
"webpack-cli": "^3.1.2"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
css/walkers.css → src/css/walkers.css
View File
index.html → src/index.html
+18 -3
View File
@@ -3,16 +3,31 @@
<head>
<title>HTML5 Genetic Algorithm Biped Walkers</title>
<link rel="stylesheet" href="css/walkers.css" type="text/css" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.3/Chart.js"></script>
<!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.3/Chart.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js"></script>
<script src="vendor/jsbox2d.js"></script>
<script src="vendor/neurojs-v2.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script> -->
<!-- <script type="text/javascript" src="/js/ddpg/models.js"></script> -->
<!-- <script type="text/javascript" src="/js/ddpg/memory.js"></script> -->
<!-- <script type="text/javascript" src="/js/ddpg/prioritized_memory.js"></script>
<script type="text/javascript" src="/js/ddpg/noise.js"></script>
<script type="text/javascript" src="/js/ddpg/ddpg.js"></script>
<script type="text/javascript" src="/js/ddpg/ddpg_agent.js"></script>
<script type="text/javascript" src="/js/ddpg-traffic/index.js"></script>
<script src="js/walker.js"></script>
<script src="js/game.js"></script>
<script src="js/floor.js"></script>
<script src="js/draw.js"></script>
<script src="js/agent.js"></script>
<script src="js/charts.js"></script>
<script src="js/charts.js"></script> -->
<script src="dist/bundle.js"></script>
<script>
function init() {
src/index.js
+1
View File
@@ -0,0 +1 @@
const { Game } = require('game')
js/charts.js → src/js/charts.js
+1
View File
@@ -91,3 +91,4 @@ Charts.prototype.update = function (agents) {
}
}
module.exports ={Chart}
src/js/config.js
+17
View File
@@ -0,0 +1,17 @@
module.exports = {
time_step: 60,
simulation_fps: 60,
draw_fps: 60,
velocity_iterations: 8,
position_iterations: 3,
max_zoom_factor: 130,
min_motor_speed: -2,
max_motor_speed: 2,
population_size: 1,
walker_health: 100,
max_floor_tiles: 50,
round_length: 1000,
min_body_delta: 0,
min_leg_delta: 0.0,
};
js/ddpg/ddpg.js → src/js/ddpg/ddpg.js
View File
js/ddpg/ddpg_agent.js → src/js/ddpg/ddpg_agent.js
View File
src/js/ddpg/memory.js
+86
View File
@@ -0,0 +1,86 @@
// class Memory {
// /**
// * @param maxlen (number) Buffer limit
// */
// constructor(maxlen){
// this.maxlen = maxlen;
// this.length = 0;
// this.start = 0;
// this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
// }
// /**
// * @param idx (number)
// */
// getItem(idx){
// if (idx < 0 || idx >= this.length){
// console.error("Memory.getItem: idx not in range.");
// }
// return this.data[(this.start + idx) % this.maxlen]
// }
// /**
// * Sample a batch
// * @param batchSize (number)
// * @return batch []
// */
// getBatch(batchSize){
// const arrLength = this.length;
// const batch = {
// 'obs0': [],
// 'obs1': [],
// 'rewards': [],
// 'actions': [],
// 'terminals': [],
// };
// if (batchSize > this.length){
// return batch;
// }
// for (let b=0; b < batchSize; b++){
// let id = Math.floor(Math.random() * arrLength);
// batch.obs0.push(this.obs0List[id]);
// batch.obs1.push(this.obs1List[id]);
// batch.rewards.push(this.rewardsList[id]);
// batch.actions.push(this.actionsList[id]);
// batch.terminals.push(this.terminals1List[id]);
// }
// return batch
// }
// /**
// * @param obs0 []
// * @param action (number)
// * @param reward (number)
// * @param obs1 []
// * @param terminal1 (boolean)
// */
// append(obs0, action, reward, obs1, terminal1){
// if (this.length < this.maxlen){
// this.length += 1;
// }
// else if (this.length == this.maxlen) {
// //this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
// //this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
// //this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
// this.start = (this.start + 1) % this.maxlen;
// }
// else {
// console.error("Memory.append: This should never be printed");
// }
// this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
// this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
// this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
// this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
// this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
// }
// }
// module.exports = Memory
js/ddpg/models.js → src/js/ddpg/models.js
View File
js/ddpg/noise.js → src/js/ddpg/noise.js
View File
js/ddpg/prioritized_memory.js → src/js/ddpg/prioritized_memory.js
View File
js/ddpg/tf_import.js → src/js/ddpg/tf_import.js
View File
js/draw.js → src/js/draw.js
View File
js/floor.js → src/js/floor.js
View File
src/js/game.js
+157
View File
@@ -0,0 +1,157 @@
const config = require('./config')
const {Charts} = require('./charts')
if typeof window !=="undefined"
var requestAnimFrame = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) { window.setTimeout(callback, 1000 / 60); };
else
var requestAnimFrame = function (callback) { window.setTimeout(callback, 1000 / 60); };
chooseQoute = function () {
var qoutes = [
'Play the funky music, robot',
'The origin of funkd',
'The chaos computer club',
'Only the humans that like to dance survived',
'Classic robot dance move - the human',
'First we dance Manhatten, then we dance the world',
'Video of subjects one hour after ingesting substance q1043',
'Red robot redemption',
'Father was a rolling robot',
'Float like a bumblebee, string like a butterfly',
'Dance evolution',
'Have you tried turning it off and on again?',
'Eurovision 2050',
'Humans must learn to crawl then walk. Robots break dance then walk',
''
]
var qoute = qoutes[Math.randi(0,qoutes.length)]
document.getElementById('page_quote').innerText = '"'+qoute+'"'
}
class Game {
constructor(params) {
var bodyParts = 16
var joints = 14
var state = bodyParts * 10 + joints * 3
var actions = joints + 4
var input = 2 * state + 1 * actions
chooseQoute()
this.world = new b2.World(new b2.Vec2(0, -10));
this.floor = createFloor(this.world);
[this.agents, this.walkers] = createPopulation();
drawInit();
this.step_counter = 0;
this.display_interval = setInterval(displayProgress, Math.round(380 * 1000 / config.draw_fps));
this.charts_interval = setInterval(updateCharts, Math.round(380 * 1000 / config.draw_fps));
this.running = true
requestAnimFrame(loop)
}
displayProgress() {
var stats = {
'trainingTime': this.step_counter / config.simulation_fps,
'meanProgress': this.walkers.map(w => w.last_position).reduce((s, v) => s + v) / this.walkers.length,
'meanReward': this.walkers.map(w => w.reward).reduce((s, v) => s + v) / this.walkers.length,
'bufferSize': this.agents[0].brain.buffer.size
}
document.getElementById('stats-prog').innerText = JSON.stringify(stats, null, 2)
}
resetSimulation() {
// turn training off temporarlity to avoid NaN's
updateIfLearning(false)
// this.running = false
this.world.Destroy() // this way we get rid of listeners and body parts and joints
this.world = new b2.World(new b2.Vec2(0, -10));
this.floor = createFloor(this.world);
for (var k = 0; k < config.population_size; k++) {
this.agents[k].walker = this.walkers[k] = new Walker(this.world, this.floor)
}
// this.running = true
setTimeout(() => updateIfLearning(true), 1000)
// setTimeout(() => requestAnimFrame(loop), 1000)
}
loop() {
drawFrame()
simulationStep()
drawFrame()
if (this.running) requestAnimFrame(loop); // start next timer
}
updateCharts() {
var groupN = 100
var maxN = 100000
if (this.agents[0].infos.length>=groupN) {
if (!this.charts) {
this.charts = new Charts()
this.charts.init(this.agents, groupN)
} else {
this.charts.update(this.agents, groupN, maxN)
}
}
}
}
function saveAs(dv, name) {
var a;
if (typeof window.downloadAnchor == 'undefined') {
a = window.downloadAnchor = document.createElement("a");
a.style = "display: none";
document.body.appendChild(a);
} else {
a = window.downloadAnchor
}
var blob = new Blob([dv], { type: 'application/octet-binary' }),
tmpURL = window.URL.createObjectURL(blob);
a.href = tmpURL;
a.download = name;
a.click();
window.URL.revokeObjectURL(tmpURL);
a.href = "";
}
// downloadBrain = function (n) {
// var ts = (new Date()).toISOString().replace(':','_')
// var buf = this.agents[n].brain.export()
// saveAs(new DataView(buf), 'walker_brain_'+n+'_'+ts+'.bin')
// };
// readBrain = function (buf) {
// var input = event.target;
// var reader = new FileReader();
// reader.onload = function(){
// var buffer = reader.result
// var imported = window.neurojs.NetOnDisk.readMultiPart(buffer)
// for (var i = 0; i < this.agents.length; i++) {
// this.agents[i].brain.algorithm.actor.set(imported.actor.clone())
// this.agents[i].brain.algorithm.critic.set(imported.critic)
// }
// };
// reader.readAsArrayBuffer(input.files[0]);
// };
module.exports = {Game, chooseQoute, saveAs}
src/js/utils.js
+7
View File
@@ -0,0 +1,7 @@
var randf = (low, high) => Math.random() * (high - low) + low
function deg2rad(deg) {
return deg / 180 * Math.PI
}
module.exports = {deg2rad, randf}
js/walker.js → src/js/walker.js
+3 -51
View File
@@ -1,10 +1,7 @@
// walker has fixed shapes and structures
// shape definitions are in the constructor
var randf = (low, high) => Math.random() * (high - low) + low
function deg2rad(deg) {
return deg / 180 * Math.PI
}
const b2 = require('../vendor/jsbox2d')
const { randf, deg2rad } = require('./utils.js')
const STRENGTH = 3
@@ -631,49 +628,4 @@ Walker.prototype.shuffle = function () {
console.log('shuffle not implemented')
}
config = {
time_step: 60,
simulation_fps: 60,
draw_fps: 60,
velocity_iterations: 8,
position_iterations: 3,
max_zoom_factor: 130,
min_motor_speed: -2,
max_motor_speed: 2,
population_size: 1,
walker_health: 100,
max_floor_tiles: 50,
round_length: 1000,
min_body_delta: 0,
min_leg_delta: 0.0,
};
var b2 = require('../vendor/jsbox2d')
var createFloor = require('./floor.js')
var DDPGAgent = require('./ddpg/ddpg_agent')
var DDPGAgent = require('./ddpg/ddpg_agent')
var world = new b2.World(new b2.Vec2(0, -10))
floor = createFloor(world, config.max_floor_tiles);
var env = new Walker(world, floor, config)
var nbActions = env.joints.length + 4
var stateSize = env.bodies.length * 10 + env.joints.length * 3
var agent = new DDPGAgent(env, {
stateSize,
nbActions,
resetEpisode: true,
desiredActionStddev: 0.4,
initialStddev: 0.4,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
nbEpochs: 1000
});
agent.train(true);
module.exports = {Walker, randf}
src/train.js
+30
View File
@@ -0,0 +1,30 @@
var config = require('./js/config')
const b2 = require('./vendor/jsbox2d')
const createFloor = require('./js/floor.js')
const DDPGAgent = require('./js/ddpg/ddpg_agent')
const {
Walker
} = require('./js/walker')
var world = new b2.World(new b2.Vec2(0, -10))
floor = createFloor(world, config.max_floor_tiles);
var env = new Walker(world, floor, config)
const nbActions = env.joints.length + 4
const stateSize = env.bodies.length * 10 + env.joints.length * 3
var agent = new DDPGAgent(env, {
stateSize,
nbActions,
resetEpisode: true,
desiredActionStddev: 0.4,
initialStddev: 0.4,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
nbEpochs: 1000
});
agent.train(true);
vendor/jsbox2d.js → src/vendor/jsbox2d.js
Vendored
View File
webpack.config.js
+9
View File
@@ -0,0 +1,9 @@
const path = require('path');
module.exports = {
entry: './js/index.js',
output: {
path: path.resolve(__dirname, 'dist'),
filename: 'bundle.js'
}
};