wassname/rl_2d_walker.js
1
0
Fork 0
mirror of https://github.com/wassname/rl_2d_walker.js.git synced 2026-09-09 11:33:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

frontend loads files

Browse Source
This commit is contained in:
wassname
2018-12-02 10:30:04 +08:00
parent fbf4edac33
commit 7d7cd556a4
14 changed files with 6324 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6
View File
@@ -38,3 +38,9 @@ TODO: Write usage instructions
- The walker code is adapted from <a href="http://rednuht.org/genetic_walkers/">http://rednuht.org/genetic_walkers/</a>
- DDPG code from metacar
# Notes
- Without node: Training: 52732.152ms
- With node: Training: 17211.739ms
- With cuda LoopTime: 12583.449ms
package-lock.json
Generated
+6091
View File
File diff suppressed because it is too large Load Diff
package.json
+6 -2
View File
@@ -6,12 +6,16 @@
"dependencies": {
"@tensorflow/tfjs-node": "^0.1.17",
"@tensorflow/tfjs-node-gpu": "^0.1.17",
"fs-extra": "^7.0.0",
"canvas": "^2.1.0",
"fs-extra": "^7.0.0",
"jsdom": "^13.0.0",
"phaser": "^3.15.1"
"phaser": "^3.15.1",
"webpack": "^4.26.1",
"webpack-dev-server": "^3.1.10"
},
"devDependencies": {
"clean-webpack-plugin": "^1.0.0",
"html-webpack-plugin": "^3.2.0",
"webpack-cli": "^3.1.2"
},
"scripts": {
src/index.html
+4 -12
View File
@@ -24,20 +24,9 @@
<script src="js/agent.js"></script>
<script src="js/charts.js"></script> -->
<script src="dist/bundle.js"></script>
<script>
function init() {
gameInit();
}
window.addEventListener("load", init, false);
</script>
</head>
<body>
<div id="main_holder">
@@ -68,5 +57,8 @@
</div>
</body>
<script src="../dist/vendor.bundle.js"></script>
<script src="../dist/lib.bundle.js"></script>
<script src="../dist/app.bundle.js"></script>
</html>
src/index.js
+12 -1
View File
@@ -1 +1,12 @@
const { Game } = require('game')
const { Game } = require('./js/game')
const config = require('./js/config')
var game
window.game = game
function init() {
window.game = new Game(config)
}
window.addEventListener("load", init, false);
module.exports = {game, Game}
src/js/charts.js
+1 -1
View File
@@ -91,4 +91,4 @@ Charts.prototype.update = function (agents) {
}
}
module.exports ={Chart}
module.exports ={Charts}
src/js/ddpg/ddpg_agent.js
+15 -8
View File
@@ -1,7 +1,7 @@
const { tf } = require('./tf_import')
const AdaptiveParamNoiseSpec = require('./noise')
const PrioritizedMemory = require('./prioritized_memory')
const { Actor, Critic, } = require('./models')
const { Actor, Critic, copyFromSave, copyModel} = require('./models')
const { DDPG, logTfMemory } = require('./ddpg')
const { mean } = require('../utils')
@@ -80,22 +80,29 @@ class DDPGAgent {
/*
Save the network
*/
if (typeof window === "undefined") {
this.ddpg.actor.model.save('file://./outputs/actor-' + name);
if (typeof WEB === "undefined") {
this.ddpg.critic.model.save('file://./outputs/critic-' + name);
this.ddpg.actor.model.save('file://./outputs/actor-' + name);
} else {
this.ddpg.critic.model.save('downloads://critic-' + name);
this.ddpg.actor.model.save('downloads://actor-'+ name);
this.ddpg.critic.model.save('downloads://critic-' + name);
}
}
async restore(folder, name){
/*
Restore the weights of the network
*/
const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
var critic, actor
if (typeof WEB === "undefined") {
critic = await tf.loadModel('file://'+folder+'/critic-'+name+'.json');
actor = await tf.loadModel("file://"+folder+"/actor-"+name+".json");
} else {
critic = await tf.loadModel(window.location.href+folder+'/critic-'+name+'.json');
actor = await tf.loadModel(window.location.href+folder+"/actor-"+name+".json");
// const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
// const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
}
this.ddpg.critic = copyFromSave(critic, Critic, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
this.ddpg.actor = copyFromSave(actor, Actor, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
@@ -119,7 +126,7 @@ class DDPGAgent {
// Pick an action
const tfActions = this.ddpg.predict(tf.tensor2d([state]));
const actions = tfActions.buffer().values;
agent.env.step(actions);
this.env.step(actions);
tfActions.dispose();
}
src/js/ddpg/tf_import.js
+5 -3
View File
@@ -1,7 +1,9 @@
const tf = require('@tensorflow/tfjs')
// Load the binding (note you may have to press enter in the terminal for some reason)
require('@tensorflow/tfjs-node-gpu');
require('@tensorflow/tfjs-node'); // seem to need this as well for save?
if (typeof WEB ==="undefined"){
// Load the binding (note you may have to press enter in the terminal for some reason)
require('@tensorflow/tfjs-node-gpu');
require('@tensorflow/tfjs-node'); // seem to need this as well for save?
}
module.exports = { tf }
src/js/game.js
+96 -46
View File
@@ -1,8 +1,14 @@
const config = require('./config')
const {Charts} = require('./charts')
const { Charts } = require('./charts')
const { randi } = require('./utils')
const b2 = require('../vendor/jsbox2d')
const createFloor = require('./floor.js')
const DDPGAgent = require('./ddpg/ddpg_agent')
const {
Walker
} = require('./walker')
if typeof window !=="undefined"
if (typeof window !=="undefined")
var requestAnimFrame = window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || window.oRequestAnimationFrame || window.msRequestAnimationFrame || function (callback) { window.setTimeout(callback, 1000 / 60); };
else
var requestAnimFrame = function (callback) { window.setTimeout(callback, 1000 / 60); };
@@ -26,34 +32,80 @@ chooseQoute = function () {
'Humans must learn to crawl then walk. Robots break dance then walk',
''
]
var qoute = qoutes[Math.randi(0,qoutes.length)]
var qoute = qoutes[randi(0,qoutes.length)]
document.getElementById('page_quote').innerText = '"'+qoute+'"'
}
class Game {
constructor(params) {
var bodyParts = 16
var joints = 14
var state = bodyParts * 10 + joints * 3
var actions = joints + 4
var input = 2 * state + 1 * actions
class HeadlessGame {
constructor(config) {
this.config = config
this.initWorld()
}
initWorld() {
var gravity = new b2.Vec2(0, -10)
this.world = new b2.World(gravity)
this.floor = createFloor(this.world, this.config.max_floor_tiles);
this.env = new Walker(this.world, this.floor, this.config)
const nbActions = this.env.joints.length + 4
const stateSize = this.env.bodies.length * 10 + this.env.joints.length * 3
this.agent = new DDPGAgent(this.env, {
stateSize,
nbActions,
resetEpisode: true,
batchSize: 128,
actorLr: 0.0001,
criticLr: 0.001,
memorySize: 30000,
gamma: 0.99,
desiredActionStddev: 0.1,
initialStddev: 0.4,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
nbEpochs: 1000,
nbEpochsCycle: 10,
nbTrainSteps: 100,
maxStep: 800,
saveDuringTraining: true,
saveInterval: 20,
tau: 0.008,
adoptionCoefficient: 1.01,
});
}
}
class Game extends HeadlessGame {
constructor(config) {
super(config)
chooseQoute()
this.world = new b2.World(new b2.Vec2(0, -10));
this.floor = createFloor(this.world);
[this.agents, this.walkers] = createPopulation();
// this.agent.stop()
// this.agent.env.render(true)
this.agent.restore('../outputs', 'model-ddpg-walker/model')
setInterval(()=>this.agent.play(), 100)
drawInit();
// drawInit();
this.step_counter = 0;
// this.step_counter = 0;
this.display_interval = setInterval(displayProgress, Math.round(380 * 1000 / config.draw_fps));
this.charts_interval = setInterval(updateCharts, Math.round(380 * 1000 / config.draw_fps));
// this.display_interval = setInterval(displayProgress, Math.round(380 * 1000 / config.draw_fps));
// this.charts_interval = setInterval(updateCharts, Math.round(380 * 1000 / config.draw_fps));
this.running = true
requestAnimFrame(loop)
// this.running = true
// requestAnimFrame(loop)
}
displayProgress() {
@@ -67,30 +119,30 @@ class Game {
}
resetSimulation() {
// turn training off temporarlity to avoid NaN's
updateIfLearning(false)
// this.running = false
// resetSimulation() {
// // turn training off temporarlity to avoid NaN's
// updateIfLearning(false)
// // this.running = false
this.world.Destroy() // this way we get rid of listeners and body parts and joints
this.world = new b2.World(new b2.Vec2(0, -10));
this.floor = createFloor(this.world);
for (var k = 0; k < config.population_size; k++) {
this.agents[k].walker = this.walkers[k] = new Walker(this.world, this.floor)
}
// this.world.Destroy() // this way we get rid of listeners and body parts and joints
// this.world = new b2.World(new b2.Vec2(0, -10));
// this.floor = createFloor(this.world);
// for (var k = 0; k < config.population_size; k++) {
// this.agents[k].walker = this.walkers[k] = new Walker(this.world, this.floor)
// }
// this.running = true
setTimeout(() => updateIfLearning(true), 1000)
// setTimeout(() => requestAnimFrame(loop), 1000)
// // this.running = true
// setTimeout(() => updateIfLearning(true), 1000)
// // setTimeout(() => requestAnimFrame(loop), 1000)
}
// }
loop() {
drawFrame()
simulationStep()
drawFrame()
if (this.running) requestAnimFrame(loop); // start next timer
}
// loop() {
// drawFrame()
// simulationStep()
// drawFrame()
// if (this.running) requestAnimFrame(loop); // start next timer
// }
updateCharts() {
@@ -152,6 +204,4 @@ function saveAs(dv, name) {
// reader.readAsArrayBuffer(input.files[0]);
// };
module.exports = {Game, chooseQoute, saveAs}
module.exports = {Game, chooseQoute, saveAs, HeadlessGame}
src/js/draw.js → src/js/renderer.js
+33 -32
View File
@@ -2,42 +2,43 @@ class Renderer {
constructor(config, walker, floor) {
this.config = config
this.walkers = [walker]
this.floor = foor
this.floor = floor
this.main_screen = document.getElementById("main_screen");
this.ctx = main_screen.getContext("2d");
resetCamera();
this.resetCamera();
}
resetCamera() {
this.zoom = config.max_zoom_factor;
this.zoom = this.config.max_zoom_factor;
this.translate_x = 0;
this.translate_y = 280;
}
setFps(fps) {
config.draw_fps = fps;
this.config.draw_fps = fps;
if(this.draw_interval)
clearInterval(this.draw_interval);
if(fps > 0 && config.simulation_fps > 0) {
this.draw_interval = setInterval(drawFrame, Math.round(1000/config.draw_fps));
if(fps > 0 && this.config.simulation_fps > 0) {
this.draw_interval = setInterval(this.drawFrame.bind(this), Math.round(1000/this.config.draw_fps));
}
}
drawFrame() {
var minmax = getMinMaxDistance();
this.target_zoom = Math.min(config.max_zoom_factor, getZoom(minmax.min_x, minmax.max_x + 4, minmax.min_y + 2, minmax.max_y + 2.5));
this.ctx.clearRect(0, 0, this.main_screen.width, this.main_screen.height);
this.ctx.save();
var minmax = this.getMinMaxDistance();
this.target_zoom = Math.min(this.config.max_zoom_factor, this.getZoom(minmax.min_x, minmax.max_x + 4, minmax.min_y + 2, minmax.max_y + 2.5));
this.zoom += 0.1*(this.target_zoom - this.zoom);
this.translate_x += 0.1*(1.5-minmax.min_x - this.translate_x);
this.translate_y += 0.3*(minmax.min_y*this.zoom + 280 - this.translate_y);
//this.translate_y = minmax.max_y*this.zoom + 150;
this.ctx.clearRect(0, 0, this.main_screen.width, this.main_screen.height);
this.ctx.save();
this.ctx.translate(this.translate_x*this.zoom, this.translate_y);
this.ctx.scale(this.zoom, -this.zoom);
drawFloor();
for(var k = config.population_size - 1; k >= 0 ; k--) {
drawWalker(this.walkers[k]);
this.drawFloor();
for(var k = this.config.population_size - 1; k >= 0 ; k--) {
this.drawWalker(this.walkers[k]);
}
this.ctx.restore();
}
@@ -57,42 +58,42 @@ class Renderer {
drawWalker (walker) {
var hue = walker.hue || 240
this.ctx.strokeStyle = "hsl(" + hue + ",100%,0%)";
this.ctx.fillStyle = "hsl("+hue+",45%,"+(100-15*walker.health/config.walker_health)+"%)";
this.ctx.fillStyle = "hsl("+hue+",45%,"+(100-15*walker.health/this.config.walker_health)+"%)";
this.ctx.lineWidth = 1/this.zoom;
// left legs and arms first
drawRect(walker.left_leg.lower_leg);
drawRect(walker.left_leg.upper_leg);
drawRect(walker.left_arm.upper_arm);
drawRect(walker.left_arm.lower_arm);
this.drawRect(walker.left_leg.lower_leg);
this.drawRect(walker.left_leg.upper_leg);
this.drawRect(walker.left_arm.upper_arm);
this.drawRect(walker.left_arm.lower_arm);
this.ctx.lineWidth = walker.left_leg.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
drawRect(walker.left_leg.foot);
this.drawRect(walker.left_leg.foot);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = walker.left_arm.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
drawRect(walker.left_arm.hand);
this.drawRect(walker.left_arm.hand);
this.ctx.lineWidth = 1/this.zoom;
// head
drawRect(walker.head.neck);
drawRect(walker.head.head);
this.drawRect(walker.head.neck);
this.drawRect(walker.head.head);
// torso
drawRect(walker.torso.lower_torso);
drawRect(walker.torso.upper_torso);
this.drawRect(walker.torso.lower_torso);
this.drawRect(walker.torso.upper_torso);
// right legs and arms
drawRect(walker.right_leg.upper_leg);
drawRect(walker.right_leg.lower_leg);
drawRect(walker.right_arm.upper_arm);
drawRect(walker.right_arm.lower_arm);
this.drawRect(walker.right_leg.upper_leg);
this.drawRect(walker.right_leg.lower_leg);
this.drawRect(walker.right_arm.upper_arm);
this.drawRect(walker.right_arm.lower_arm);
this.ctx.lineWidth = walker.right_leg.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
drawRect(walker.right_leg.foot);
this.drawRect(walker.right_leg.foot);
this.ctx.lineWidth = 1/this.zoom;
this.ctx.lineWidth = walker.right_arm.frictionJoint.maxForce? 4/this.zoom : 1/this.zoom;
drawRect(walker.right_arm.hand);
this.drawRect(walker.right_arm.hand);
this.ctx.lineWidth = 1/this.zoom;
}
@@ -151,4 +152,4 @@ class Renderer {
return zoom;
}
}
module.export = {Renderer}
module.exports = {Renderer}
src/js/utils.js
+2 -1
View File
@@ -1,4 +1,5 @@
var randf = (low, high) => Math.random() * (high - low) + low
var randi = (low, high) => (Math.random() * (high - low) + low)//1
function deg2rad(deg) {
return deg / 180 * Math.PI
@@ -29,4 +30,4 @@ function mean(array){
}
module.exports = {deg2rad, randf, MovingAverage, mean}
module.exports = {deg2rad, randf, randi, MovingAverage, mean}
src/js/walker.js
+9 -11
View File
@@ -3,9 +3,9 @@
const b2 = require('../vendor/jsbox2d')
const {
randf,
deg2rad,
MovingAverage
deg2rad
} = require('./utils.js')
const {Renderer} = require('./renderer')
const STRENGTH = 3
@@ -17,8 +17,6 @@ class Walker {
this.floor = floor
this.config = config
this.rewardAverage = new MovingAverage(5000)
this.density = 106.2; // common for all fixtures, no reason to be too specific
this.max_distance = -5;
@@ -85,8 +83,10 @@ class Walker {
}
// if (typeof document!==undefined)
// this.renderer = new Renderer()
if (typeof WEB !== "undefined") {
console.log('rendering', typeof WEB)
this.renderer = new Renderer(this.config, this, this.floor)
}
this.build()
this.initGrip()
@@ -553,9 +553,10 @@ class Walker {
@delta (Float) time since the last update
@action: (Integer) The action to take (can be null if no action)
*/
this.simulationPreStep(motorSpeeds)
for (let i = 0; i < this.config.action_repeat; i++) {
for (let i = 0; i < this.config.action_repeat; i++) {
this.simulationPreStep(motorSpeeds)
this.world.Step(1 / this.config.time_step, this.config.velocity_iterations, this.config.position_iterations);
if (typeof WEB!=="undefined") this.renderer.drawFrame()
}
this.steps++
/* score/reward */
@@ -616,9 +617,6 @@ class Walker {
}
var done = 0
this.world.ClearForces();
this.rewardAverage.add(this.reward)
if (this.steps % 1000 == 0)
console.debug('reward', this.steps, this.rewardAverage.mean())
return [this.getState(), this.reward, done, info]
}
src/train.js
+5 -48
View File
@@ -1,52 +1,9 @@
var config = require('./js/config')
const b2 = require('./vendor/jsbox2d')
const createFloor = require('./js/floor.js')
const DDPGAgent = require('./js/ddpg/ddpg_agent')
const {
Walker
} = require('./js/walker')
var gravity = new b2.Vec2(0, -10)
var world = new b2.World(gravity)
floor = createFloor(world, config.max_floor_tiles);
var env = new Walker(world, floor, config)
const { HeadlessGame } = require('./js/game')
var game = new HeadlessGame(config)
game.agent.train(true);
const nbActions = env.joints.length + 4
const stateSize = env.bodies.length * 10 + env.joints.length * 3
var agent = new DDPGAgent(env, {
stateSize,
nbActions,
resetEpisode: true,
batchSize: 128,
actorLr: 0.0001,
criticLr: 0.001,
memorySize: 30000,
gamma: 0.99,
desiredActionStddev: 0.1,
initialStddev: 0.4,
actorFirstLayerSize: 128,
actorSecondLayerSize: 64,
criticFirstLayerSSize: 128,
criticFirstLayerASize: 128,
criticSecondLayerSize: 64,
nbEpochs: 1000,
nbEpochsCycle: 10,
nbTrainSteps: 100,
maxStep: 800,
saveDuringTraining: true,
saveInterval: 20,
tau: 0.008,
adoptionCoefficient: 1.01,
});
agent.train(true);
agent.save("model-ddpg-traffic");
// game.agent.save("model-ddpg-traffic");
game.agent.save("model-ddpg-walker");
webpack.config.js
+39 -4
View File
@@ -1,9 +1,44 @@
const path = require('path');
const webpack = require('webpack')
module.exports = {
entry: './js/index.js',
entry: {
app: './src/index.js',
// lib: './src/vendor/index.js'
},
target: 'web',
devtool: 'dev-source-map',
output: {
path: path.resolve(__dirname, 'dist'),
filename: 'bundle.js'
}
path: path.join(__dirname),
filename: 'dist/[name].bundle.js',
// path:'/dist',
},
devServer: {
},
optimization: {
splitChunks: {
cacheGroups: {
lib: {
test: /jsbox2d/,
chunks: 'initial',
name: 'lib',
priority:20,
enforce:true
},
vendors: {
test: /[\\/]node_modules[\\/]/,
chunks: 'initial',
name: 'vendor',
priority: 10,
enforce: true,
}
}
}
},
plugins: [
new webpack.DefinePlugin({
// A flag to disable node imports, and enable window/dom usage
WEB: true
})
]
};