diff --git a/js/draw.js b/js/draw.js index 9aff55c..8f24a25 100644 --- a/js/draw.js +++ b/js/draw.js @@ -32,9 +32,7 @@ drawFrame = function() { globals.ctx.scale(globals.zoom, -globals.zoom); drawFloor(); for(var k = config.population_size - 1; k >= 0 ; k--) { - if(globals.walkers[k].health > 0) { - drawWalker(globals.walkers[k]); - } + drawWalker(globals.walkers[k]); } globals.ctx.restore(); } @@ -53,7 +51,7 @@ drawFloor = function() { drawWalker = function (walker) { var hue = walker.hue || 240 - globals.ctx.strokeStyle = "hsl("+hue+",100%,"+(90-49*walker.health/config.walker_health)+"%)"; + globals.ctx.strokeStyle = "hsl(" + hue + ",100%,0%)"; globals.ctx.fillStyle = "hsl("+hue+",45%,"+(100-15*walker.health/config.walker_health)+"%)"; globals.ctx.lineWidth = 1/globals.zoom; diff --git a/js/game.js b/js/game.js index 5522b40..6291b5f 100644 --- a/js/game.js +++ b/js/game.js @@ -8,60 +8,86 @@ config = { min_motor_speed: -2, max_motor_speed: 2, population_size: 4, - mutation_chance: 0.1, - mutation_amount: 0.5, - walker_health: 100, // 300 - fitness_criterium: 'score', - check_health: true, - elite_clones: 2, - max_floor_tiles: 50, - round_length: 1200, + walker_health: 100, + max_floor_tiles: 30, + round_length: 8000, min_body_delta: 0, min_leg_delta: 0.0, - instadeath_delta: 0.4 }; globals = {}; +chooseQoute = function () { + var qoutes = [ + 'Play the funky music, robot', + 'The origin of funkd', + 'The chaos computer club', + 'Only the humans that like to dance survived', + 'Classic robot dance move - the human', + 'First we dance Manhatten, then we dance the world', + 'One hour after ingesting substance 1043', + 'Red robot redemption', + 'Father was a rolling robot', + 'Light as a trash can, nimble as a ox', + 'Have you tried turning it off and on again?', + 'Eurovision 2050', + ] + var qoute = qoutes[Math.randi(0,qoutes.length)] + document.getElementById('page_quote').innerText = '"'+qoute+'"' + +} + +displayProgress = function () { + // TODO show stats + var stats = { + 'trainingTime': globals.step_counter / config.simulation_fps, + 'meanProgress': globals.walkers.map(w => w.last_position).reduce((s, v) => s + v) / globals.walkers.length, + 'meanReward': globals.walkers.map(w => w.reward).reduce((s, v) => s + v) / globals.walkers.length, + } + document.getElementById('stats-prog').innerText = JSON.stringify(stats, null, 2) + document.getElementById('stats-ag0').innerText = JSON.stringify(globals.walkers[0].rewards, null, 2) +} + gameInit = function() { var joints = 12 var bodyParts = 14 - var sensors = 14 - var state = sensors * 7 + var state = bodyParts * 7 var actions = joints var input = 2 * state + 1 * actions + globals.brains = { - actor: new window.neurojs.Network.Model([ - { type: 'input', size: input }, - { type: 'fc', size: 60, activation: 'relu' }, - { type: 'fc', size: 40, activation: 'relu' }, - { type: 'fc', size: 40, activation: 'relu', dropout: 0.30 }, - { type: 'fc', size: actions, activation: 'tanh' }, - { type: 'regression' } - - ]), - - critic: new window.neurojs.Network.Model([ - - { type: 'input', size: input + actions }, - { type: 'fc', size: 80, activation: 'relu' }, - { type: 'fc', size: 70, activation: 'relu' }, - { type: 'fc', size: 60, activation: 'relu' }, - { type: 'fc', size: 50, activation: 'relu' }, - { type: 'fc', size: 1 }, - { type: 'regression' } - - ]) - + actor: new window.neurojs.Network.Model([ + { type: 'input', size: input }, + { type: 'fc', size: 60, activation: 'relu' }, + { type: 'fc', size: 40, activation: 'relu' }, + { type: 'fc', size: 40, activation: 'relu', dropout: 0.30 }, + { type: 'fc', size: actions, activation: 'tanh' }, + { type: 'regression' } + + ]), + + critic: new window.neurojs.Network.Model([ + + { type: 'input', size: input + actions }, + { type: 'fc', size: 80, activation: 'relu' }, + { type: 'fc', size: 70, activation: 'relu' }, + { type: 'fc', size: 60, activation: 'relu' }, + { type: 'fc', size: 50, activation: 'relu' }, + { type: 'fc', size: 1 }, + { type: 'regression' } + + ]) + } - + globals.brains.shared = new window.neurojs.Shared.ConfigPool() - + // this.brains.shared.set('actor', this.brains.actor.newConfiguration()) globals.brains.shared.set('critic', globals.brains.critic.newConfiguration()) - - + + + chooseQoute() globals.world = new b2.World(new b2.Vec2(0, -10)); [globals.agents, globals.walkers] = createPopulation(); @@ -71,17 +97,17 @@ gameInit = function() { globals.step_counter = 0; globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps)); globals.draw_interval = setInterval(drawFrame, Math.round(1000 / config.draw_fps)); - globals.reset_interval = setInterval(resetSimulation, Math.round(8000 * 1000 / config.draw_fps)); + globals.reset_interval = setInterval(resetSimulation, Math.round(config.round_length * 1000 / config.draw_fps)); globals.logr_interval = setInterval(logRewards, Math.round(800 * 1000 / config.draw_fps)); + globals.display_interval = setInterval(displayProgress, Math.round(80 * 1000 / config.draw_fps)); } logRewards = function () { for(var k = 0; k < config.population_size; k++) { - console.log(k, globals.walkers[k].rewards) + console.table(globals.walkers[k].rewards) } } - resetSimulation = function () { // turn training off temporarlity to avoid NaN's updateIfLearning(false) @@ -92,34 +118,34 @@ resetSimulation = function () { setTimeout(()=>updateIfLearning(true), 1000) } -simulationStep = function() { +simulationStep = function () { + globals.step_counter++; + + // step world globals.world.Step(1/config.time_step, config.velocity_iterations, config.position_iterations); globals.world.ClearForces(); + + // step agents populationSimulationStep(); - if(typeof globals.step_counter == 'undefined') { - globals.step_counter = 0; - } else { - globals.step_counter++; - } } -setSimulationFps = function(fps) { - config.simulation_fps = fps; - clearInterval(globals.simulation_interval); - if(fps > 0) { - globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps)); - if(globals.paused) { - globals.paused = false; - if(config.draw_fps > 0) { - globals.draw_interval = setInterval(drawFrame, Math.round(1000/config.draw_fps)); - } - } - } else { - // pause the drawing as well - clearInterval(globals.draw_interval); - globals.paused = true; - } -} +// setSimulationFps = function(fps) { +// config.simulation_fps = fps; +// clearInterval(globals.simulation_interval); +// if(fps > 0) { +// globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps)); +// if(globals.paused) { +// globals.paused = false; +// if(config.draw_fps > 0) { +// globals.draw_interval = setInterval(drawFrame, Math.round(1000/config.draw_fps)); +// } +// } +// } else { +// // pause the drawing as well +// clearInterval(globals.draw_interval); +// globals.paused = true; +// } +// } createPopulation = function(genomes) { var walkers = []; @@ -165,7 +191,7 @@ function saveAs(dv, name) { downloadBrain = function (n) { var ts = (new Date()).toISOString().replace(':','_') var buf = globals.agents[n].brain.export() - saveAs(new DataView(buf), 'walker_brain'+ts+'.bin') + saveAs(new DataView(buf), 'walker_brain_'+n+'_'+ts+'.bin') }; @@ -179,7 +205,6 @@ readBrain = function (buf) { for (var i = 0; i < globals.agents.length; i++) { globals.agents[i].brain.algorithm.actor.set(imported.actor.clone()) globals.agents[i].brain.algorithm.critic.set(imported.critic) - // window.gcd.world.agents[i].car.brain.learning = false } }; diff --git a/js/walker.js b/js/walker.js index 23049a8..bb06225 100644 --- a/js/walker.js +++ b/js/walker.js @@ -19,7 +19,7 @@ Walker.prototype.__constructor = function(world) { this.density = 106.2; // common for all fixtures, no reason to be too specific this.max_distance = -5; - this.health = config.walker_health; + this.health = 10 //config.walker_health; this.score = 0; this.low_foot_height = 0; this.head_height = 0; @@ -337,31 +337,33 @@ Walker.prototype.simulationPreStep = function (motorSpeeds) { } Walker.prototype.simulationStep = function (motorSpeeds) { - /* score/reward */ - var head_height_reward = this.head.head.GetPosition().y * 2; // it's head should be above it's feet 2*(-0.25-2) - - // TODO reward for moving one leg beyond the other? - var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x; - var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 1:0 - - var joint_angle_cost = - 0.02 * this.joints.map(j=>j.GetJointAngle()-j.GetReferenceAngle()).reduce((o,v)=>o+v*v,0) // - 0.02 * (0 - 20) - // reward copied from OpenAI Gym Humanoid Walker https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid.py // also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67 // https://github.com/openai/gym/blob/master/gym/envs/mujoco/assets/humanoidstandup.xml + + // reward for keeping head up + var head_height_reward = this.head.head.GetPosition().y * 2; // it's head should be above it's feet 2*(-0.25-2) + + // reward for moving one leg beyond the other (stepping) + var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x; + var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 1:0 + + // cost for moving joints to unnatural positions + var joint_angle_cost = - 0.02 * this.joints.map(j=>j.GetJointAngle()-j.GetReferenceAngle()).reduce((o,v)=>o+v*v,0) // - 0.02 * (0 - 20) + + // reward for moving rights var position = this.torso.upper_torso.GetPosition().x if (this.last_position === undefined) this.last_position = position var velocity = (position - this.last_position) * 40 this.last_position = position - // var velocity = this.torso.upper_torso.GetLinearVelocity().x lin_vel_reward = 6 * velocity // punish for using energy, squared var quad_ctrl_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2) - var alive_bonus = 5 // May they find happiness for all their days + var bonus_happiness = 5 // May they find happiness for all their days - // we don't have data on external forces, so I will just punish for contact + // we don't have data on external forces, so I will just punish for contact with the ground var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length quad_impact_cost = -Math.min(contacts - 4, 10)/8 @@ -370,12 +372,12 @@ Walker.prototype.simulationStep = function (motorSpeeds) { quad_ctrl_cost, quad_impact_cost, joint_angle_cost, - alive_bonus, + bonus_happiness, head_height_reward, leg_switch_reward } - this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0)/10 + this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0) * 10 this.last_left_left_forward = left_leg_forward