mirror of
https://github.com/wassname/rl_2d_walker.js.git
synced 2026-09-09 11:33:20 +08:00
clean outline
This commit is contained in:
wassname
+2
-4
@@ -32,9 +32,7 @@ drawFrame = function() {
|
||||
globals.ctx.scale(globals.zoom, -globals.zoom);
|
||||
drawFloor();
|
||||
for(var k = config.population_size - 1; k >= 0 ; k--) {
|
||||
if(globals.walkers[k].health > 0) {
|
||||
drawWalker(globals.walkers[k]);
|
||||
}
|
||||
drawWalker(globals.walkers[k]);
|
||||
}
|
||||
globals.ctx.restore();
|
||||
}
|
||||
@@ -53,7 +51,7 @@ drawFloor = function() {
|
||||
|
||||
drawWalker = function (walker) {
|
||||
var hue = walker.hue || 240
|
||||
globals.ctx.strokeStyle = "hsl("+hue+",100%,"+(90-49*walker.health/config.walker_health)+"%)";
|
||||
globals.ctx.strokeStyle = "hsl(" + hue + ",100%,0%)";
|
||||
globals.ctx.fillStyle = "hsl("+hue+",45%,"+(100-15*walker.health/config.walker_health)+"%)";
|
||||
globals.ctx.lineWidth = 1/globals.zoom;
|
||||
|
||||
|
||||
+90
-65
@@ -8,60 +8,86 @@ config = {
|
||||
min_motor_speed: -2,
|
||||
max_motor_speed: 2,
|
||||
population_size: 4,
|
||||
mutation_chance: 0.1,
|
||||
mutation_amount: 0.5,
|
||||
walker_health: 100, // 300
|
||||
fitness_criterium: 'score',
|
||||
check_health: true,
|
||||
elite_clones: 2,
|
||||
max_floor_tiles: 50,
|
||||
round_length: 1200,
|
||||
walker_health: 100,
|
||||
max_floor_tiles: 30,
|
||||
round_length: 8000,
|
||||
min_body_delta: 0,
|
||||
min_leg_delta: 0.0,
|
||||
instadeath_delta: 0.4
|
||||
};
|
||||
|
||||
globals = {};
|
||||
|
||||
chooseQoute = function () {
|
||||
var qoutes = [
|
||||
'Play the funky music, robot',
|
||||
'The origin of funkd',
|
||||
'The chaos computer club',
|
||||
'Only the humans that like to dance survived',
|
||||
'Classic robot dance move - the human',
|
||||
'First we dance Manhatten, then we dance the world',
|
||||
'One hour after ingesting substance 1043',
|
||||
'Red robot redemption',
|
||||
'Father was a rolling robot',
|
||||
'Light as a trash can, nimble as a ox',
|
||||
'Have you tried turning it off and on again?',
|
||||
'Eurovision 2050',
|
||||
]
|
||||
var qoute = qoutes[Math.randi(0,qoutes.length)]
|
||||
document.getElementById('page_quote').innerText = '"'+qoute+'"'
|
||||
|
||||
}
|
||||
|
||||
displayProgress = function () {
|
||||
// TODO show stats
|
||||
var stats = {
|
||||
'trainingTime': globals.step_counter / config.simulation_fps,
|
||||
'meanProgress': globals.walkers.map(w => w.last_position).reduce((s, v) => s + v) / globals.walkers.length,
|
||||
'meanReward': globals.walkers.map(w => w.reward).reduce((s, v) => s + v) / globals.walkers.length,
|
||||
}
|
||||
document.getElementById('stats-prog').innerText = JSON.stringify(stats, null, 2)
|
||||
document.getElementById('stats-ag0').innerText = JSON.stringify(globals.walkers[0].rewards, null, 2)
|
||||
}
|
||||
|
||||
gameInit = function() {
|
||||
var joints = 12
|
||||
var bodyParts = 14
|
||||
var sensors = 14
|
||||
var state = sensors * 7
|
||||
var state = bodyParts * 7
|
||||
var actions = joints
|
||||
var input = 2 * state + 1 * actions
|
||||
|
||||
|
||||
globals.brains = {
|
||||
actor: new window.neurojs.Network.Model([
|
||||
{ type: 'input', size: input },
|
||||
{ type: 'fc', size: 60, activation: 'relu' },
|
||||
{ type: 'fc', size: 40, activation: 'relu' },
|
||||
{ type: 'fc', size: 40, activation: 'relu', dropout: 0.30 },
|
||||
{ type: 'fc', size: actions, activation: 'tanh' },
|
||||
{ type: 'regression' }
|
||||
|
||||
]),
|
||||
|
||||
critic: new window.neurojs.Network.Model([
|
||||
|
||||
{ type: 'input', size: input + actions },
|
||||
{ type: 'fc', size: 80, activation: 'relu' },
|
||||
{ type: 'fc', size: 70, activation: 'relu' },
|
||||
{ type: 'fc', size: 60, activation: 'relu' },
|
||||
{ type: 'fc', size: 50, activation: 'relu' },
|
||||
{ type: 'fc', size: 1 },
|
||||
{ type: 'regression' }
|
||||
|
||||
])
|
||||
|
||||
actor: new window.neurojs.Network.Model([
|
||||
{ type: 'input', size: input },
|
||||
{ type: 'fc', size: 60, activation: 'relu' },
|
||||
{ type: 'fc', size: 40, activation: 'relu' },
|
||||
{ type: 'fc', size: 40, activation: 'relu', dropout: 0.30 },
|
||||
{ type: 'fc', size: actions, activation: 'tanh' },
|
||||
{ type: 'regression' }
|
||||
|
||||
]),
|
||||
|
||||
critic: new window.neurojs.Network.Model([
|
||||
|
||||
{ type: 'input', size: input + actions },
|
||||
{ type: 'fc', size: 80, activation: 'relu' },
|
||||
{ type: 'fc', size: 70, activation: 'relu' },
|
||||
{ type: 'fc', size: 60, activation: 'relu' },
|
||||
{ type: 'fc', size: 50, activation: 'relu' },
|
||||
{ type: 'fc', size: 1 },
|
||||
{ type: 'regression' }
|
||||
|
||||
])
|
||||
|
||||
}
|
||||
|
||||
|
||||
globals.brains.shared = new window.neurojs.Shared.ConfigPool()
|
||||
|
||||
|
||||
// this.brains.shared.set('actor', this.brains.actor.newConfiguration())
|
||||
globals.brains.shared.set('critic', globals.brains.critic.newConfiguration())
|
||||
|
||||
|
||||
|
||||
|
||||
chooseQoute()
|
||||
globals.world = new b2.World(new b2.Vec2(0, -10));
|
||||
[globals.agents, globals.walkers] = createPopulation();
|
||||
|
||||
@@ -71,17 +97,17 @@ gameInit = function() {
|
||||
globals.step_counter = 0;
|
||||
globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps));
|
||||
globals.draw_interval = setInterval(drawFrame, Math.round(1000 / config.draw_fps));
|
||||
globals.reset_interval = setInterval(resetSimulation, Math.round(8000 * 1000 / config.draw_fps));
|
||||
globals.reset_interval = setInterval(resetSimulation, Math.round(config.round_length * 1000 / config.draw_fps));
|
||||
globals.logr_interval = setInterval(logRewards, Math.round(800 * 1000 / config.draw_fps));
|
||||
globals.display_interval = setInterval(displayProgress, Math.round(80 * 1000 / config.draw_fps));
|
||||
}
|
||||
|
||||
logRewards = function () {
|
||||
for(var k = 0; k < config.population_size; k++) {
|
||||
console.log(k, globals.walkers[k].rewards)
|
||||
console.table(globals.walkers[k].rewards)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
resetSimulation = function () {
|
||||
// turn training off temporarlity to avoid NaN's
|
||||
updateIfLearning(false)
|
||||
@@ -92,34 +118,34 @@ resetSimulation = function () {
|
||||
setTimeout(()=>updateIfLearning(true), 1000)
|
||||
}
|
||||
|
||||
simulationStep = function() {
|
||||
simulationStep = function () {
|
||||
globals.step_counter++;
|
||||
|
||||
// step world
|
||||
globals.world.Step(1/config.time_step, config.velocity_iterations, config.position_iterations);
|
||||
globals.world.ClearForces();
|
||||
|
||||
// step agents
|
||||
populationSimulationStep();
|
||||
if(typeof globals.step_counter == 'undefined') {
|
||||
globals.step_counter = 0;
|
||||
} else {
|
||||
globals.step_counter++;
|
||||
}
|
||||
}
|
||||
|
||||
setSimulationFps = function(fps) {
|
||||
config.simulation_fps = fps;
|
||||
clearInterval(globals.simulation_interval);
|
||||
if(fps > 0) {
|
||||
globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps));
|
||||
if(globals.paused) {
|
||||
globals.paused = false;
|
||||
if(config.draw_fps > 0) {
|
||||
globals.draw_interval = setInterval(drawFrame, Math.round(1000/config.draw_fps));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// pause the drawing as well
|
||||
clearInterval(globals.draw_interval);
|
||||
globals.paused = true;
|
||||
}
|
||||
}
|
||||
// setSimulationFps = function(fps) {
|
||||
// config.simulation_fps = fps;
|
||||
// clearInterval(globals.simulation_interval);
|
||||
// if(fps > 0) {
|
||||
// globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps));
|
||||
// if(globals.paused) {
|
||||
// globals.paused = false;
|
||||
// if(config.draw_fps > 0) {
|
||||
// globals.draw_interval = setInterval(drawFrame, Math.round(1000/config.draw_fps));
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// // pause the drawing as well
|
||||
// clearInterval(globals.draw_interval);
|
||||
// globals.paused = true;
|
||||
// }
|
||||
// }
|
||||
|
||||
createPopulation = function(genomes) {
|
||||
var walkers = [];
|
||||
@@ -165,7 +191,7 @@ function saveAs(dv, name) {
|
||||
downloadBrain = function (n) {
|
||||
var ts = (new Date()).toISOString().replace(':','_')
|
||||
var buf = globals.agents[n].brain.export()
|
||||
saveAs(new DataView(buf), 'walker_brain'+ts+'.bin')
|
||||
saveAs(new DataView(buf), 'walker_brain_'+n+'_'+ts+'.bin')
|
||||
};
|
||||
|
||||
|
||||
@@ -179,7 +205,6 @@ readBrain = function (buf) {
|
||||
for (var i = 0; i < globals.agents.length; i++) {
|
||||
globals.agents[i].brain.algorithm.actor.set(imported.actor.clone())
|
||||
globals.agents[i].brain.algorithm.critic.set(imported.critic)
|
||||
// window.gcd.world.agents[i].car.brain.learning = false
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
+17
-15
@@ -19,7 +19,7 @@ Walker.prototype.__constructor = function(world) {
|
||||
this.density = 106.2; // common for all fixtures, no reason to be too specific
|
||||
|
||||
this.max_distance = -5;
|
||||
this.health = config.walker_health;
|
||||
this.health = 10 //config.walker_health;
|
||||
this.score = 0;
|
||||
this.low_foot_height = 0;
|
||||
this.head_height = 0;
|
||||
@@ -337,31 +337,33 @@ Walker.prototype.simulationPreStep = function (motorSpeeds) {
|
||||
}
|
||||
|
||||
Walker.prototype.simulationStep = function (motorSpeeds) {
|
||||
|
||||
/* score/reward */
|
||||
var head_height_reward = this.head.head.GetPosition().y * 2; // it's head should be above it's feet 2*(-0.25-2)
|
||||
|
||||
// TODO reward for moving one leg beyond the other?
|
||||
var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x;
|
||||
var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 1:0
|
||||
|
||||
var joint_angle_cost = - 0.02 * this.joints.map(j=>j.GetJointAngle()-j.GetReferenceAngle()).reduce((o,v)=>o+v*v,0) // - 0.02 * (0 - 20)
|
||||
|
||||
// reward copied from OpenAI Gym Humanoid Walker https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid.py
|
||||
// also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67
|
||||
// https://github.com/openai/gym/blob/master/gym/envs/mujoco/assets/humanoidstandup.xml
|
||||
|
||||
// reward for keeping head up
|
||||
var head_height_reward = this.head.head.GetPosition().y * 2; // it's head should be above it's feet 2*(-0.25-2)
|
||||
|
||||
// reward for moving one leg beyond the other (stepping)
|
||||
var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x;
|
||||
var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 1:0
|
||||
|
||||
// cost for moving joints to unnatural positions
|
||||
var joint_angle_cost = - 0.02 * this.joints.map(j=>j.GetJointAngle()-j.GetReferenceAngle()).reduce((o,v)=>o+v*v,0) // - 0.02 * (0 - 20)
|
||||
|
||||
// reward for moving rights
|
||||
var position = this.torso.upper_torso.GetPosition().x
|
||||
if (this.last_position === undefined) this.last_position = position
|
||||
var velocity = (position - this.last_position) * 40
|
||||
this.last_position = position
|
||||
// var velocity = this.torso.upper_torso.GetLinearVelocity().x
|
||||
lin_vel_reward = 6 * velocity
|
||||
|
||||
// punish for using energy, squared
|
||||
var quad_ctrl_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2)
|
||||
var alive_bonus = 5 // May they find happiness for all their days
|
||||
var bonus_happiness = 5 // May they find happiness for all their days
|
||||
|
||||
// we don't have data on external forces, so I will just punish for contact
|
||||
// we don't have data on external forces, so I will just punish for contact with the ground
|
||||
var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length
|
||||
quad_impact_cost = -Math.min(contacts - 4, 10)/8
|
||||
|
||||
@@ -370,12 +372,12 @@ Walker.prototype.simulationStep = function (motorSpeeds) {
|
||||
quad_ctrl_cost,
|
||||
quad_impact_cost,
|
||||
joint_angle_cost,
|
||||
alive_bonus,
|
||||
bonus_happiness,
|
||||
head_height_reward,
|
||||
leg_switch_reward
|
||||
}
|
||||
|
||||
this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0)/10
|
||||
this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0) * 10
|
||||
|
||||
this.last_left_left_forward = left_leg_forward
|
||||
|
||||
|
||||
Reference in New Issue
Block a user