diff --git a/README.md b/README.md index ccb831c..960f337 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,14 @@ TODO: Write usage instructions - make diff env's, copy humanoidwalker, humanoid, learning2run etc - display - display/save time for each agent + - globals.step_counter + - x progress - smile frown? - save checkpoints - +- [ ] obstacles later on +- [ ] load pretrained +- [ ] reward hacking + - [ ] head high, feet on ground, legs crossing, moving forward, angles near 0 # Credits diff --git a/js/floor.js b/js/floor.js index ef4fd76..a25ddca 100644 --- a/js/floor.js +++ b/js/floor.js @@ -17,7 +17,7 @@ function createFloor() { // add uneven floor by continuing from the last point, plus some random jittering edges.push(new b2.Vec2( edges[edges.length - 1].x + (1 + ratio * Math.random() - ratio / 2), - edges[edges.length - 1].y + (ratio * Math.random() - ratio / 2)* 2)); + edges[edges.length - 1].y + (ratio * Math.random() - ratio / 2))); // edges.push(new b2.Vec2(edges[edges.length-1].x + 1,-0.16)); } edges.push(new b2.Vec2(edges[edges.length-1].x, edges[edges.length-1].y + 8)); // front wall diff --git a/js/game.js b/js/game.js index 6ed3521..341c297 100644 --- a/js/game.js +++ b/js/game.js @@ -11,8 +11,8 @@ config = { max_motor_speed: 2, population_size: 4, walker_health: 100, - max_floor_tiles: 30, - round_length: 18000, + max_floor_tiles: 50, + round_length: 4000, min_body_delta: 0, min_leg_delta: 0.0, }; @@ -30,9 +30,12 @@ chooseQoute = function () { 'Video of subjects one hour after ingesting substance q1043', 'Red robot redemption', 'Father was a rolling robot', - 'Light as a trash can, nimble as a ox', + 'Float like a bumblebee, string like a butterfly', + 'Dance evolution', 'Have you tried turning it off and on again?', 'Eurovision 2050', + 'Humans must learn to crawl then walk. Robots break dance then walk', + '' ] var qoute = qoutes[Math.randi(0,qoutes.length)] document.getElementById('page_quote').innerText = '"'+qoute+'"' diff --git a/js/walker.js b/js/walker.js index b78253c..f9d6599 100644 --- a/js/walker.js +++ b/js/walker.js @@ -113,12 +113,11 @@ Walker.prototype.createTorso = function() { position.y -= this.torso_def.upper_height/2; position.x -= this.torso_def.lower_width/3; jd.Initialize(upper_torso, lower_torso, position); - jd.lowerAngle = deg2rad(-30/2); - jd.upperAngle = deg2rad(75 / 2); - jd.user_data = 'torso_joint' - + jd.lowerAngle = deg2rad(-75/2); + jd.upperAngle = deg2rad(30 / 2); + jd.user_data = 'torso_joint' jd.enableLimit = true; - jd.maxMotorTorque = 200 * STRENGTH; + jd.maxMotorTorque = 100 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -160,6 +159,7 @@ Walker.prototype.createLeg = function() { jd.maxMotorTorque = 160 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; + jd.user_data = 'leg_joint' this.joints.push(this.world.CreateJoint(jd)); // foot joint @@ -171,6 +171,7 @@ Walker.prototype.createLeg = function() { jd.enableLimit = true; jd.maxMotorTorque = 70 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'foot_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -202,6 +203,7 @@ Walker.prototype.createArm = function() { jd.enableLimit = true; jd.maxMotorTorque = 100 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'arm_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -234,6 +236,7 @@ Walker.prototype.createHead = function() { jd.maxMotorTorque = 4 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; + jd.user_data = 'neck_joint' this.joints.push(this.world.CreateJoint(jd)); return {head: head, neck: neck}; @@ -260,6 +263,7 @@ Walker.prototype.connectParts = function() { jd.enableLimit = true; jd.maxMotorTorque = 200 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'shoulder_right_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -270,6 +274,7 @@ Walker.prototype.connectParts = function() { jd.enableLimit = true; jd.maxMotorTorque = 200 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'shoulder_right_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -283,6 +288,7 @@ Walker.prototype.connectParts = function() { jd.enableLimit = true; jd.maxMotorTorque = 350 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'waist_right_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -293,6 +299,7 @@ Walker.prototype.connectParts = function() { jd.enableLimit = true; jd.maxMotorTorque = 350 * STRENGTH; jd.motorSpeed = 0; + jd.user_data = 'waist_left_joint' jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); } @@ -369,31 +376,35 @@ Walker.prototype.simulationStep = function (motorSpeeds) { // also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67 // https://github.com/openai/gym/blob/master/gym/envs/mujoco/assets/humanoidstandup.xml - // reward for keeping head up - var head_height_reward = this.head.head.GetPosition().y * 8; // it's head should be above it's feet 2*(-0.25-2) + // reward for keeping head up, compared to feet + var mean_foot_height = (this.left_leg.foot.GetPosition().y + this.right_leg.foot.GetPosition().y)/2 + + var head_height_reward = (this.head.head.GetPosition().y - mean_foot_height)* 20; // it's head should be above it's feet 2*(-0.25-2) // reward for moving one leg beyond the other (stepping) var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x; - var leg_switch_reward = (left_leg_forward != this.last_left_left_forward) ? 1 : 0 + var leg_switch_reward = (left_leg_forward != this.last_left_left_forward) ? 4 : 0 + this.last_left_left_forward = left_leg_forward - // cost for moving joints to unnatural positions - - var jointFractionMovement = j => j.GetJointAngle() > 0 ? j.GetJointAngle() / (j.GetUpperLimit() + 1) : j.GetJointAngle() / (j.GetLowerLimit() + 1) - var quad_joint_angle_cost = - 0.004 * this.joints.map(j => jointFractionMovement(j) * 1.2) + // cost for moving joints to unnatural positions (fraction of movement range in the relevant direction) + var jointFractionMovement = j => j.GetJointAngle() > 0 ? j.GetJointAngle() / (j.GetUpperLimit() + 1) : j.GetJointAngle() / (j.GetLowerLimit() - 1) + var quad_joint_angle_cost = - 0.15 * this.joints.map(j => jointFractionMovement(j) * 1.2) .reduce((o, v) => o + v * v, 0) quad_joint_angle_cost = Math.max(quad_joint_angle_cost, -10) - // reward for moving rights + // reward for moving right var position = this.torso.upper_torso.GetPosition().x if (this.last_position === undefined) this.last_position = position - var velocity = (position - this.last_position) * 40 + var velocity = (position - this.last_position) * 130 this.last_position = position lin_vel_reward = 6 * velocity // punish for using energy, squared - var quad_ctrl_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2) - quad_ctrl_cost = Math.max(quad_ctrl_cost, -10) - var bonus_happiness = 5 // May they find happiness for all their days + var quad_power_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2) + quad_power_cost = Math.max(quad_power_cost, -10) + + // Lets be nice, all entities should find overall happiness in what they do + var bonus_happiness = 5 // we don't have data on external forces, so I will just punish for contact with the ground http://blog.sethladd.com/2011/09/box2d-collision-damage-for-javascript.html @@ -405,12 +416,12 @@ Walker.prototype.simulationStep = function (motorSpeeds) { // globals.world.SetContactListener(listener) // } var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length - quad_impact_cost = -Math.min(contacts - 4, 10)/3 + quad_contact_cost = -Math.min(contacts - 4, 10)/2 this.rewards = { lin_vel_reward, - quad_ctrl_cost, - quad_impact_cost, + quad_power_cost, + quad_contact_cost, quad_joint_angle_cost, bonus_happiness, head_height_reward, @@ -419,8 +430,6 @@ Walker.prototype.simulationStep = function (motorSpeeds) { this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0) - this.last_left_left_forward = left_leg_forward - var info = { episodeSteps: this.steps, reward:this.reward,