diff --git a/js/walker.js b/js/walker.js index a152e6f..d38461d 100644 --- a/js/walker.js +++ b/js/walker.js @@ -6,7 +6,7 @@ function deg2rad(deg) { return deg/180*Math.PI } -const STRENGTH = 6 +const STRENGTH = 2 var Walker = function() { this.__constructor.apply(this, arguments); @@ -109,11 +109,12 @@ Walker.prototype.createTorso = function() { position.y -= this.torso_def.upper_height/2; position.x -= this.torso_def.lower_width/3; jd.Initialize(upper_torso, lower_torso, position); - jd.lowerAngle = deg2rad(-30); - jd.upperAngle = deg2rad(75); + jd.lowerAngle = deg2rad(-30/2); + jd.upperAngle = deg2rad(75 / 2); + jd.user_data = 'torso_joint' jd.enableLimit = true; - jd.maxMotorTorque = 250 * STRENGTH; + jd.maxMotorTorque = 200 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -226,7 +227,7 @@ Walker.prototype.createHead = function() { jd.lowerAngle = -0.1; jd.upperAngle = 0.2; jd.enableLimit = true; - jd.maxMotorTorque = 2 * STRENGTH; + jd.maxMotorTorque = 4 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -276,7 +277,7 @@ Walker.prototype.connectParts = function() { jd.lowerAngle = deg2rad(-10); jd.upperAngle = deg2rad(80); jd.enableLimit = true; - jd.maxMotorTorque = 250 * STRENGTH; + jd.maxMotorTorque = 350 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -286,7 +287,7 @@ Walker.prototype.connectParts = function() { jd.lowerAngle = deg2rad(-10); jd.upperAngle = deg2rad(80); jd.enableLimit = true; - jd.maxMotorTorque = 250 * STRENGTH; + jd.maxMotorTorque = 350 * STRENGTH; jd.motorSpeed = 0; jd.enableMotor = true; this.joints.push(this.world.CreateJoint(jd)); @@ -337,21 +338,25 @@ Walker.prototype.simulationPreStep = function (motorSpeeds) { } Walker.prototype.simulationStep = function (motorSpeeds) { - this.steps ++ + this.steps++ /* score/reward */ // reward copied from OpenAI Gym Humanoid Walker https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid.py // also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67 // https://github.com/openai/gym/blob/master/gym/envs/mujoco/assets/humanoidstandup.xml // reward for keeping head up - var head_height_reward = this.head.head.GetPosition().y * 2; // it's head should be above it's feet 2*(-0.25-2) + var head_height_reward = this.head.head.GetPosition().y * 8; // it's head should be above it's feet 2*(-0.25-2) // reward for moving one leg beyond the other (stepping) var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x; - var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 1:0 + var leg_switch_reward = (left_leg_forward != this.last_left_left_forward) ? 1 : 0 // cost for moving joints to unnatural positions - var joint_angle_cost = - 0.02 * this.joints.map(j=>j.GetJointAngle()-j.GetReferenceAngle()).reduce((o,v)=>o+v*v,0) // - 0.02 * (0 - 20) + + var jointFractionMovement = j => j.GetJointAngle() > 0 ? j.GetJointAngle() / (j.GetUpperLimit() + 1) : j.GetJointAngle() / (j.GetLowerLimit() + 1) + var quad_joint_angle_cost = - 0.004 * this.joints.map(j => jointFractionMovement(j) * 1.2) + .reduce((o, v) => o + v * v, 0) + quad_joint_angle_cost = Math.max(quad_joint_angle_cost, -10) // reward for moving rights var position = this.torso.upper_torso.GetPosition().x @@ -362,28 +367,37 @@ Walker.prototype.simulationStep = function (motorSpeeds) { // punish for using energy, squared var quad_ctrl_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2) + quad_ctrl_cost = Math.max(quad_ctrl_cost, -10) var bonus_happiness = 5 // May they find happiness for all their days // we don't have data on external forces, so I will just punish for contact with the ground + http://blog.sethladd.com/2011/09/box2d-collision-damage-for-javascript.html + // However we could use a listener or calc force + // var listener = new b2.ContactListener() + // listener.PostSolve = function (contact, impulse) { + // if (contact) console.log(contact) + // } + // globals.world.SetContactListener(listener) + // } var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length - quad_impact_cost = -Math.min(contacts - 4, 10)/8 + quad_impact_cost = -Math.min(contacts - 4, 10)/3 this.rewards = { lin_vel_reward, quad_ctrl_cost, quad_impact_cost, - joint_angle_cost, + quad_joint_angle_cost, bonus_happiness, head_height_reward, leg_switch_reward } - this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0) * 10 + this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0) this.last_left_left_forward = left_leg_forward var info = { - x: this.steps, + episodeSteps: this.steps, reward:this.reward, position, ...this.rewards