change reward to be head height & forward velocity

README.md

@@ -18,14 +18,7 @@ TODO: Write usage instructions
 4. Push to the branch: `git push origin my-new-feature`
 5. Submit a pull request :D
 
-## History
+## TODO
 
-TODO: Write history
-
-## Credits
-
-TODO: Write credits
-
-## License
-
-TODO: Write license
+- display/save time for each agent
+- display save loss curves

js/agent.js

@@ -69,15 +69,17 @@ Agent.prototype.step = function () {
     if (this.timer % this.timerFrequency === 0) {
         
         // reward from last step
-        this.reward = this.walker.score * 10
-        // console.log(this.reward)
-        this.walker.score = 0
+        this.reward = this.walker.reward
+        // this.reward = this.walker.score * 10
+        // // punish for using energy
+        // this.reward -= this.walker.joints.map(j=>j.GetJointSpeed()).reduce((sum,speed)=>sum+speed**2)
+        // // console.log(this.reward)
+        // this.walker.score = 0
         
         // train
-        this.loss = this.brain.learn(this.reward)
+        this.loss = this.brain.learn(this.reward) 
         this.action = this.brain.policy(this.walker.getState())
     }
-    
     if (this.action) {
         this.walker.simulationStep(this.action)
     }

js/game.js

@@ -10,14 +10,14 @@ config = {
   population_size: 4,
   mutation_chance: 0.1,
   mutation_amount: 0.5,
-  walker_health: 300,
+  walker_health: 100, // 300
   fitness_criterium: 'score',
   check_health: true,
   elite_clones: 2,
   max_floor_tiles: 50,
   round_length: 1200,
   min_body_delta: 0,
-  min_leg_delta: 0.4,
+  min_leg_delta: 0.0,
   instadeath_delta: 0.4
 };
 
@@ -71,7 +71,14 @@ gameInit = function() {
   globals.step_counter = 0;
   globals.simulation_interval = setInterval(simulationStep, Math.round(1000/config.simulation_fps));
   globals.draw_interval = setInterval(drawFrame, Math.round(1000 / config.draw_fps));
-  globals.draw_interval = setInterval(resetSimulation, Math.round(8000 * 1000 / config.draw_fps));
+  globals.reset_interval = setInterval(resetSimulation, Math.round(8000 * 1000 / config.draw_fps));
+  globals.logr_interval = setInterval(logRewards, Math.round(800 * 1000 / config.draw_fps));
+}
+
+logRewards = function () { 
+  for(var k = 0; k < config.population_size; k++) {
+    console.log(k, globals.walkers[k].rewards)
+  }
 }
 
 resetSimulation = function () { 

js/walker.js

@@ -17,6 +17,8 @@ Walker.prototype.__constructor = function(world) {
   this.low_foot_height = 0;
   this.head_height = 0;
   this.steps = 0;
+  this.distance = 0 
+  this.last_left_left_forward = true
 
   this.hue = Math.randf(200,360)
 
@@ -310,41 +312,41 @@ Walker.prototype.simulationStep = function (motorSpeeds) {
   
   // act
   for(var k = 0; k < this.joints.length; k++) {
-    // var oldSpeed = this.joints[k].GetMotorSpeed()
-    // this.joints[k].SetMotorSpeed(oldSpeed + motorSpeeds[k]/10); // action can range from -3 to 3, radians per second
-    this.joints[k].SetMotorSpeed(motorSpeeds[k]*90); // action can range from -3 to 3, radians per second
+    this.joints[k].SetMotorSpeed(motorSpeeds[k]*3); // action can range from -3 to 3, radians per second
   }
-  var oldmax = this.max_distance;
-  var distance = this.torso.upper_torso.GetPosition().x;
-  this.max_distance = Math.max(this.max_distance, distance);
 
   /* score/reward */
-  // it's head should be above it's feet
-  this.head_height = this.head.head.GetPosition().y; 
-  this.low_foot_height = Math.min(this.left_leg.foot.GetPosition().y, this.right_leg.foot.GetPosition().y);
-  var body_delta = this.head_height-this.low_foot_height;
-  var leg_delta = this.right_leg.foot.GetPosition().x - this.left_leg.foot.GetPosition().x;
+  var head_height_reward = this.head.head.GetPosition().y;  // it's head should be above it's feet
 
-  if(body_delta > config.min_body_delta) {
-    this.score += body_delta/50;
-    if(this.max_distance > oldmax) {
-      if(Math.abs(leg_delta) > config.min_leg_delta && this.head.head.m_linearVelocity.y > -2) {
-        if(typeof this.leg_delta_sign == 'undefined') {
-          this.leg_delta_sign = leg_delta/Math.abs(leg_delta);
-        } else if(this.leg_delta_sign * leg_delta < 0) {
-          this.leg_delta_sign = leg_delta/Math.abs(leg_delta);
-          this.steps++;
-          this.score += 100;
-          this.score += this.max_distance;
-          this.health = config.walker_health;
-        }
-      }
-    }
+  // TODO reward for moving one leg beyond the other?
+  var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x;
+  var leg_switch_reward = (left_leg_forward!=this.last_left_left_forward)? 5:0
+
+  // reward copied from OpenAI Gym Humanoid Walker https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid.py
+  // also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67
+  var velocity = this.torso.upper_torso.GetLinearVelocity().x
+  lin_vel_reward = 6 * velocity
+
+  // punish for using energy, squared
+  var quad_ctrl_cost = -0.1 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2)
+  var alive_bonus = 5
+
+  // we don't have data on external forces, so I will just punish for contact
+  var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length
+  quad_impact_cost = -Math.min(contacts - 2, 10)/4
+
+  this.rewards = {
+    lin_vel_reward,
+    quad_ctrl_cost,
+    quad_impact_cost,
+    alive_bonus,
+    head_height_reward,
+    leg_switch_reward
   }
-  this.health = this.score * 800 + 10
-  // console.log(body_delta, leg_delta)
+  
+  this.reward = Object.values(this.rewards).reduce((tot,v)=>tot+v, 0)
 
+  this.last_left_left_forward = left_leg_forward
 
-
-  return;
+  return
 }