tweaks

.gitignore

@@ -1,5 +1,6 @@
 
 /outputs/
+/checkpoints/
 
 # Created by https://www.gitignore.io/api/node,linux
 # Edit at https://www.gitignore.io/?templates=node,linux

scripts/train.js

@@ -3,6 +3,7 @@ var config = require('../src/js/config')
 
 const { HeadlessGame } = require('../src/js/game')
 var game = new HeadlessGame(config)
+// game.agent.restore('../checkpoints', 'model-ddpg-walker-22h/model') // load checkpoint
 game.agent.restore('./outputs', 'model-ddpg-walker/model') // resume
 game.agent.train(true); 
 

src/index.html

@@ -66,6 +66,7 @@
 function init() {
   var canvas_id = 'main_screen2'
   window.game = new Game(config, canvas_id)
+  game.agent.restore('./checkpoints', 'model-ddpg-walker-4h/model') // load checkpoint
   game.agent.restore('./checkpoints', 'model-ddpg-walker-60h/model') // load checkpoint
   game.agent.restore('../outputs', 'model-ddpg-walker/model') // load latest
   game.loop()

src/index.js

@@ -1,15 +1,7 @@
-const { Game } = require('./js/game')
+const { Game, chooseQoute } = require('./js/game')
 window.b2 = require('./vendor/jsbox2d')
-const config = require('./js/config')
+window.config = require('./js/config')
+window.chooseQoute = chooseQoute
+window.Game = Game
 
-require('./css/walkers.css')
-
-var game
-window.game = game
-
-function init() {
-  window.game = new Game(config)
-}
-
-window.addEventListener("load", init, false);
-module.exports = {game, Game}
+module.exports = {Game}

src/js/config.js

@@ -1,17 +1,17 @@
 module.exports = {
-    time_step: 60,
-    simulation_fps: 60,
-    draw_fps: 30,
-    velocity_iterations: 8,
-    position_iterations: 3,
-    max_zoom_factor: 130,
-    min_motor_speed: -2,
-    max_motor_speed: 2,
-    population_size: 1,
-    walker_health: 100,
-    max_floor_tiles: 50,
-    round_length: 1000,
-    min_body_delta: 0,
-    min_leg_delta: 0.0,
-    action_repeat: 4,
+  time_step: 60,
+  simulation_fps: 60,
+  draw_fps: 30,
+  velocity_iterations: 8,
+  position_iterations: 3,
+  max_zoom_factor: 130,
+  min_motor_speed: -2,
+  max_motor_speed: 2,
+  population_size: 1,
+  walker_health: 100,
+  max_floor_tiles: 50,
+  round_length: 1000,
+  min_body_delta: 0,
+  min_leg_delta: 0,
+  action_repeat: 4
 };

src/js/ddpg/ddpg_agent.js

@@ -43,7 +43,7 @@ class DDPGAgent {
             "batchSize": config.batchSize || 128,
             "actorLr": config.actorLr || 0.0001,
             "criticLr": config.criticLr || 0.001,
-            "memorySize": config.memorySize || 30000,
+            "memorySize": config.memorySize || 40000,
             "gamma": config.gamme || 0.99,
             "noiseDecay": config.noiseDecay || 0.99,
             "rewardScale": config.rewardScale || 1,
@@ -52,6 +52,7 @@ class DDPGAgent {
             "nbTrainSteps": config.nbTrainSteps || 110,
             "tau": config.tau || 0.008,
             "initialStddev": config.initialStddev || 0.1,
+            "minActionStddev": config.minActionStddev || 0.01,
             "desiredActionStddev": config.desiredActionStddev || 0.1,
             "adoptionCoefficient": config.adoptionCoefficient || 1.01,
             "actorFirstLayerSize": config.actorFirstLayerSize || 64,
@@ -108,13 +109,15 @@ class DDPGAgent {
         */
         var critic, actor
         if (typeof WEB === "undefined") {
+            console.log('loading', 'file://' + folder + '/critic-' + name + '.json')
+            console.log('loading', 'file://' + folder + '/actor-' + name + '.json')
             critic = await tf.loadModel('file://' + folder + '/critic-' + name + '.json');
             actor = await tf.loadModel("file://" + folder + "/actor-" + name + ".json");
         } else {
-            critic = await tf.loadModel(window.location.href + folder + '/critic-' + name + '.json');
-            actor = await tf.loadModel(window.location.href + folder + "/actor-" + name + ".json");
-            // const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
-            // const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
+            console.log('loading', window.location.origin + window.location.pathname + folder + '/actor-' + name + '.json')
+            console.log('loading', window.location.origin + window.location.pathname + folder + '/critic-' + name + '.json')
+            critic = await tf.loadModel(window.location.origin + window.location.pathname + folder + '/critic-' + name + '.json');
+            actor = await tf.loadModel(window.location.origin + window.location.pathname + folder + "/actor-" + name + ".json");
         }
 
         this.ddpg.critic = copyFromSave(critic, Critic, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
@@ -193,7 +196,7 @@ class DDPGAgent {
      * Optimize models and log states
      */
     _optimize() {
-        this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
+        this.ddpg.noise.desiredActionStddev = Math.max(this.config.minActionStddev, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
         let lossValuesCritic = [];
         let lossValuesActor = [];
         console.time("Training");

src/js/floor.js

@@ -23,7 +23,10 @@ function createFloor(world, max_floor_tiles) {
       edges[edges.length - 1].y + (ratio * Math.random() - ratio / 2)));
     //    edges.push(new b2.Vec2(edges[edges.length-1].x + 1,-0.16));
   }
-  edges.push(new b2.Vec2(edges[edges.length - 1].x, edges[edges.length - 1].y + 18)); // front wall
+  edges.push(new b2.Vec2(edges[edges.length - 1].x + 5, edges[edges.length - 1].y + 5)); // front wall
+  edges.push(new b2.Vec2(edges[edges.length - 1].x + 7, edges[edges.length - 1].y + 15)); // front wall
+  edges.push(new b2.Vec2(edges[edges.length - 1].x + 8, edges[edges.length - 1].y + 35)); // front wall
+  edges.push(new b2.Vec2(edges[edges.length - 1].x + 8, edges[edges.length - 1].y + 45)); // front wall
   fix_def.shape.CreateChain(edges, edges.length);
   body.CreateFixture(fix_def);
   return body;

src/js/renderer.js

@@ -4,8 +4,8 @@ class Renderer {
     this.walkers = [walker]
     this.floor = floor
 
-    this.main_screen = document.getElementById("main_screen");
-    this.ctx = main_screen.getContext("2d");
+    this.main_screen = document.getElementById(config.canvas_id);
+    this.ctx = this.main_screen.getContext("2d");
     this.resetCamera();
   }
 

src/js/walker.js

@@ -10,7 +10,8 @@ const {
   Renderer
 } = require('./renderer')
 
-const STRENGTH = 2
+const STRENGTH = 2.2
+const SPEED = 15
 
 class Walker {
   constructor(world, floor, config) {
@@ -34,7 +35,8 @@ class Walker {
 
     this.hue = randf(200, 360)
 
-
+    // http://www.ele.uri.edu/faculty/vetter/BME207/anthropometric-data.pdf
+    // https://multisite.eos.ncsu.edu/www-ergocenter-ncsu-edu/wp-content/uploads/sites/18/2016/06/Anthropometric-Detailed-Data-Tables.pdf
     this.torso_def = {
       upper_width: 0.25,
       upper_height: 0.45,
@@ -89,14 +91,14 @@ class Walker {
     this.bd.position.x += randf(-10, 10)
     this.bd.type = b2.Body.b2_dynamicBody;
     this.bd.linearDamping = 0;
-    this.bd.angularDamping = 20; // decay in force/ air friction
+    this.bd.angularDamping = 10; // decay in force/ air friction
     this.bd.allowSleep = true;
     this.bd.awake = true;
 
     this.fd = new b2.FixtureDef();
     this.fd.density = this.density;
     this.fd.restitution = 0.1; // bounciness
-    this.fd.friction = 10000; // only on contact (grip)
+    this.fd.friction = 100000000; // only on contact (grip)
     this.fd.shape = new b2.PolygonShape();
     this.fd.filter.groupIndex = -1;
 
@@ -133,17 +135,17 @@ class Walker {
         var otherFixture = contact.m_fixtureA.m_body.m_userData == "floor" ? contact.m_fixtureB : contact.m_fixtureA
         if (otherFixture.m_body === self.right_leg.foot) {
           // TODO let the agent act to grip or not. Only if palm or foot down?
-          self.right_leg.frictionJoint.maxForce = 1000 * self.grips[0]
-          self.right_leg.frictionJoint.maxTorque = 1000 * self.grips[0]
+          self.right_leg.frictionJoint.maxForce = 0 * self.grips[0]
+          self.right_leg.frictionJoint.maxTorque = 100000 * self.grips[0]
         } else if (otherFixture.m_body === self.left_leg.foot) {
-          self.left_leg.frictionJoint.maxForce = 1000 * self.grips[1]
-          self.left_leg.frictionJoint.maxTorque = 1000 * self.grips[1]
+          self.left_leg.frictionJoint.maxForce = 0 * self.grips[1]
+          self.left_leg.frictionJoint.maxTorque = 100000 * self.grips[1]
         } else if (otherFixture.m_body == self.right_arm.hand) {
-          self.right_arm.frictionJoint.maxForce = 1000 * self.grips[2]
-          self.right_arm.frictionJoint.maxTorque = 1000 * self.grips[2]
+          self.right_arm.frictionJoint.maxForce = 0   * self.grips[2]
+          self.right_arm.frictionJoint.maxTorque = 100000 * self.grips[2]
         } else if (otherFixture.m_body === self.left_arm.hand) {
-          self.left_arm.frictionJoint.maxForce = 1000 * self.grips[3]
-          self.left_arm.frictionJoint.maxTorque = 1000 * self.grips[3]
+          self.left_arm.frictionJoint.maxForce = 0 * self.grips[3]
+          self.left_arm.frictionJoint.maxTorque = 100000 * self.grips[3]
         }
       }
     }
@@ -210,10 +212,10 @@ class Walker {
     position.y -= this.torso_def.upper_height / 2;
     position.x -= this.torso_def.lower_width / 3;
     jd.Initialize(upper_torso, lower_torso, position);
-    jd.lowerAngle = deg2rad(-75 / 2);
+    jd.lowerAngle = deg2rad(-10 / 2);
     jd.upperAngle = deg2rad(30 / 2);
     jd.enableLimit = true;
-    jd.maxMotorTorque = 150 * STRENGTH;
+    jd.maxMotorTorque = 10 * STRENGTH;
     jd.motorSpeed = 0;
     jd.enableMotor = true;
     var j = this.world.CreateJoint(jd)
@@ -397,10 +399,10 @@ class Walker {
     var position = neck.GetPosition().Clone();
     position.y += this.head_def.neck_height / 2;
     jd.Initialize(head, neck, position);
-    jd.lowerAngle = -0.1;
-    jd.upperAngle = 0.2;
+    jd.lowerAngle = -0.1/2;
+    jd.upperAngle = 0.2/2;
     jd.enableLimit = true;
-    jd.maxMotorTorque = 4 * STRENGTH;
+    jd.maxMotorTorque = 2 * STRENGTH;
     jd.motorSpeed = 0;
     jd.enableMotor = true;
     var j = this.world.CreateJoint(jd)
@@ -497,7 +499,7 @@ class Walker {
     // Create a fixture definition to apply our shape to
     var fixtureDef = new b2.FixtureDef();
     fixtureDef.shape = circle;
-    fixtureDef.density = 20;
+    fixtureDef.density = 50;
     fixtureDef.friction = 0.4;
     fixtureDef.restitution = 0.6; // Make it bounce a little bit
 
@@ -571,7 +573,7 @@ class Walker {
   simulationPreStep(motorSpeeds) {
     // act
     for (var k = 0; k < this.joints.length; k++) {
-      this.joints[k].SetMotorSpeed(motorSpeeds[k] * 30); // action can range from -3 to 3, radians per second
+      this.joints[k].SetMotorSpeed(motorSpeeds[k] * SPEED); // action can range from -3 to 3, radians per second
     }
     for (let i = 0; i < motorSpeeds.length - this.joints.length; i++) {
       this.grips[i] = motorSpeeds[i] > 0
@@ -605,6 +607,7 @@ class Walker {
     }
     this.steps++
     this.episodeSteps++
+    if ((typeof WEB === "undefined") && (this.steps%400)) this.shuffle()
     /* score/reward */
     // reward copied from OpenAI Gym Humanoid Walker https://github.com/openai/gym/blob/master/gym/envs/mujoco/humanoid.py
     // also see https://github.com/AdamStelmaszczyk/learning2run/blob/master/osim-rl/osim/env/run.py#L67
@@ -613,7 +616,7 @@ class Walker {
     // reward for keeping head up, compared to feet
     var mean_foot_height = (this.left_leg.foot.GetPosition().y + this.right_leg.foot.GetPosition().y) / 2
 
-    var head_height_reward = (this.head.head.GetPosition().y - mean_foot_height) * 400; // it's head should be above it's feet 2*(-0.25-2)
+    var head_height_reward = (this.head.head.GetPosition().y - mean_foot_height) * 500; // it's head should be above it's feet 2*(-0.25-2)
 
     // reward for moving one leg beyond the other (stepping)
     var left_leg_forward = this.right_leg.foot.GetPosition().x > this.left_leg.foot.GetPosition().x;
@@ -622,7 +625,7 @@ class Walker {
 
     // cost for moving joints to unnatural positions (fraction of movement range in the relevant direction)
     var jointFractionMovement = j => j.GetJointAngle() > 0 ? j.GetJointAngle() / (j.GetUpperLimit() + 1) : j.GetJointAngle() / (j.GetLowerLimit() - 1)
-    var quad_joint_angle_cost = -0.10 * this.joints.map(j => jointFractionMovement(j) * 1.2)
+    var quad_joint_angle_cost = -0.40 * this.joints.map(j => jointFractionMovement(j) * 1.2)
       .reduce((o, v) => o + v * v, 0)
     quad_joint_angle_cost = Math.max(quad_joint_angle_cost, -10)
 
@@ -631,17 +634,17 @@ class Walker {
     if (this.last_position === undefined) this.last_position = position
     var velocity = (position - this.last_position) * 100
     this.last_position = position
-    var lin_vel_reward = 3 * velocity
+    var lin_vel_reward = 2 * velocity
 
     // punish for using energy, squared
-    var quad_power_cost = -0.01 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2)
+    var quad_power_cost = -0.05 * this.joints.map(j => j.GetJointSpeed()).reduce((sum, speed) => sum + speed ** 2)
     quad_power_cost = Math.max(quad_power_cost, -10)
 
     // Lets be nice, all entities should find overall happiness in what they do
     var bonus_happiness = 40
 
     var contacts = this.bodies.map(b => b.GetContactList()).filter(b => b).length
-    var quad_contact_cost = -Math.min(contacts - 4, 10) / 2
+    var quad_contact_cost = -Math.min(contacts - 4, 10)
 
     this.rewards = {
       lin_vel_reward,
@@ -650,12 +653,7 @@ class Walker {
       quad_joint_angle_cost,
       bonus_happiness,
       head_height_reward,
-      leg_switch_reward,
-      head_height: (this.head.head.GetPosition().y - mean_foot_height),
-      center_x: this.torso.upper_torso.GetPosition().x,
-      center_y: this.torso.upper_torso.GetPosition().y,
-      mean_foot_height,
-      date: new Date(),
+      // leg_switch_reward
     }
 
     this.reward = Object.values(this.rewards).reduce((tot, v) => tot + v, 0) / 3
@@ -665,6 +663,11 @@ class Walker {
       steps: this.steps,
       reward: this.reward,
       position,
+      head_height: (this.head.head.GetPosition().y - mean_foot_height),
+      center_x: this.torso.upper_torso.GetPosition().x,
+      center_y: this.torso.upper_torso.GetPosition().y,
+      mean_foot_height,
+      date: new Date(),
       ...this.rewards
     }
     var done = 0
@@ -694,8 +697,8 @@ class Walker {
   chuckBalls() { 
     for (const ball of this.balls) {
       let pb = ball.GetPosition()
-      let dx = (this.torso.upper_torso.GetPosition().x-pb.x)*4
-      let dy = (this.torso.upper_torso.GetPosition().y - pb.y) * 6
+      let dx = (this.torso.upper_torso.GetPosition().x-pb.x)*10
+      let dy = (this.torso.upper_torso.GetPosition().y - pb.y) * 10
       dx = clamp(dx, -16, 16)
       dy = clamp(dy, -16, 16)
       ball.SetLinearVelocity(new b2.Vec2(dx,dy))
@@ -714,6 +717,7 @@ class Walker {
 
     this.randomise(0.5)
 
+    this.addBallOnWalker()
     this.addBallOnWalker()
     this.addBallOnWalker()
 

webpack.config.js

@@ -8,22 +8,12 @@ var ENV = process.env.NODE_ENV || "development";
 module.exports = {
   entry: {
     app: './src/index.js',
-    // lib: './src/vendor/index.js'
   },
   target: 'web',
-  devtool: ENV==="development"? 'dev-source-map': null,
+  devtool: ENV==="development"? 'dev-source-map': false,
   output: {
     path: path.join(__dirname, 'dist'), 
     filename: '[name].bundle.js',
-    // path:'/dist',
-  },
-  module: {
-    rules: [
-      {
-        test: /\.css$/,
-        use: [ 'style-loader', 'css-loader' ]
-      }
-    ]
   },
   optimization: {
     splitChunks: {
@@ -46,16 +36,16 @@ module.exports = {
     }
   },
   plugins: [
-    new webpack.DefinePlugin({
-      // A flag to disable node imports, and enable window/dom usage
-      WEB: true
-    }),
     new HtmlWebpackPlugin({ template: '!!html-loader!src/index.html' }),
     new CopyWebpackPlugin([
       { from: 'src/images', to: 'images' },
-      {from:'checkpoints',to:'checkpoints'}
+      { from: 'checkpoints', to: 'checkpoints' },
+      {from:'src/css',to:'css'}
       
   ]), 
-    
+  new webpack.DefinePlugin({
+    // A flag to disable node imports, and enable window/dom usage
+    WEB: true
+  }),
   ]
 };