diff --git a/checkpoints/actor-model-ddpg-walker-33h/model.json b/checkpoints/actor-model-ddpg-walker-33h/model.json new file mode 100644 index 0000000..dd79be7 --- /dev/null +++ b/checkpoints/actor-model-ddpg-walker-33h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model7","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense22","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense22","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense23","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense23","trainable":true},"inbound_nodes":[[["dense_Dense22",0,0,{}]]]},{"name":"dense_Dense24","class_name":"Dense","config":{"units":18,"activation":"tanh","use_bias":false,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.03,"maxval":0.03,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense24","trainable":true},"inbound_nodes":[[["dense_Dense23",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense24",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense22/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense22/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense23/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense23/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense24/kernel","shape":[64,18],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/actor-model-ddpg-walker-33h/weights.bin b/checkpoints/actor-model-ddpg-walker-33h/weights.bin new file mode 100644 index 0000000..62478cb Binary files /dev/null and b/checkpoints/actor-model-ddpg-walker-33h/weights.bin differ diff --git a/checkpoints/actor-model-ddpg-walker-42h/model.json b/checkpoints/actor-model-ddpg-walker-42h/model.json new file mode 100644 index 0000000..dd79be7 --- /dev/null +++ b/checkpoints/actor-model-ddpg-walker-42h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model7","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense22","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense22","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense23","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense23","trainable":true},"inbound_nodes":[[["dense_Dense22",0,0,{}]]]},{"name":"dense_Dense24","class_name":"Dense","config":{"units":18,"activation":"tanh","use_bias":false,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.03,"maxval":0.03,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense24","trainable":true},"inbound_nodes":[[["dense_Dense23",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense24",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense22/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense22/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense23/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense23/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense24/kernel","shape":[64,18],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/actor-model-ddpg-walker-42h/weights.bin b/checkpoints/actor-model-ddpg-walker-42h/weights.bin new file mode 100644 index 0000000..62478cb Binary files /dev/null and b/checkpoints/actor-model-ddpg-walker-42h/weights.bin differ diff --git a/checkpoints/actor-model-ddpg-walker-60h/model.json b/checkpoints/actor-model-ddpg-walker-60h/model.json new file mode 100644 index 0000000..dd79be7 --- /dev/null +++ b/checkpoints/actor-model-ddpg-walker-60h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model7","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense22","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense22","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense23","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense23","trainable":true},"inbound_nodes":[[["dense_Dense22",0,0,{}]]]},{"name":"dense_Dense24","class_name":"Dense","config":{"units":18,"activation":"tanh","use_bias":false,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.03,"maxval":0.03,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense24","trainable":true},"inbound_nodes":[[["dense_Dense23",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense24",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense22/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense22/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense23/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense23/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense24/kernel","shape":[64,18],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/actor-model-ddpg-walker-60h/weights.bin b/checkpoints/actor-model-ddpg-walker-60h/weights.bin new file mode 100644 index 0000000..40279e8 Binary files /dev/null and b/checkpoints/actor-model-ddpg-walker-60h/weights.bin differ diff --git a/checkpoints/critic-model-ddpg-walker-33h/model.json b/checkpoints/critic-model-ddpg-walker-33h/model.json new file mode 100644 index 0000000..5c16eea --- /dev/null +++ b/checkpoints/critic-model-ddpg-walker-33h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model6","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,18],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense19","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense19","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense18","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense18","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add3","class_name":"Add","config":{"name":"add_Add3","trainable":true},"inbound_nodes":[[["dense_Dense19",0,0,{}],["dense_Dense18",0,0,{}]]]},{"name":"dense_Dense20","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense20","trainable":true},"inbound_nodes":[[["add_Add3",0,0,{}]]]},{"name":"dense_Dense21","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.3,"maxval":0.3,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense21","trainable":true},"inbound_nodes":[[["dense_Dense20",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense21",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense19/kernel","shape":[18,128],"dtype":"float32"},{"name":"dense_Dense19/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense18/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense18/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense20/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense20/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense21/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense21/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/critic-model-ddpg-walker-33h/weights.bin b/checkpoints/critic-model-ddpg-walker-33h/weights.bin new file mode 100644 index 0000000..49fcf1e Binary files /dev/null and b/checkpoints/critic-model-ddpg-walker-33h/weights.bin differ diff --git a/checkpoints/critic-model-ddpg-walker-42h/model.json b/checkpoints/critic-model-ddpg-walker-42h/model.json new file mode 100644 index 0000000..5c16eea --- /dev/null +++ b/checkpoints/critic-model-ddpg-walker-42h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model6","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,18],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense19","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense19","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense18","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense18","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add3","class_name":"Add","config":{"name":"add_Add3","trainable":true},"inbound_nodes":[[["dense_Dense19",0,0,{}],["dense_Dense18",0,0,{}]]]},{"name":"dense_Dense20","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense20","trainable":true},"inbound_nodes":[[["add_Add3",0,0,{}]]]},{"name":"dense_Dense21","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.3,"maxval":0.3,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense21","trainable":true},"inbound_nodes":[[["dense_Dense20",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense21",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense19/kernel","shape":[18,128],"dtype":"float32"},{"name":"dense_Dense19/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense18/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense18/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense20/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense20/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense21/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense21/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/critic-model-ddpg-walker-42h/weights.bin b/checkpoints/critic-model-ddpg-walker-42h/weights.bin new file mode 100644 index 0000000..49fcf1e Binary files /dev/null and b/checkpoints/critic-model-ddpg-walker-42h/weights.bin differ diff --git a/checkpoints/critic-model-ddpg-walker-60h/model.json b/checkpoints/critic-model-ddpg-walker-60h/model.json new file mode 100644 index 0000000..5c16eea --- /dev/null +++ b/checkpoints/critic-model-ddpg-walker-60h/model.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model6","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,18],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,202],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense19","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense19","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense18","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense18","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add3","class_name":"Add","config":{"name":"add_Add3","trainable":true},"inbound_nodes":[[["dense_Dense19",0,0,{}],["dense_Dense18",0,0,{}]]]},{"name":"dense_Dense20","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense20","trainable":true},"inbound_nodes":[[["add_Add3",0,0,{}]]]},{"name":"dense_Dense21","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":-0.3,"maxval":0.3,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense21","trainable":true},"inbound_nodes":[[["dense_Dense20",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense21",0,0]]},"keras_version":"tfjs-layers 0.8.5","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense19/kernel","shape":[18,128],"dtype":"float32"},{"name":"dense_Dense19/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense18/kernel","shape":[202,128],"dtype":"float32"},{"name":"dense_Dense18/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense20/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense20/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense21/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense21/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/checkpoints/critic-model-ddpg-walker-60h/weights.bin b/checkpoints/critic-model-ddpg-walker-60h/weights.bin new file mode 100644 index 0000000..6dd26f1 Binary files /dev/null and b/checkpoints/critic-model-ddpg-walker-60h/weights.bin differ diff --git a/src/index.html b/src/index.html index 2a2125f..2068b8d 100644 --- a/src/index.html +++ b/src/index.html @@ -15,16 +15,17 @@
- Teaching 2d humanoids to walk in your browser using reinforcement learning. You can view the progress at - differen't stages of training with the buttons below. + What is this? An experiment to teach 2D humanoids to walk in your browser using reinforcement learning. The walker is reward for keeping it's head above its feet and moving to the right of the screen. + First it wiggled to the right, then crawed, loped, then learnt to a wierd skipping hopping motion. + + You can view the progress at differen't stages of training with the buttons below.


- Technical: This uses use reinforcement learning to teach them + How does it work?: This uses use reinforcement learning to teach them to walk. This is a branch of machine learning targeted at controlling systems such as system of limbs. Training is done offline in tensorflow.js. The environment is in box2d for javascript. The aglorithm is DDPG with prioritized experince replay.

- The dark outlines are when the agent grips the floor, since I found walking was difficult otherwise. The balls - are to provide obstacles. + The dark outlines are when the agent grips the floor, since I found walking was difficult otherwise. The balls are to provide obstacles.


Credits: The walker code is adapted from rednuht.org/genetic_walkers/, the DDPG code was adapted from github.com/thibo73800/metacar. @@ -38,6 +39,8 @@ + +

Be mean:

@@ -51,7 +54,7 @@ function init() { var canvas_id = 'main_screen2' window.game = new Game(config, canvas_id) - game.agent.restore('./checkpoints', 'model-ddpg-walker-22h/model') // load checkpoint + game.agent.restore('./checkpoints', 'model-ddpg-walker-60h/model') // load checkpoint game.agent.restore('../outputs', 'model-ddpg-walker/model') // load latest game.loop()