diff --git a/demo/webapp/public/js/DDPG/ddpg.js b/demo/webapp/public/js/DDPG/ddpg.js index 3655528..63fd1cc 100644 --- a/demo/webapp/public/js/DDPG/ddpg.js +++ b/demo/webapp/public/js/DDPG/ddpg.js @@ -211,26 +211,7 @@ class DDPG { return tf.mean(tfQPredictions0).mul(tf.scalar(-1.)) }, true, this.actorWeights); - /* - const sanityTfLoss = tf.tidy(() => { - const tfQPredictions0 = this.criticTargetWithActor(tfObs0); - const mn = tf.mean(tfQPredictions0); - const loss = mn.mul(tf.scalar(-1)); - return loss; - }); - - const sanityLoss = sanityTfLoss.buffer().values[0]; - - - if (sanityLoss == loss){ - console.warn("Sanity check failed. The optimisation have no effet here."); - } - */ - - //if (this.trainActorCt % 200 == 0 && this.trainActorCt != 0){ targetUpdate(this.actorTarget, this.actor, this.config); - //} - //this.trainActorCt += 1; const loss = actorLoss.buffer().values[0]; actorLoss.dispose(); diff --git a/demo/webapp/public/js/DDPG/ddpg_agent.js b/demo/webapp/public/js/DDPG/ddpg_agent.js index 553575b..2a6050c 100644 --- a/demo/webapp/public/js/DDPG/ddpg_agent.js +++ b/demo/webapp/public/js/DDPG/ddpg_agent.js @@ -51,6 +51,22 @@ class DDPGAgent { this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config); } + save(env){ + /* + Save the network + */ + this.ddpg.critic.model.save('downloads://critic-model-ddpg-agent'); + this.ddpg.actor.model.save('downloads://actor-model-ddpg-agent'); + } + + async restore(){ + /* + Restore the weights of the network + */ + this.ddpg.critic.model = await tf.loadModel('http://localhost:3000/public/models/ddpg/critic-model-ddpg-agent.json'); + this.ddpg.actor.model = await tf.loadModel("http://localhost:3000/public/models/ddpg/actor-model-ddpg-agent.json"); + } + /** * Play one step */ diff --git a/demo/webapp/public/js/DDPG/index.js b/demo/webapp/public/js/DDPG/index.js index 6ceed76..2eacb40 100644 --- a/demo/webapp/public/js/DDPG/index.js +++ b/demo/webapp/public/js/DDPG/index.js @@ -51,10 +51,6 @@ env.load().then(() => { RECORD = false; }); - env.addEvent("trainOnRecord", () => { - agent.ddpg.trainRecord(); - }); - env.addEvent("TrainRealTime", () => { env.steping(false); agent.train(true); @@ -66,17 +62,12 @@ env.load().then(() => { env.addEvent("reset_env"); - env.addEvent("load", (content) => { - content = JSON.parse(content); + env.addEvent("save", () => { + agent.save(); + }); - for (let c=0; c < content.obs0.length; c++){ - if (content.obs0[c] != 0){ - agent.ddpg.memory.append(content.obs0[c], content.actions[c], content.rewards[c], content.obs0[c], content.terminals[c]); - } - } - - console.log("dataReady"); - - }, {local: true}); + env.addEvent("load", () => { + agent.restore() + }); }); diff --git a/demo/webapp/public/js/level1.js b/demo/webapp/public/js/level1.js index 513f8de..c40dc16 100644 --- a/demo/webapp/public/js/level1.js +++ b/demo/webapp/public/js/level1.js @@ -30,6 +30,6 @@ env.load().then(() => { env.addEvent("save", () => agent.save()); env.addEvent("load", () => { document.getElementById("metacar_canvas_button_train").style.display = "none"; - agent.restore() + agent.restore(); }); }); diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json new file mode 100644 index 0000000..3cc61fd --- /dev/null +++ b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin new file mode 100644 index 0000000..5ed364c Binary files /dev/null and b/demo/webapp/public/models/ddpg/actor-model-ddpg-agent.weights.bin differ diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json new file mode 100644 index 0000000..0f8345c --- /dev/null +++ b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.json @@ -0,0 +1 @@ +{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,17],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-agent.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[17,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]} \ No newline at end of file diff --git a/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin new file mode 100644 index 0000000..82a1e3c Binary files /dev/null and b/demo/webapp/public/models/ddpg/critic-model-ddpg-agent.weights.bin differ