Solve leak problem + Refactoring

demo/webapp/public/js/level1.js

@@ -6,7 +6,6 @@ var env = new metacar.env("canvas", levelUrl);
 // Create the Policy agent
 var agent = new PolicyAgent(env);
 
-
 env.load().then(() => {
     // The level is loaded. Add listernes
     env.addEvent("train", () => agent.train());

demo/webapp/public/js/policy_agent.js

@@ -22,7 +22,7 @@ class PolicyAgent {
             Build the Value function
             @weights (Object) Weights for the layer
         */
-        const LEARNING_RATE = 0.01;
+        const LEARNING_RATE = 0.05;
         const value_optimizer = tf.train.adam(LEARNING_RATE);
         /*
             -----------------------
@@ -59,7 +59,7 @@ class PolicyAgent {
             Build the policy network
             @weights (Object) Weights for the layer
         */
-        const LEARNING_RATE = 0.01;
+        const LEARNING_RATE = 0.05;
         this.policy_optimizer = tf.train.adam(LEARNING_RATE);
         /*
             -----------------------
@@ -123,17 +123,21 @@ class PolicyAgent {
     trainPolicy(states, actions, advantages, batch_size, mini_batch_size){
         /*
             Train the policy model
-            @states (Js array)
-            @actions (Js array)
-            @advantages (Js array)
+            @states Tensor2D
+            @actions Tensor2D
+            @advantages Tensor2D
             @batch_size (Integer) Size of the batch size
             @mini_batch_size (Integer) Size of each mini batch size
          */
-        for (var b = 0; b < batch_size; b+=mini_batch_size) {
+        const size = states.shape[0];
 
-            const tf_states = tf.tensor3d(states.slice(b, b+mini_batch_size)).reshape([-1, this.ttLidarPts]);
-            const tf_actions = tf.tensor1d(actions.slice(b, b+mini_batch_size), "int32");
-            const tf_advantages = tf.tensor1d(advantages.slice(b, b+mini_batch_size));
+        for (var b = 0; b < batch_size; b+=mini_batch_size) {
+        
+            let to = (b + mini_batch_size < size) ?  mini_batch_size  : (size - b);
+
+            const tf_states = states.slice(b, to);
+            const tf_actions = actions.slice(b, to);
+            const tf_advantages = advantages.slice(b, to);
 
             this.policy_optimizer.minimize(() => {
                 let softmaxs = this.policyPredict(tf_states);
@@ -152,7 +156,7 @@ class PolicyAgent {
         // Maximum number of step per episode
         this.nb_step = 800;
         this.mini_batch_size = 200;
-        this.episodeNb = 250;
+        this.episodeNb = 100;
     }
 
     save(env){
@@ -211,28 +215,32 @@ class PolicyAgent {
             let reward = 0;
             const rewards = [];
             const states = [];
+            const stateValues = [];
             const actions = [];
 
             console.log("---");
             console.time("Exploring");
             for (var step = 0; step < this.nb_step; step++) {
                 // Get the current state
-                const array_st = this.env.getState();
+                const array_st = this.env.getState(true);
                 // Convert the state into a tensor
-                const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
+                //const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
+                const st = tf.tensor2d([array_st]);
                 // Predict the policy
-                const softmax = this.policyPredict(st);
+                const softmax = this.policyModel.predict(st);
+                // Predict the value
+                const value = this.valueModel.predict(st);
                 // Get the action. Pseudo Random choice. We prefer action with
                 // Higher probability
                 const action = randomChoice(softmax.buffer().values);
                 // Create the next batch
                 rewards.push(reward);
                 states.push(array_st);
+                stateValues.push(value);
                 actions.push(action);
                 // Stop the episode if the car go out of the road or crash an
                 // other car
                 if (reward == -10){
-                    console.log("Early stop Episode");
                     softmax.dispose();
                     st.dispose();
                     break;
@@ -258,24 +266,28 @@ class PolicyAgent {
                 G = rewards[t] + (this.gamma*G);
                 returns.push(G);
                 // Predict the value function for this state
-                const st = tf.tensor2d(states[t], [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
-                const Vs = this.valueModel.predict(st);
+                const V = stateValues[t];
                 // Advantage
-                advantages.push(G - Vs.buffer().values[0]);
-                st.dispose();
-                Vs.dispose();
+                advantages.push(G - V.buffer().values[0]);
+                V.dispose();
             }
             returns = returns.reverse();
             advantages = advantages.reverse();
 
             // Train the value model
-            const tf_batch_states = tf.tensor3d(states).reshape([batch_size, this.ttLidarPts]);
+            const tf_batch_states = tf.tensor2d(states);
             const tf_value_target = tf.tensor1d(returns);
+            const tf_actions = tf.tensor1d(actions, "int32");
+            const tf_advantages = tf.tensor1d(advantages);
             await this.trainValueFc(tf_batch_states, tf_value_target, mini_batch_size);
+            // Train the policy model
+            this.trainPolicy(tf_batch_states, tf_actions, tf_advantages, batch_size, mini_batch_size);
+
             tf_batch_states.dispose();
             tf_value_target.dispose();
-            // Train the policy model
-            this.trainPolicy(states, actions, advantages, batch_size, mini_batch_size);
+            tf_actions.dispose();
+            tf_advantages.dispose();
+
             // Set the agent on a new free road
             this.env.randomRoadPosition();
             //env.reset();

demo/webapp/public/models/policy-model-policy-agent.json

@@ -1 +0,0 @@
-{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,25],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":9,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomNormal","config":{"mean":0,"stddev":0.05,"seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true,"batch_input_shape":[null,25],"dtype":"float32"},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":3,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"RandomNormal","config":{"mean":0,"stddev":0.05,"seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true,"batch_input_shape":[null,9],"dtype":"float32"},"inbound_nodes":[[["dense_Dense3",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense4",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./policy-model-policy-agent.weights.bin"],"weights":[{"name":"dense_Dense3/kernel","shape":[25,9],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[9],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[9,3],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[3],"dtype":"float32"}]}]}

demo/webapp/public/models/value-model-policy-agent.json

@@ -1 +0,0 @@
-{"modelTopology":{"class_name":"Sequential","config":[{"class_name":"Dense","config":{"units":9,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomNormal","config":{"mean":0,"stddev":0.05,"seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true,"batch_input_shape":[null,25],"dtype":"float32"}},{"class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomNormal","config":{"mean":0,"stddev":0.05,"seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true,"batch_input_shape":[null,9],"dtype":"float32"}}],"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./value-model-policy-agent.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[25,9],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[9],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[9,1],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[1],"dtype":"float32"}]}]}

src/car.ts

@@ -11,6 +11,7 @@ import {
     CAR_IMG, Sprite, MAP, ROADSIZE, Container, Graphics
 } from "./global";
 import { RoadSprite } from "./asset_manager";
+import { stat } from "fs";
 
 
 var Global_carId = 0;
@@ -174,16 +175,22 @@ export class Car {
         }
     }
 
-    getState(): number[][]{
+    getState(linear:boolean = false): number[][]|number[]{
         /*
             Get the current state of the car
             The state is the current value of each point
             of the lidar.
         */
-        return this.motion.state.map(function(arr: any) { return arr.slice(); });
+        if (!linear)
+            return this.motion.state.map(function(arr: any) { return arr.slice(); });
+        else{
+            let state: number[] = [];
+            this.motion.state.map((row: number[]) => { state = state.concat(row);});
+            return state;
+        }
     }
 
-    step(delta: number, action:number=null){
+    step(delta: number, action:number|number[]=null){
         /*
             Take one step into the environement
             @delta (Float) time since the last update

src/level.ts

@@ -75,7 +75,7 @@ export class Level extends World {
         return reward;
     }
 
-    step(delta: number, action:number=null){
+    step(delta: number, action:number|number[]=null){
         /*
             Process one step into the environement
             @delta (Float) time since the last update

src/metacar.ts

@@ -7,6 +7,10 @@ import {actionSpaceDescription} from "./motion_engine";
 import {UIEvent} from "./ui_event";
 import * as U from "./utils";
 
+/**
+ * @local Chooce whether to load a file from the computer.
+ * A popup is open if True.
+ */
 export interface eventLoadOptions {
     local: boolean;
 }
@@ -17,44 +21,28 @@ export class MetaCar {
     private level: Level;
     private canvasId: string;
     private levelToLoad: string|Object;
-    private eventList: string[] = ["train", "play", "stop", "reset_env", "reset_agent", "load"]
+    private eventList: string[] = ["train", "play", "stop", "reset_env", "load"]
     private eventCallback: any[];
     private event: UIEvent;
 
+    /**
+     * Class used to create a new environement.
+     * @canvasId: HTML canvas ID
+     * @levelToLoad: URL of the level or directly the level's object.
+     *  URL format: embedded://... or http(s)://...
+    */
     constructor(canvasId: string, levelToLoad: string|Object) {
-        /**
-         * @canvasId: HTML canvas ID to used
-         * @level: URL or Local storage URL.
-         *  localstorage://level-name
-         *  embedded://
-         *  http(s)://
-        */
-        if (!canvasId || this.levelToLoad){
+        if (!canvasId || !levelToLoad){
             console.error("You must specify the canvasId and the levelToLoad");
         }
         this.canvasId = canvasId;
         this.levelToLoad = levelToLoad;
     }
 
-    private _setEvents(){
-        // SetEvents callback
-        this.event = new UIEvent(this.level, this.canvasId);
-        this.eventCallback = [
-            (fc: any) => this.event.onTrain(fc),
-            (fc: any) => this.event.onPlay(fc),
-            (fc:any) => this.event.onStop(fc),
-            (fc: any) => this.event.onResetEnv(fc),
-            (fc: any) => this.event.onResetAgent(fc),
-            (fc: any, opt: eventLoadOptions) => this.event.onLoad(fc, opt)
-        ];
-    }
-
-    load(level: string, agent: any): Promise<void>{
-        /*
-            Load the environement
-            @level (String) Name of the json level to load
-            @agent (Agent class)
-        */
+    /*
+        Load the environement with the parameters passed in the constructor.
+    */
+    public load(): Promise<void>{
 
         return new Promise((resolve, reject) => {
             console.log(typeof this.levelToLoad, this.levelToLoad);
@@ -62,56 +50,34 @@ export class MetaCar {
                 U.loadCustomURL(<string>this.levelToLoad, (content: LevelInfo) => {
                     this.level = new Level(content, this.canvasId);    
                     this._setEvents();
-                    this.level.load((delta: number) => this.loop(delta));
+                    this.level.load((delta: number) => this._loop(delta));
                     resolve();
                 });
             }
             else{
                 this.level = new Level(<LevelInfo>this.levelToLoad, this.canvasId);    
                 this._setEvents();
-                this.level.load((delta: number) => this.loop(delta));
+                this.level.load((delta: number) => this._loop(delta));
                 resolve(); 
             }
         });
-
-        /*
-        
-        this.agent = agent;
-        this.level = new Level(level, "canvas");
-        this.level.load((delta: number) => this.loop(delta));
-
-        document.getElementById("train").addEventListener("click", () => {
-            this.level.app.ticker.stop(); // .add(delta => this.loop(delta));
-            this.agent.train(this);
-        });
-        document.getElementById("stop").addEventListener("click", () => {
-            this.isPlaying = false;
-            this.level.render();
-            this.agent.stop();
-        });
-        document.getElementById("reset").addEventListener("click", () => {
-            this.level.reset();
-        });
-        document.getElementById("play").addEventListener("click", () => {
-            this.isPlaying = true
-        });
-        document.getElementById("saveAgent").addEventListener("click", () => {
-            this.agent.save(this);
-        });
-        document.getElementById("dumpFile").addEventListener("change", (e) => {
-            //readDump(e, (content) => this.agent.restore(this, content));
-        });
-        */
     }
     
     /**
-     * This method is used to add button under the canvas. When a
+     * This method is used to add a button under the canvas. When a
      * click is detected on the window, the associated @fc is called.
      * Some events are recognized by the environement, others can be custom.
+     * The following are recognized:
+     * - train: The render is stopped before to called @fc. You must called render(true) once your training is done.
+     * - play: Your function (@fc) will be called at each frame update.
+     * - stop: The last function passed to the play event will not be called anymore. Then @fc is called.
+     * - reset_env: Reset the environement. Then, @fc is called.
+     * - load: Load: @fc is called. You can set @options to {local:true} to load the content of a file from your computer.
+     * If @options is set, a content variable will be passed to the @fc function (the content of the selected file).
      * @eventName Name of the event to listen.
      * @fc Function to call each time this event is raised.
      */
-    addEvent(eventName: string, fc: any, options?: eventLoadOptions):void {
+    public addEvent(eventName: string, fc: any, options?: eventLoadOptions):void {
         const index = this.eventList.indexOf(eventName);
         if (index == -1){
             this.event.onCustomEvent(eventName, fc);
@@ -126,14 +92,20 @@ export class MetaCar {
         }
     }
 
-    render(val: boolean){
-        /*
-            Choose whether to render the environement.
-        */
+    /**
+     * Choose whether to render the environement.
+     * @val: True or False.
+     */
+    public render(val: boolean){
        this.level.render(val);
     }
 
-    save(content: any, file_name: string){
+    /**
+     * Usefull method to save/download a string as file.
+     * @content The content of the file
+     * @file_name The name of the file
+     */
+    public save(content: string, file_name: string){
         /*
             Save the agent
         */
@@ -142,35 +114,44 @@ export class MetaCar {
 
     /**
      * Get the action space of the environement
+     * @return The Description of the action space.
      */
-    actionSpace(): actionSpaceDescription{
+    public actionSpace(): actionSpaceDescription{
         return this.level.agent.motion.actionSpace();
     }
 
     /**
      * Return the current state of the environement.
      * The size of the state depends of the size of the Lidar.
+     * @return The state as a 2D Array or 1D Array (linear:true)
     */
-    getState(): number[][]{
-        return this.level.agent.getState();
+    public getState(linear:boolean = false): number[][]|number[]{
+        return this.level.agent.getState(linear);
     }
 
-    step(action: number){
-        /*
-            Step into the environement
-            @action (Integer)
-        */
+    /** 
+        Step into the environement
+        @action Action to process to step 
+        @return Reward value
+    */
+    public step(action: number|number[]): number{
         return this.level.step(1, action);
     }
 
-    reset(){
+    /**
+     * Reset the environement
+     */
+    public reset(): void{
         /*
             Reset the agent position
         */
         this.level.reset();
     }
 
-    randomRoadPosition(){
+    /**
+     * Set the agent on a new random road on the map.
+     */
+    randomRoadPosition(): void{
         /*
             This position
         */
@@ -186,7 +167,10 @@ export class MetaCar {
         }
     }
 
-    loop(delta: number){
+    /**
+     * @delta Time since the last update
+     */
+    private _loop(delta: number): void{
         if (this.event.isPlaying()){
             this.event.playCallback();
         }
@@ -195,4 +179,20 @@ export class MetaCar {
         }
     }
 
+    /**
+     * Create the UIEvent instance and set the events 
+     * callbacks relative to the UI.
+     */
+    private _setEvents(): void{
+        // SetEvents callback
+        this.event = new UIEvent(this.level, this.canvasId);
+        this.eventCallback = [
+            (fc: any) => this.event.onTrain(fc),
+            (fc: any) => this.event.onPlay(fc),
+            (fc:any) => this.event.onStop(fc),
+            (fc: any) => this.event.onResetEnv(fc),
+            (fc: any, opt: eventLoadOptions) => this.event.onLoad(fc, opt)
+        ];
+    }
+
 }

src/ui_event.ts

@@ -97,14 +97,6 @@ export class UIEvent {
             if (fc) fc();
         });
     }
-
-    public onResetAgent(fc: any){
-        // Create the button
-        const button = this._createButton(this.buttonsContainer, "reset_agent");
-        button.addEventListener("click", () => {
-            if (fc) fc();
-        });
-    }
  
     public onCustomEvent(name: string, fc: any){
         // Create the button