WebViewer + Api

.gitignore

@@ -4,4 +4,5 @@ dist/dist-es6/*
 demo/dist/dist-es6/*
 demo/node_modules/
 *package-lock.json*
-docs
+docs
+dist-es6

demo/webapp/level0.html

@@ -23,11 +23,25 @@
         <div class="canvas" id="canvas"></div>
       </div>
 
+      <div class="body_container">
+          <h3>Current state (Lidar points)</h3><br>
+          <div id="realtime_viewer" style="float: left"></div>
+          <p>
+              Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.
+          </p>
+      </div>
+
+      <div class="body_container">
+        <h3>Q-table</h3><br>
+        <div id="q_table"></div>
+      </div>
+
     <script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
     <script src="/dist/metacar.min.js"></script>
 
     <script type="text/javascript" src="/public/js/utils.js"></script>
+    <script type="text/javascript" src="/public/js/viewer.js"></script>
     <script type="text/javascript" src="/public/js/q_table_agent.js"></script>
     <script type="text/javascript" src="/public/js/level0.js"></script>
 

demo/webapp/level1.html

@@ -22,12 +22,21 @@
         <div class="canvas" id="canvas"></div>
     </div>
 
+    <div class="body_container">
+        <h3>Current state (Lidar points)</h3><br>
+        <div id="realtime_viewer" style="float: left"></div>
+        <p>
+            Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.
+        </p>
+    </div>
+
     <script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
     <script src="/dist/metacar.min.js"></script>
 
     <script type="text/javascript" src="/public/js/utils.js"></script>
     <script type="text/javascript" src="/public/js/policy_agent.js"></script>
+    <script type="text/javascript" src="/public/js/viewer.js"></script>
     <script type="text/javascript" src="/public/js/level1.js"></script>
 
 </body>

demo/webapp/public/js/level0.js

@@ -7,9 +7,18 @@ console.log(env);
 
 env.setAgentMotion(metacar.motion.BasicMotion, {rotationStep: 0.1});
 env.setAgentLidar({pts: 2, width: 1, height: 1, pos: 1});
+env.carsMoving(false);
 
 // Create the Policy agent
-var agent = new QTableAgent(env);
+var agent = new QTableAgent(env, 2);
+
+env.loop(() => {
+    let state = env.getState();
+    displayState("realtime_viewer", state, 200, 200);
+    let scores = agent.getStateValues(state);
+    let reward = env.getLastReward();
+    //displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]);
+})
 
 env.load().then(() => {
     // The level is loaded. Add listernes
@@ -20,5 +29,10 @@ env.load().then(() => {
         console.log("On reset env!");
     });
     env.addEvent("save", () => agent.save());
-    env.addEvent("load", (content) => agent.restore(content), {local: true});
+    env.addEvent("load", (content) => {
+        agent.restore(content);
+        displayQTable("q_table", agent.stateList, agent, ["Top", "Left", "Right"]);
+    } , {local: true});
+    env.addEvent("custom", () => {
+    });
 });

demo/webapp/public/js/level1.js

@@ -6,6 +6,14 @@ var env = new metacar.env("canvas", levelUrl);
 // Create the Policy agent
 var agent = new PolicyAgent(env);
 
+env.loop(() => {
+    let state = env.getState();
+    displayState("realtime_viewer", state, 200, 200);
+    let scores = agent.getStateValues(state);
+    let reward = env.getLastReward();
+    displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]);
+});
+
 env.load().then(() => {
     // The level is loaded. Add listernes
     env.addEvent("train", () => agent.train());

demo/webapp/public/js/policy_agent.js

@@ -9,8 +9,8 @@ class PolicyAgent {
         this.lidarPts = 5;
         this.ttLidarPts = 5*5;
         this.actionsNb = 3;
-
-        this.env = env
+        this.lastPrediction = [0., 0., 0.];
+        this.env = env;
 
         // Build the policy model and the value model
         this.buildValueFc();
@@ -120,6 +120,10 @@ class PolicyAgent {
         });
     }
 
+    getStateValues(){
+        return this.lastPrediction;
+    }
+
     trainPolicy(states, actions, advantages, batch_size, mini_batch_size){
         /*
             Train the policy model
@@ -171,8 +175,14 @@ class PolicyAgent {
         /*
             Restore the weights of the network
         */
-       this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json');
-       this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json");
+       if (window.location.href.indexOf("localhost") == -1){
+            this.valueModel = await tf.loadModel('https://metacar-project.com/public/models/policy/value-model-policy-agent.json');
+            this.policyModel = await tf.loadModel("https://metacar-project.com/public/models/policy/policy-model-policy-agent.json");
+       }
+       else{
+        this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json');
+        this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json");
+       }
     }
 
     play(){
@@ -181,11 +191,10 @@ class PolicyAgent {
             const st = tf.tensor2d(this.env.getState(), [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
             // Predict the policy
             const softmax = this.policyModel.predict(st);
-            softmax.print();
             // Get the action
+            this.lastPrediction = softmax.buffer().values;
             const argmax = softmax.argMax(1);
-            const a = argmax.buffer().values[0];
-
+            const a = argmax.buffer().values;
             argmax.dispose();
             st.dispose();
             softmax.dispose();

demo/webapp/public/js/q_table_agent.js

@@ -4,7 +4,9 @@ class QTableAgent {
         Monte Carlo Agent
     */
 
-    constructor(env) {
+    constructor(env, pts) {
+        this.stateList = [];
+        this.pts = pts;
         this.env = env;
         this.Q = {};
         this.m = 0;
@@ -17,26 +19,49 @@ class QTableAgent {
         console.log("Q table saved");
     }
 
+    stringStateToState(state){
+        state = state.split(",");
+        var nState = [];
+        var lineCt = 0;
+        var line = [];
+        for (let l = 0; l < state.length; l++){
+            line.push(parseInt(state[l]));
+            if ((l+1) % this.pts == 0){
+                nState.push(line);
+                line = [];
+            }
+        }
+        return nState;
+    }
+
     restore(content){
         this.Q = {};
+        this.stateList = [];
         content = JSON.parse(content);
         for (const key in content){
-            this.Q[key] = [];
-            for (var i = 0; i < content[key].length; i++) {
-                if (content[key][i] != null){
-                    this.Q[key].push(content[key][i]);
+            var nStateToPush = this.stringStateToState(key);
+            var st = key.toString();
+            if (nStateToPush.length == this.pts)
+                this.stateList.push(nStateToPush);
+            this.Q[st] = [];
+            for (var i = 0; i < content[st].length; i++) {
+                if (content[st][i] != null){
+                    this.Q[st].push(content[st][i]);
                 }
                 else{
-                    this.Q[key].push(-Infinity);
+                    this.Q[st].push(-Infinity);
                 }
             }
         }
+        console.log(this.Q);
+        console.log(this.stateList);
         console.log("Q table loaded");
     }
 
     play(){
         // Get the current state
-        let state = this.env.getState().toString();
+        let state = this.env.getState();
+        state = state.toString();
         // In this state in not in the Q(s, a) function
         if (!(state in this.Q)){
             let action_space = this.env.actionSpace();
@@ -54,10 +79,17 @@ class QTableAgent {
         if (!(st in this.Q)){
             let action_space = this.env.actionSpace();
             action_space.range = [0, 1, 3];
+            this.stateList.push(st);
             this.Q[st] = Array.apply(null, Array(action_space.range.length)).map(Number.prototype.valueOf, 0);
         }
     }
 
+    getStateValues(state){
+        state = state.toString();
+        this.createStateIfNotExist(state);
+        return this.Q[state];
+    }
+
     pickAction(st, eps){
         this.createStateIfNotExist(st);
         let act;
@@ -88,7 +120,7 @@ class QTableAgent {
             let st2;
             let act2;
             for (var t = 0; t < 800; t++) {
-                act = this.pickAction(this.env, st, eps);
+                act = this.pickAction(st, eps);
                 let reward = this.env.step(act);
                 mean_reward.push(reward);
                 st2 = this.env.getState().toString();
@@ -102,6 +134,5 @@ class QTableAgent {
             this.env.randomRoadPosition();
         }
         this.env.render(true);
-        console.log(this.Q);
     }
 }

demo/webapp/public/js/viewer.js

@@ -0,0 +1,116 @@
+/**
+ * Display a list of state with the associated score
+ */
+function displayQTable(id, states, agent, labels){
+    var scores = [];
+
+    var container = document.getElementById(id);
+    container.innerHTML = "";
+
+    for (let s=0; s < states.length; s++){
+        scores.push(agent.Q[states[s].toString()]);
+        let nContainer = document.getElementById(id + "_" + s);
+        if (!nContainer){
+            nContainer = document.createElement("div");
+            nContainer.id = id + "_" + s;
+            container.appendChild(nContainer);
+        }
+    }
+
+    for (let s=0; s < states.length; s++){
+        let _id = id + "_" + s;
+        displayState(_id, states[s], 100, 100);
+        displayScores(_id, scores[s], undefined, ["Top", "Left", "Right"]);
+    }
+
+}
+
+/**
+ * @id: div container to use
+ * @score
+ */
+function displayScores(id, score, reward, labels){
+    var container = document.getElementById(id);
+    container.style.position = "relative";
+
+    var textContainer = document.getElementById(id + "_text");
+    if (!textContainer){
+        textContainer = document.createElement("div");
+        textContainer.style.position = "absolute";
+        textContainer.style.top = "0px";
+        textContainer.id = id + "_text"
+        textContainer.style.right = "0px";
+        container.appendChild(textContainer);
+    }
+
+    console.log(score);
+    var text = "";
+    for (let c = 0; c < score.length; c++){
+        text += labels[c] + ": <b>" + Number((parseFloat(score[c])).toFixed(2)) + "</b><br>";
+    }
+
+    if (reward){
+        if (reward < 0)
+            text += "<br><p style='color: #ee4c32'>Reward: <b>" +Number((parseFloat(reward)).toFixed(2)) + "</b></p>"
+        else
+            text += "<br><p style='color: #80bf3e'>Reward: <b>" +Number((parseFloat(reward)).toFixed(2)) + "</b></p>"
+    }
+
+    textContainer.innerHTML = text;
+    textContainer.style.right = - textContainer.offsetWidth-10 + "px";
+    container.style.marginRight = textContainer.offsetWidth+30 + "px";
+}
+
+
+/**
+ * @id: div container to use
+ * @state of the map
+ * @width and @height desired width and height
+ */
+function displayState(id, state, width, height){
+    var xSize = width / state[0].length;
+    var ySize = height / state.length;
+    var exist = false;
+    var container = document.getElementById(id);
+    
+    if (container.innerHTML.length != 0){
+        exist = true;
+    }
+
+    container.style.display = "inline-block";
+    container.style.width = width+"px";
+    container.style.height = height+"px";
+    container.style.background = "red";
+
+    let yPos = 0;
+    for (let y = 0; y < state.length; y++){
+        let xPos = 0;
+        for (let x = 0; x < state[0].length; x++){
+            let nDiv;
+            if (!exist){
+                nDiv = document.createElement('div');
+                nDiv.id = id + "_" + y + "_" + x;
+                nDiv.style.display = "block";
+                nDiv.style.width = xSize+"px";
+                nDiv.style.height = ySize+"px";
+                nDiv.style.float = "left";
+                container.appendChild(nDiv);
+            }
+            else{
+                nDiv = document.getElementById(id + "_" + y + "_" + x);
+            }
+            if (state[y][x] == -1){ // road
+                nDiv.style.background = "#484848";
+            }
+            else if (state[y][x] == 0){ // Default
+                nDiv.style.background = "#80bf3e";
+            }
+            else{
+                nDiv.style.background = "#fdf9f5";
+            }
+            xPos += xSize;
+        }
+        yPos += ySize;
+    }
+
+}

src/asset_manager.ts

@@ -250,7 +250,7 @@ export class AssetManger {
         }
     }
 
-    createAgent(map: (string|number)[][], info: any, textures: (PIXI.Texture|PIXI.loaders.TextureDictionary)){
+    createAgent(map: (string|number)[][], info: any, textures: (PIXI.Texture|PIXI.loaders.TextureDictionary), displayLidar:boolean=true){
         /*
             Method used to create the agent's car.
             @map (2dim Array)
@@ -263,8 +263,9 @@ export class AssetManger {
             lidarInfo: this.agentLidarInfo,
             motionEngine: new this.agentMotionEngine(<Level>this.level, this.agentMotionOptions)
         });
-
-        this.level.addChild(agent.lidar);
+        if (displayLidar){
+            this.level.addChild(agent.lidar);
+        }
         this.level.addCar(agent);
         agent.core.agent = true;
 

src/editor.ts

@@ -42,7 +42,7 @@ export class Editor extends World {
         this.am.createMap(this.map, info, textures, false);
         this.am.createCars(this.map, info, textures);
         if (info.agent)
-            this.agent = this.am.createAgent(this.map, info, textures);
+            this.agent = this.am.createAgent(this.map, info, textures, false);
 
         var that = this;
         for (var i = 0; i < this.envs.length; i++) {

src/global.ts

@@ -13,7 +13,7 @@ export const Graphics      = PIXI.Graphics;
 export const Container     = PIXI.Container;
 
 // Main server
-export const URL = "http://localhost:3000/";
+export const URL = window.location.href.indexOf("localhost") == -1 ? "https://metacar-project.com/" : "http://localhost:3000/";
 
 // Textures files
 export const JSON_TEXTURES = URL + "public/textures/textures.json";
@@ -27,7 +27,7 @@ export const ROADSIZE = 60;
 
 export interface CARIMGI{
     DEFAULT: string;
-    AGENT: string;    
+    AGENT: string;
 }
 
 export const CAR_IMG: CARIMGI = {

src/level.ts

@@ -27,6 +27,7 @@ export interface Roads {
 export class Level extends World {
 
     public isCarsMoving: boolean = true;
+    private lastReward: number = 0;
 
     constructor(levelContent: LevelInfo, canvasId: string) {
         /*
@@ -104,16 +105,23 @@ export class Level extends World {
         if (action == 0 || this.agent.core.v == 1)
             reward += 0.5;
         if (agent_col.length > 0){
-            console.log("collision!");
             reward = -10;
         }
         else if (!on_road){
-            console.log("out");
             reward = -10;
         }
+        this.lastReward = reward;
         return reward;
     }
 
+    /**
+     * Return the last reward given by the environement
+     */
+    public getLastReward(): number{
+        return this.lastReward;
+    }
+
+
     step(delta: number, action:number|number[]=null){
         /*
             Process one step into the environement

src/metacar.ts

@@ -31,6 +31,7 @@ export class MetaCar {
     private agentMotionOptions: BasicMotionOptions = {}
     private agentLidarInfo: LidarInfoI;
     private isCarsMoving: boolean = true;
+    private loopCallback: any = undefined;
 
     /**
      * Class used to create a new environement.
@@ -84,6 +85,13 @@ export class MetaCar {
         this.isCarsMoving = moving;
     }
 
+    /**
+     * Return the last reward given by the environement
+     */
+    public getLastReward(): number{
+        return this.level.getLastReward();
+    }
+
     /**
      * options Options to change the lidar options of the agent.
      * This method should be called before to called 'load'.
@@ -209,6 +217,15 @@ export class MetaCar {
         }
     }
 
+    /**
+     * Using thid method you can call your own method
+     * at each loop update.
+     * @fc method to call at each loop update
+     */
+    public loop(fc: any){
+        this.loopCallback = fc;
+    }
+
     /**
      * @delta Time since the last update
      */
@@ -219,6 +236,9 @@ export class MetaCar {
         else {
             this.level.step(delta);
         }
+        if (this.loopCallback){
+            this.loopCallback();
+        }
     }
 
     /**

src/utils.ts

@@ -172,7 +172,7 @@ export function keyboard(keyCode: any ) {
         key.isDown = true;
         key.isUp = false;
       }
-      //event.preventDefault();
+      event.preventDefault();
     };
   
     key.upHandler = (event: any) => {
@@ -181,7 +181,7 @@ export function keyboard(keyCode: any ) {
         key.isDown = false;
         key.isUp = true;
       }
-      //event.preventDefault();
+      event.preventDefault();
     };
 
     //Attach event listeners