wassname/metacar
1
0
Fork 0
mirror of https://github.com/wassname/metacar.git synced 2026-09-17 12:30:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Qlearning

Browse Source
This commit is contained in:
Thibault Neveu
2018-06-14 21:14:15 +01:00
parent 85f0a6ddec
commit ad6fad9cf4
18 changed files with 474 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
demo/dist/metacar.min.js
Vendored
+21 -21
View File
File diff suppressed because one or more lines are too long
demo/webapp/level0.html
+20
View File
@@ -0,0 +1,20 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<title>Hello World</title>
</head>
<body>
<div class="canvas" id="canvas"></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
<script src="/dist/metacar.min.js"></script>
<script type="text/javascript" src="/public/js/utils.js"></script>
<script type="text/javascript" src="/public/js/q_table_agent.js"></script>
<script type="text/javascript" src="/public/js/level0.js"></script>
</body>
</html>
demo/webapp/public/js/editor.js
+2 -3
View File
@@ -1,5 +1,5 @@
// Get the url of the desired level
let levelUrl = metacar.level.fullCity;
let levelUrl = metacar.level.level0;
// Create the editor (canvasID, levelUrl)
var editor = new metacar.editor("editor", levelUrl);
@@ -8,6 +8,5 @@ editor.load().then(() => {
console.log(content);
// Put the object into storage
localStorage.setItem('mylevel.json', JSON.stringify(content));
}, {download: false, name: "mylevel.json"});
}, {download: true, name: "level0.json"});
});
demo/webapp/public/js/level0.js
+24
View File
@@ -0,0 +1,24 @@
// Get the url of the desired level
let levelUrl = metacar.level.level0;
// Create the environement (canvasID, levelUrl)
var env = new metacar.env("canvas", levelUrl);
console.log(env);
env.setAgentMotion(metacar.motion.BasicMotion, {rotationStep: 0.1});
env.setAgentLidar({pts: 2, width: 1, height: 1, pos: 1});
// Create the Policy agent
var agent = new QTableAgent(env);
env.load().then(() => {
// The level is loaded. Add listernes
env.addEvent("train", () => agent.train());
env.addEvent("play", () => agent.play());
env.addEvent("stop");
env.addEvent("reset_env", () => {
console.log("On reset env!");
});
env.addEvent("save", () => agent.save());
env.addEvent("load", (content) => agent.restore(content), {local: true});
});
demo/webapp/public/js/level1.js
-3
View File
@@ -14,9 +14,6 @@ env.load().then(() => {
env.addEvent("reset_env", () => {
console.log("On reset env!");
});
env.addEvent("reset_agent", () => {
console.log("On reset agent");
});
env.addEvent("save", () => agent.save());
env.addEvent("load", () => agent.restore());
});
demo/webapp/public/js/q_table_agent.js
+107
View File
@@ -0,0 +1,107 @@
class QTableAgent {
/*
Monte Carlo Agent
*/
constructor(env) {
this.env = env;
this.Q = {};
this.m = 0;
}
save(){
let save_content = JSON.stringify(this.Q);
console.log(save_content);
this.env.save(save_content, "mc_agent.json");
console.log("Q table saved");
}
restore(content){
this.Q = {};
content = JSON.parse(content);
for (const key in content){
this.Q[key] = [];
for (var i = 0; i < content[key].length; i++) {
if (content[key][i] != null){
this.Q[key].push(content[key][i]);
}
else{
this.Q[key].push(-Infinity);
}
}
}
console.log("Q table loaded");
}
play(){
// Get the current state
let state = this.env.getState().toString();
// In this state in not in the Q(s, a) function
if (!(state in this.Q)){
let action_space = this.env.actionSpace();
action_space.range = [0, 1, 2]; // Simplification of the numbers of actions
this.Q[state] = Array.apply(null, Array(action_space.range.length)).map(Number.prototype.valueOf, 0.0);
}
// Select the max action a in Q
let action = argMax(this.Q[state]);
// Take this action and get the associated reward
let reward = this.env.step(action);
}
createStateIfNotExist(st){
if (!(st in this.Q)){
let action_space = this.env.actionSpace();
action_space.range = [0, 1, 3];
this.Q[st] = Array.apply(null, Array(action_space.range.length)).map(Number.prototype.valueOf, 0);
}
}
pickAction(st, eps){
this.createStateIfNotExist(st);
let act;
if (Math.random() < eps){ // Pick a random action
act = Math.floor(Math.random()*this.Q[st].length);
}
else{
act = argMax(this.Q[st]);
}
return act;
}
train(){
let episode = 10000;
let eps = 1.0;
let eps_decrease = 0.99;
let mean_reward = [];
for (let ep = 0; ep < episode; ep++) {
if (ep % 50 == 0){
eps = Math.max(0.1, eps*eps_decrease);
console.log("episode=", ep, "eps=", eps, "mean_reward", mean(mean_reward));
}
mean_reward = [];
let st = this.env.getState().toString();
let act;
let gamma = 0.99;
let st2;
let act2;
for (var t = 0; t < 800; t++) {
act = this.pickAction(this.env, st, eps);
let reward = this.env.step(act);
mean_reward.push(reward);
st2 = this.env.getState().toString();
// Pick greedy action (eps = 0)
act2 = this.pickAction(st2, 0.);
this.createStateIfNotExist(st2);
this.createStateIfNotExist(st);
this.Q[st][act] = this.Q[st][act] + 0.05*(reward + (gamma*this.Q[st2][act2]) - this.Q[st][act]);
st = st2;
}
this.env.randomRoadPosition();
}
this.env.render(true);
console.log(this.Q);
}
}
demo/webapp/public/js/utils.js
+11
View File
@@ -10,4 +10,15 @@ function mean(array){
var sum = array.reduce(function(a, b) { return a + b; });
var avg = sum / array.length;
return avg;
}
function argMax(array) {
/*
Return the argmax of an array.
TODO: Should be replace soon by the usage of Tensorflow.js
*/
if (array.every((v) => v == -Infinity)){
return Math.floor(Math.random()*array.length);
}
return array.map((x, i) => [x, i]).reduce((r, a) => (a[0] > r[0] ? a : r))[1];
}
dist/metacar.min.js
Vendored
+21 -21
View File
File diff suppressed because one or more lines are too long
src/asset_manager.ts
+29 -7
View File
@@ -10,11 +10,12 @@ import {
ROADSIZE, Graphics, Sprite, ASSETS, MAP, CAR_IMG
} from "./global";
import {CarOptions, Car} from "./car";
import {CarOptions, Car, LidarInfoI} from "./car";
import {BasicMotionEngine} from "./basic_motion_engine";
import {ControlMotionEngine} from "./control_motion_engine";
import {BotMotionEngine} from "./bot_motion_engine";
import { Editor } from "./editor";
import { MotionEngine } from "./motion_engine";
export interface AssetInfo {
readonly mx?: number;
@@ -58,6 +59,9 @@ export class AssetManger {
private motion: any;
// List of all items on the map
public assets: SimpleSprite[] = [];
private agentMotionEngine: any = BasicMotionEngine;
private agentMotionOptions: Object = {};
private agentLidarInfo: LidarInfoI = {};
constructor(level: Level|Editor) {
this.level = level;
@@ -67,6 +71,27 @@ export class AssetManger {
}
}
/**
*
* options Options to change the lidar options of the agent.
* Changing the lidar change the state representation of the car in the
* environement.
*/
public setAgentLidar(options: LidarInfoI){
this.agentLidarInfo = options;
}
/**
* Change the motion engine of the agent. BasicMotionEngine by default.
* This method should be called before to called 'load'.
* @motion The motion engine to used for the agent when the environement is loaded.
* @options Options to change the behavior of the motion engine.
*/
public setAgentMotion(motion: any, options: Object){
this.agentMotionEngine = motion;
this.agentMotionOptions = options;
}
createRoadSide(info: AssetInfo, textures: any){
/*
@textures: (Pixi textures)
@@ -221,7 +246,7 @@ export class AssetManger {
for (let c in info.cars){
let options: CarOptions = {lidar: true, lidarInfo: {pts: 2, width: 0.5, height: 1, pos: 1}};
options.lidar = true;
options.motionEngine = new BotMotionEngine(this.level);
//options.motionEngine = new BotMotionEngine(this.level);
let n_car = new Car(this.level, info.cars[c], textures, options);
// Append the car to the canvas
this.level.addCar(n_car);
@@ -236,14 +261,11 @@ export class AssetManger {
@info (Object) Level's json.
@textures: (Pixi textures)
*/
const motionOptions = {
"rotationStep": 0.5,
"actions": ["UP", "LEFT", "RIGHT", "DOWN", "WAIT"]
}
let agent = new Car(this.level, info.agent, textures, {
image: CAR_IMG.AGENT,
lidar: true,
motionEngine: new BasicMotionEngine(<Level>this.level, motionOptions)
lidarInfo: this.agentLidarInfo,
motionEngine: new this.agentMotionEngine(<Level>this.level, this.agentMotionOptions)
});
this.level.addChild(agent.lidar)
src/basic_motion_engine.ts
+10 -5
View File
@@ -1,4 +1,4 @@
import { MotionEngine, MotionOption } from "./motion_engine";
import { MotionEngine} from "./motion_engine";
import {Level} from "./level";
import {
@@ -8,6 +8,11 @@ import {
import * as U from "./utils";
import {actionSpaceDescription} from "./motion_engine";
export interface BasicMotionOptions{
readonly rotationStep?: number;
readonly actions?: string[];
}
export class BasicMotionEngine extends MotionEngine {
/*
Basic Motion Engine
@@ -17,10 +22,10 @@ export class BasicMotionEngine extends MotionEngine {
private rotationStep: number;
private actions: string[];
constructor(level: Level, options: MotionOption) {
constructor(level: Level, options: BasicMotionOptions) {
super(level);
this.rotationStep = options.rotationStep;
this.actions = options.actions;
this.rotationStep = options.rotationStep || 0.5;
this.actions = options.actions || ["UP", "LEFT", "RIGHT", "DOWN", "WAIT"];
}
setUp(car: any, lidar: any){
@@ -131,7 +136,7 @@ export class BasicMotionEngine extends MotionEngine {
return {
type: "Discrete",
size: 1,
range: [0, 2]
range: [0, 1, 2, 3, 4]
}
}
src/car.ts
+5 -1
View File
@@ -213,11 +213,15 @@ export class Car {
@width: Width of the lidar (in proportion to the car)
@height: Height of the lidar (in proportion to the car)
*/
lidarOptions.pts = lidarOptions.pts || 5;
lidarOptions.width = lidarOptions.width || 4;
lidarOptions.height = lidarOptions.height || 5;
lidarOptions.pos = lidarOptions.pos || 0;
this.lidar = new Container();
// Area of the lidar
let area: LidarChild = new Graphics();
area.pt = false;
area.alpha = 0.5;
area.beginFill(0x515151);
area.drawRect(0, 0, this.core.width*lidarOptions.width, this.core.height*lidarOptions.height);
area.endFill();
src/editor.ts
-1
View File
@@ -103,7 +103,6 @@ export class Editor extends World {
this.agent = undefined;
this.app.stage.removeChild(item);
item.isremove = true;
console.log(item.isremove, item.agent, this.agent);
}
exportMap(width: number, height: number, file_name: string, download: boolean): any{
src/embedded.ts
+8 -4
View File
@@ -1,5 +1,6 @@
import {fullCity} from "./embedded/level/full_city";
import {level1} from "./embedded/level/level_1";
import {level0} from "./embedded/level/level_0";
/**
* Object used to enumerate each
@@ -10,18 +11,21 @@ import {level1} from "./embedded/level/level_1";
*
*/
export interface embeddedUrlI {
fullCity: string
level1: string
fullCity: string;
level1: string;
level0: string;
};
export const embeddedUrl: embeddedUrlI = {
fullCity: "embedded://level/fullCity",
level1: "embedded://level/level1"
level1: "embedded://level/level1",
level0: "embedded://level/level0"
}
export const embeddedContent: any = {
level: {
fullCity: fullCity,
level1: level1
level1: level1,
level0: level0
}
}
src/embedded/level/level_0.ts
+132
View File
@@ -0,0 +1,132 @@
export const level0: any = {
"cars": [
{
"mx": 3,
"my": 2,
"line": 0
},
{
"mx": 1,
"my": 2,
"line": 0
},
{
"mx": 2,
"my": 3,
"line": 0
},
{
"mx": 3,
"my": 1,
"line": 0
},
{
"mx": 1,
"my": 1,
"line": 0
}
],
"house": [
{
"x": 263,
"y": 108
},
{
"x": 112,
"y": 258
}
],
"house3": [
{
"x": 262,
"y": 16
}
],
"bench": [
{
"x": 135,
"y": 34
}
],
"tree": [
{
"x": 124,
"y": 134
},
{
"x": 7,
"y": 11
},
{
"x": 69,
"y": 10
},
{
"x": 19,
"y": 238
},
{
"x": 5,
"y": 133
},
{
"x": 260,
"y": 211
}
],
"map": [
[
0,
0,
0,
0,
0
],
[
0,
"↱",
"↔",
"↰",
0
],
[
0,
"↕",
0,
"↕",
0
],
[
0,
"↳",
"↠",
"↲",
0
],
[
0,
0,
0,
0,
0
]
],
"agent": {
"mx": 2,
"my": 1,
"line": 1,
"motion": {
"type": "BasicMotionEngine",
"options": {
"rotationStep": 0.5,
"actions": [
"UP",
"LEFT",
"RIGHT",
"DOWN",
"WAIT"
]
}
}
}
}
src/index.ts
+7 -1
View File
@@ -1,11 +1,17 @@
import {MetaCar} from "./metacar";
import {embeddedUrl, embeddedUrlI} from "./embedded";
import {MetaCarEditor} from "./metacar_editor";
import {BasicMotionEngine} from "./basic_motion_engine";
import {ControlMotionEngine} from "./control_motion_engine";
const metacar = {
env: MetaCar,
editor: MetaCarEditor,
level: embeddedUrl
level: embeddedUrl,
motion: {
BasicMotion: BasicMotionEngine,
ControlMotion: ControlMotionEngine
}
}
export default metacar;
src/level.ts
+27 -5
View File
@@ -9,8 +9,10 @@ import {
} from "./global";
import {AssetManger, RoadSprite} from "./asset_manager";
import {Car, CarSprite} from "./car";
import {Car, CarSprite, LidarChild, LidarInfoI} from "./car";
import {World} from "./world";
import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine";
import { ControlMotionEngine } from "./control_motion_engine";
export interface LevelInfo {
map: (string|number)[][];
@@ -34,6 +36,26 @@ export class Level extends World {
this.am = new AssetManger(this);
}
/**
* Change the motion engine of the agent. BasicMotionEngine by default.
* This method should be called before to called 'load'.
* @motion The motion engine to used for the agent when the environement is loaded.
* @options Options to change the behavior of the motion engine.
*/
public setAgentMotion(motion: typeof BasicMotionEngine|typeof ControlMotionEngine, options: BasicMotionOptions){
this.am.setAgentMotion(motion, options);
}
/**
*
* options Options to change the lidar options of the agent.
* Changing the lidar change the state representation of the car in the
* environement.
*/
public setAgentLidar(options: LidarInfoI){
this.am.setAgentLidar(options);
}
protected _setup(info: LevelInfo){
/*
Setup all the element of the map
@@ -82,10 +104,10 @@ export class Level extends World {
@action: (Integer) The action to take (can be null if no action)
*/
// Go through all cars to move each one
for (var c = 0; c < this.cars.length; c++) {
if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move
this.cars[c].step(delta);
}
//for (var c = 0; c < this.cars.length; c++) {
// if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move
// this.cars[c].step(delta);
//}
// Move the agent
if (this.agent){
let {agent_col, on_road} = this.agent.step(delta, action);
src/metacar.ts
+34 -3
View File
@@ -3,9 +3,12 @@
*/
import {Level, LevelInfo} from "./level";
import {actionSpaceDescription} from "./motion_engine";
import {actionSpaceDescription, MotionEngine} from "./motion_engine";
import {UIEvent} from "./ui_event";
import * as U from "./utils";
import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine";
import { ControlMotionEngine } from "./control_motion_engine";
import { LidarInfoI } from "./car";
/**
* @local Chooce whether to load a file from the computer.
@@ -24,6 +27,9 @@ export class MetaCar {
private eventList: string[] = ["train", "play", "stop", "reset_env", "load"]
private eventCallback: any[];
private event: UIEvent;
private agentMotionEngine: typeof BasicMotionEngine|typeof ControlMotionEngine = BasicMotionEngine;
private agentMotionOptions: BasicMotionOptions = {}
private agentLidarInfo: LidarInfoI;
/**
* Class used to create a new environement.
@@ -48,7 +54,9 @@ export class MetaCar {
console.log(typeof this.levelToLoad, this.levelToLoad);
if (typeof this.levelToLoad == "string"){
U.loadCustomURL(<string>this.levelToLoad, (content: LevelInfo) => {
this.level = new Level(content, this.canvasId);
this.level = new Level(content, this.canvasId);
this.level.setAgentMotion(this.agentMotionEngine, this.agentMotionOptions);
this.level.setAgentLidar(this.agentLidarInfo);
this._setEvents();
this.level.load((delta: number) => this._loop(delta));
resolve();
@@ -56,13 +64,36 @@ export class MetaCar {
}
else{
this.level = new Level(<LevelInfo>this.levelToLoad, this.canvasId);
this.level.setAgentMotion(this.agentMotionEngine, this.agentMotionOptions);
this.level.setAgentLidar(this.agentLidarInfo);
this._setEvents();
this.level.load((delta: number) => this._loop(delta));
resolve();
}
});
}
/**
*
* options Options to change the lidar options of the agent.
* Changing the lidar change the state representation of the car in the
* environement.
*/
public setAgentLidar(options: LidarInfoI){
this.agentLidarInfo = options;
}
/**
* Change the motion engine of the agent. BasicMotionEngine by default.
* This method should be called before to called 'load'.
* @motion The motion engine to used for the agent when the environement is loaded.
* @options Options to change the behavior of the motion engine.
*/
public setAgentMotion(motion: typeof BasicMotionEngine|typeof ControlMotionEngine, options: BasicMotionOptions){
this.agentMotionEngine = motion;
this.agentMotionOptions = options;
}
/**
* This method is used to add a button under the canvas. When a
* click is detected on the window, the associated @fc is called.
src/motion_engine.ts
+16 -13
View File
@@ -10,11 +10,6 @@ import {
} from "./global";
import { Editor } from "./editor";
export interface MotionOption{
readonly rotationStep?: number;
readonly actions: string[];
}
/**
* Structure used to describe the action space.
* @type: Discrete or continous values
@@ -38,15 +33,23 @@ export class MotionEngine {
this.level = level;
}
protected boxesIntersect(a: any, b: any) {
protected boxesIntersect(a: any, b: any, reduce: boolean) {
/*
Chack if a the two elements intersect each others
@a (Pixi sprite)
@b (Pixi sprite)
*/
var ab = a.getBounds();
var bb = b.getBounds();
return ab.x + ab.width > bb.x && ab.x < bb.x + bb.width && ab.y + ab.height > bb.y && ab.y < bb.y + bb.height;
var ab = a.getBounds();
var bb = b.getBounds();
if (reduce){
ab.width = 15;
ab.height = 15;
bb.width = 15;
bb.height = 15;
}
return ab.x + ab.width > bb.x && ab.x < bb.x + bb.width && ab.y + ab.height > bb.y && ab.y < bb.y + bb.height;
}
protected detectInteractions(){
@@ -63,13 +66,13 @@ export class MotionEngine {
let on_road = false;
for (let i = 0; i < envs.length; i++) {
if (envs[i] != this.car && envs[i].obstacle && this.boxesIntersect(envs[i], this.car)){
if (envs[i] != this.car && envs[i].obstacle && this.boxesIntersect(envs[i], this.car, true)){
agent_col.push(envs[i]);
}
else if (envs[i].mapId == MAP.ROAD && this.boxesIntersect(envs[i], this.car)){
else if (envs[i].mapId == MAP.ROAD && this.boxesIntersect(envs[i], this.car, false)){
on_road = true;
}
if (this.boxesIntersect(envs[i], this.lidar)){
if (this.boxesIntersect(envs[i], this.lidar, false)){
lidar_collisions.push(envs[i]);
}
}
@@ -86,7 +89,7 @@ export class MotionEngine {
let pt_id = 0;
for (var i = 0; i < this.lidar.children.length; i++) {
this.lidar.children[i].alpha = 0.1;
this.lidar.children[i].alpha = 0.3;
if (this.lidar.children[i].pt){ // If this is a lidar point
let pt_y = Math.floor(pt_id/this.lidar.pts);