mirror of
https://github.com/wassname/metacar.git
synced 2026-09-09 11:26:47 +08:00
Thibault Neveu
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,66 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Metacar: Continous control in traffic</title>
|
||||
<link rel="icon" href="/public/img/icon.png">
|
||||
<link rel="stylesheet" href="/public/css/general.css">
|
||||
<link href="https://fonts.googleapis.com/css?family=Tajawal" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="header_container">
|
||||
<h1><a href="/">Metacar</a></h1>
|
||||
<a href="https://github.com/thibo73800/metacar"><img src="/public/img/github-logo.png" /></a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="canvas_container">
|
||||
<!--
|
||||
You just have to add a div with an ID as bellow and include the js to start
|
||||
using metacar. -->
|
||||
<div class="canvas" id="canvas"></div>
|
||||
</div>
|
||||
|
||||
<div class="body_container">
|
||||
<h3 id="trainingProgress"></h3>
|
||||
<h3 style="float: left" >Current state (Lidar points)</h3><br><br><br>
|
||||
<div id="realtime_viewer" style="float: left"></div>
|
||||
<p>
|
||||
The algorithm is based on the following papers:
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
</ul>
|
||||
<br>
|
||||
You can use the <b>arrow keys</b> to control the car by yourself.<br><br>
|
||||
|
||||
The motion control is based on two continuous values for the throttle and steering angle of the car.
|
||||
|
||||
<br><br>The left window gives you an overview of what the autonomous vehicle (in red) sees.<br><br>
|
||||
|
||||
You can find the code of this demo <a href="https://github.com/thibo73800/metacar/tree/master/demo/webapp/public/js/ddpg/">here</a> and <a href="https://github.com/thibo73800/metacar/tree/master/demo/webapp/public/js/ddpg-traffic/">here</a>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<div class="body_container" id="statContainer" style="position:relative"></div>
|
||||
|
||||
<script src="https://canvasjs.com/assets/script/canvasjs.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
|
||||
<script src="/dist/metacar.min.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/ddpg/models.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/prioritized_memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/noise.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg-traffic/index.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,66 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Metacar: Continous control</title>
|
||||
<link rel="icon" href="/public/img/icon.png">
|
||||
<link rel="stylesheet" href="/public/css/general.css">
|
||||
<link href="https://fonts.googleapis.com/css?family=Tajawal" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="header_container">
|
||||
<h1><a href="/">Metacar</a></h1>
|
||||
<a href="https://github.com/thibo73800/metacar"><img src="/public/img/github-logo.png" /></a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="canvas_container">
|
||||
<!--
|
||||
You just have to add a div with an ID as bellow and include the js to start
|
||||
using metacar. -->
|
||||
<div class="canvas" id="canvas"></div>
|
||||
</div>
|
||||
|
||||
<div class="body_container">
|
||||
<h3 id="trainingProgress"></h3>
|
||||
<h3 style="float: left" >Current state (Lidar points)</h3><br><br><br>
|
||||
<div id="realtime_viewer" style="float: left"></div>
|
||||
<p>
|
||||
The algorithm is based on the following papers:
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
</ul>
|
||||
<br>
|
||||
You can use the <b>arrow keys</b> to control the car by yourself.<br><br>
|
||||
|
||||
The motion control is based on two continuous values for the throttle and steering angle of the car.
|
||||
|
||||
<br><br>The left window gives you an overview of what the autonomous vehicle (in red) sees.<br><br>
|
||||
|
||||
You can find the code of this demo <a href="https://github.com/thibo73800/metacar/tree/master/demo/webapp/public/js/ddpg/">here</a>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<div class="body_container" id="statContainer" style="position:relative"></div>
|
||||
|
||||
<script src="https://canvasjs.com/assets/script/canvasjs.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
|
||||
<script src="/dist/metacar.min.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/ddpg/models.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/prioritized_memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/noise.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/index.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
+39
-14
@@ -37,45 +37,70 @@
|
||||
<br><br><br>
|
||||
|
||||
<div class="level_link_box" style="height: 300px;">
|
||||
<a href="/level0.html"><img class="floatleft" src="/public/img/level0.png" /></a>
|
||||
<a href="/qtable.html"><img class="floatleft" src="/public/img/level0.png" /></a>
|
||||
<h4>
|
||||
<br><a href="/level0.html">Q-learning with table</a>
|
||||
<br><a href="/qtable.html">Discrete Actions</a>
|
||||
</h4>
|
||||
<p>
|
||||
Algorithm: <b>Q-learning with table</b> <br><br>
|
||||
In this example, the goal is to demonstrate a simple working example of a reinforcement learning
|
||||
algorithm using the metacar environment. You can check out
|
||||
<a href="/level0.html">the demo</a> or take a look at the <a href="#">medium tutorial</a>.
|
||||
<a href="/qtable.html">the demo</a> or take a look at the <a href="#">medium tutorial</a>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="level_link_box" style="height: 298px;">
|
||||
<a href="/level1.html"><img class="floatright" src="/public/img/level1.png" /></a>
|
||||
<a href="/policy.html"><img class="floatright" src="/public/img/level1.png" /></a>
|
||||
<h4>
|
||||
<br><a href="/level1.html">Policy Monte carlo</a>
|
||||
<br><a href="/policy.html">Discrete Actions in traffic</a>
|
||||
</h4>
|
||||
<p>
|
||||
Here is an axample of a policy gradient algorithm based on Monte Carlo exploration.
|
||||
The neural network is build using <a href="http://js.tensorflow.org">tensorflow.js</a>. Check out <a href="/level1.html"> the demo</a>.
|
||||
Algorithm: <b>Policy gradient algorithm based on Monte Carlo exploration (REINFORCE).</b> <br><br>
|
||||
|
||||
The neural network is build using <a href="http://js.tensorflow.org">tensorflow.js</a>. Check out <a href="/policy.html"> the demo</a>.
|
||||
<br><br>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="level_link_box" style="height: 312px;">
|
||||
<a href="/level2.html"><img class="floatleft" src="/public/img/level2.png" /></a>
|
||||
<div class="level_link_box" style="height: 263px;">
|
||||
<a href="/ddpg.html"><img width="393px" class="floatleft" src="/public/img/ddpg.png" /></a>
|
||||
<h4>
|
||||
<br><a href="/level2.html">Full control</a>
|
||||
<br><a href="/ddpg.html">Continous Actions</a>
|
||||
</h4>
|
||||
<br>
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
This level is not solved yet (working on it). However, you can <a href="https://github.com/thibo73800/metacar/blob/master/CONTRIBUTE.md">contribute</a> to the project by solving it and sharing your implementation!<br><br>
|
||||
<br><br>
|
||||
The control is based on two continuous values for the throttle and steering angle of the car.
|
||||
The model is made using <a href="http://js.tensorflow.org">tensorflow.js</a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="level_link_box" style="height: 263px;">
|
||||
<a href="/ddpg-traffic.html"><img width="366px" class="floatright" src="/public/img/ddpg-traffic.png" /></a>
|
||||
<h4>
|
||||
<br><a href="/ddpg-traffic.html">Continous Actions in traffic</a>
|
||||
</h4>
|
||||
<br>
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
</ul>
|
||||
<p>
|
||||
The control is based on two continuous values for the throttle and steering angle of the car.
|
||||
The model is made using <a href="http://js.tensorflow.org">tensorflow.js</a>
|
||||
<br><br>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="level_link_box" style="height: 261px;">
|
||||
<a href="/editor.html"><img class="floatright" src="/public/img/editor.png" /></a>
|
||||
<a href="/editor.html"><img class="floatleft" src="/public/img/editor.png" /></a>
|
||||
<h4>
|
||||
<br><a href="/editor.html">Create your own level!</a>
|
||||
</h4>
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Metacar: Full control</title>
|
||||
<link rel="icon" href="/public/img/icon.png">
|
||||
<link rel="stylesheet" href="/public/css/general.css">
|
||||
<link href="https://fonts.googleapis.com/css?family=Tajawal" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="header_container">
|
||||
<h1><a href="/">Metacar</a></h1>
|
||||
<a href="https://github.com/thibo73800/metacar"><img src="/public/img/github-logo.png" /></a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="canvas_container">
|
||||
<!--
|
||||
You just have to add a div with an ID as bellow and include the js to start
|
||||
using metacar. -->
|
||||
<div class="canvas" id="canvas"></div>
|
||||
</div>
|
||||
|
||||
<div class="body_container">
|
||||
<h3 style="float: left" >Current state (Lidar points)</h3><br><br>
|
||||
<div id="realtime_viewer" style="float: left"></div>
|
||||
<p>
|
||||
You can use the <b>arrow keys</b> to control the car by yourself.<br><br>
|
||||
|
||||
This level is not solved yet (working on it). However, You can <a href="https://github.com/thibo73800/metacar/blob/master/CONTRIBUTE.md">contribute</a> to the project by solving it and sharing your implementation!<br><br>
|
||||
|
||||
The motion control is based on two continuous values for the throttle and steering angle of the car.
|
||||
|
||||
<br><br>The left window gives you an overview of what the autonomous vehicle (in red) sees.<br><br>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
|
||||
<script src="/dist/metacar.min.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
<script type="text/javascript" src="/public/js/level2.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -2,7 +2,7 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Metacar: Policy gradient demo</title>
|
||||
<title>Metacar: Discrete Actions in traffic.</title>
|
||||
<link rel="icon" href="/public/img/icon.png">
|
||||
<link rel="stylesheet" href="/public/css/general.css">
|
||||
<link href="https://fonts.googleapis.com/css?family=Tajawal" rel="stylesheet">
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
<div class="body_container">
|
||||
<h3 id="trainingProgress"></h3>
|
||||
<h3 style="float: left">Current state (Lidar points)</h3><br><br>
|
||||
<h3 style="float: left">Current state (Lidar points)</h3><br><br><br>
|
||||
<div id="realtime_viewer" style="float: left"></div>
|
||||
<p>
|
||||
<b>Policy Gradient</b> algorithm based on Monte Carlo exploration. <br>
|
||||
@@ -45,9 +45,10 @@
|
||||
<script src="/dist/metacar.min.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/policy_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
<script type="text/javascript" src="/public/js/level1.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/policy_monte_carlo/policy_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/policy_monte_carlo/index.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 26 KiB |
@@ -0,0 +1,73 @@
|
||||
|
||||
let levelUrl = metacar.level.level3;
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
env.setAgentLidar({pts: 7, width: 3, height: 7, pos: -0.5})
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var agent = new DDPGAgent(env, {
|
||||
stateSize: 50,
|
||||
resetEpisode: true,
|
||||
desiredActionStddev: 0.4,
|
||||
initialStddev: 0.4,
|
||||
actorFirstLayerSize: 128,
|
||||
actorSecondLayerSize: 64,
|
||||
criticFirstLayerSSize: 128,
|
||||
criticFirstLayerASize: 128,
|
||||
criticSecondLayerSize: 64,
|
||||
nbEpochs: 1000
|
||||
});
|
||||
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]);
|
||||
|
||||
let it = 0;
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let reward = env.getLastReward();
|
||||
const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
|
||||
if (it % 10 == 0)
|
||||
displayScores("realtime_viewer", [qValue, state.a, state.steering], reward, ["Q(a, s)", "Acceleration", "Steering Angle"]);
|
||||
it += 1;
|
||||
});
|
||||
|
||||
env.load().then(() => {
|
||||
|
||||
env.addEvent("train [Background]", () => {
|
||||
let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model.");
|
||||
if (train){
|
||||
env.render(false);
|
||||
agent.train(false);
|
||||
}
|
||||
});
|
||||
|
||||
env.addEvent("Train [Show the training]", () => {
|
||||
env.steping(false);
|
||||
agent.train(true);
|
||||
});
|
||||
|
||||
env.addEvent("play", () => {
|
||||
agent.play();
|
||||
});
|
||||
|
||||
env.addEvent("stop", () => {
|
||||
agent.stop();
|
||||
});
|
||||
|
||||
env.addEvent("reset_env");
|
||||
|
||||
env.addEvent("save", () => {
|
||||
agent.save("model-ddpg-traffic");
|
||||
});
|
||||
|
||||
env.addEvent("load", () => {
|
||||
agent.restore("ddpg-traffic", "model-ddpg-traffic-epoch-120");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
|
||||
function logTfMemory(){
|
||||
let mem = tf.memory();
|
||||
console.log("numBytes:" + mem.numBytes +
|
||||
"\nnumBytesInGPU:" + mem.numBytesInGPU +
|
||||
"\nnumDataBuffers:" + mem.numDataBuffers +
|
||||
"\nnumTensors:" + mem.numTensors);
|
||||
}
|
||||
|
||||
// This class is called from js/DDPG/ddpg_agent.js
|
||||
class DDPG {
|
||||
|
||||
/**
|
||||
* @param config (Object)
|
||||
* @param actor (Actor class)
|
||||
* @param critic (Critic class)
|
||||
* @param memory (Memory class)
|
||||
* @param noise (Noise class)
|
||||
*/
|
||||
constructor(actor, critic, memory, noise, config){
|
||||
this.actor = actor;
|
||||
this.critic = critic;
|
||||
this.memory = memory;
|
||||
this.noise = noise;
|
||||
this.config = config;
|
||||
this.tfGamma = tf.scalar(config.gamma);
|
||||
|
||||
// Inputs
|
||||
this.obsInput = tf.input({batchShape: [null, this.config.stateSize]});
|
||||
this.actionInput = tf.input({batchShape: [null, this.config.nbActions]});
|
||||
|
||||
// Randomly Initialize actor network μ(s)
|
||||
this.actor.buildModel(this.obsInput);
|
||||
// Randomly Initialize critic network Q(s, a)
|
||||
this.critic.buildModel(this.obsInput, this.actionInput);
|
||||
|
||||
|
||||
// Define in js/DDPG/models.js
|
||||
// Init target network Q' and μ' with the same weights
|
||||
this.actorTarget = copyModel(this.actor, Actor);
|
||||
this.criticTarget = copyModel(this.critic, Critic);
|
||||
// Perturbed Actor (See parameter space noise Exploration paper)
|
||||
this.perturbedActor = copyModel(this.actor, Actor);
|
||||
//this.adaptivePerturbedActor = copyModel(this.actor, Actor);
|
||||
|
||||
this.setLearningOp();
|
||||
}
|
||||
|
||||
setLearningOp(){
|
||||
this.criticWithActor = (tfState) => {
|
||||
return tf.tidy(() => {
|
||||
const tfAct = this.actor.predict(tfState);
|
||||
return this.critic.predict(tfState, tfAct);
|
||||
});
|
||||
};
|
||||
this.criticTargetWithActorTarget = (tfState) => {
|
||||
return tf.tidy(() => {
|
||||
const tfAct = this.actorTarget.predict(tfState);
|
||||
return this.criticTarget.predict(tfState, tfAct);
|
||||
});
|
||||
};
|
||||
|
||||
this.actorOptimiser = tf.train.adam(this.config.actorLr);
|
||||
this.criticOptimiser = tf.train.adam(this.config.criticLr);
|
||||
|
||||
this.criticWeights = [];
|
||||
for (let w = 0; w < this.critic.model.trainableWeights.length; w++){
|
||||
this.criticWeights.push(this.critic.model.trainableWeights[w].val);
|
||||
}
|
||||
this.actorWeights = [];
|
||||
for (let w = 0; w < this.actor.model.trainableWeights.length; w++){
|
||||
this.actorWeights.push(this.actor.model.trainableWeights[w].val);
|
||||
}
|
||||
|
||||
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Distance Measure for DDPG
|
||||
* See parameter space noise Exploration paper
|
||||
* @param observations (Tensor2d) Observations
|
||||
*/
|
||||
distanceMeasure(observations) {
|
||||
return tf.tidy(() => {
|
||||
const pertubedPredictions = this.perturbedActor.model.predict(observations);
|
||||
const predictions = this.actor.model.predict(observations);
|
||||
|
||||
const distance = tf.square(pertubedPredictions.sub(predictions)).mean().sqrt();
|
||||
return distance;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* AdaptParamNoise
|
||||
*/
|
||||
adaptParamNoise(){
|
||||
const batch = this.memory.getBatch(this.config.batchSize);
|
||||
if (batch.obs0.length == 0){
|
||||
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
|
||||
return [0];
|
||||
}
|
||||
|
||||
let distanceV = null;
|
||||
|
||||
if (batch.obs0.length > 0){
|
||||
const tfObs0 = tf.tensor2d(batch.obs0);
|
||||
const distance = this.distanceMeasure(tfObs0);
|
||||
|
||||
assignAndStd(this.actor, this.perturbedActor, this.noise.currentStddev, this.config.seed);
|
||||
|
||||
distanceV = distance.buffer().values;
|
||||
this.noise.adapt(distanceV[0]);
|
||||
|
||||
distance.dispose();
|
||||
tfObs0.dispose();
|
||||
}
|
||||
|
||||
return distanceV;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the estimation of the Q value given the state
|
||||
* and the action
|
||||
* @param state number[]
|
||||
* @param action [a, steering]
|
||||
*/
|
||||
getQvalue(state, a){
|
||||
const st = tf.tensor2d([state]);
|
||||
const tfa = tf.tensor2d([a]);
|
||||
const q = this.critic.model.predict([st, tfa]);
|
||||
const v = q.buffer().values
|
||||
st.dispose();
|
||||
tfa.dispose();
|
||||
q.dispose();
|
||||
return v[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param observation (tf.tensor2d)
|
||||
* @return (tf.tensor1d)
|
||||
*/
|
||||
predict(observation){
|
||||
const tfActions = this.actor.model.predict(observation);
|
||||
return tfActions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param observation (tf.tensor2d)
|
||||
* @return (tf.tensor1d)
|
||||
*/
|
||||
perturbedPrediction(observation){
|
||||
const tfActions = this.perturbedActor.model.predict(observation);
|
||||
return tfActions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the two target network
|
||||
*/
|
||||
targetUpdate(){
|
||||
// Define in js/DDPG/models.js
|
||||
targetUpdate(this.criticTarget, this.critic, this.config);
|
||||
targetUpdate(this.actorTarget, this.actor, this.config);
|
||||
}
|
||||
|
||||
trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals){
|
||||
|
||||
let costs;
|
||||
|
||||
const criticLoss = this.criticOptimiser.minimize(() => {
|
||||
const tfQPredictions0 = this.critic.model.predict([tfObs0, tfActions]);
|
||||
const tfQPredictions1 = this.criticTargetWithActorTarget(tfObs1);
|
||||
|
||||
const tfQTargets = tfRewards.add(tf.scalar(1).sub(tfTerminals).mul(this.tfGamma).mul(tfQPredictions1));
|
||||
|
||||
const erros = tf.sub(tfQTargets, tfQPredictions0).square();
|
||||
costs = erros.buffer().values;
|
||||
|
||||
return erros.mean();
|
||||
}, true, this.criticWeights);
|
||||
|
||||
// For experience Replay
|
||||
this.memory.appendBackWithCost(batch, costs);
|
||||
|
||||
const loss = criticLoss.buffer().values[0];
|
||||
criticLoss.dispose();
|
||||
|
||||
targetUpdate(this.criticTarget, this.critic, this.config);
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
trainActor(tfObs0){
|
||||
|
||||
const actorLoss = this.actorOptimiser.minimize(() => {
|
||||
const tfQPredictions0 = this.criticWithActor(tfObs0);
|
||||
return tf.mean(tfQPredictions0).mul(tf.scalar(-1.))
|
||||
}, true, this.actorWeights);
|
||||
|
||||
targetUpdate(this.actorTarget, this.actor, this.config);
|
||||
|
||||
const loss = actorLoss.buffer().values[0];
|
||||
actorLoss.dispose();
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
getTfBatch(){
|
||||
// Get batch
|
||||
const batch = this.memory.popBatch(this.config.batchSize);
|
||||
// Convert to tensors
|
||||
const tfActions = tf.tensor2d(batch.actions);
|
||||
const tfObs0 = tf.tensor2d(batch.obs0);
|
||||
const tfObs1 = tf.tensor2d(batch.obs1);
|
||||
const _tfRewards = tf.tensor1d(batch.rewards);
|
||||
const _tfTerminals = tf.tensor1d(batch.terminals);
|
||||
|
||||
const tfRewards = _tfRewards.expandDims(1);
|
||||
const tfTerminals = _tfTerminals.expandDims(1);
|
||||
|
||||
_tfRewards.dispose();
|
||||
_tfTerminals.dispose();
|
||||
|
||||
return {
|
||||
batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals
|
||||
}
|
||||
}
|
||||
|
||||
optimizeCritic(){
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const loss = this.trainCritic(tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
|
||||
|
||||
tfActions.dispose();
|
||||
tfObs0.dispose();
|
||||
tfObs1.dispose();
|
||||
tfRewards.dispose();
|
||||
tfTerminals.dispose();
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
optimizeActor(){
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const loss = this.trainActor(tfObs0);
|
||||
|
||||
tfActions.dispose();
|
||||
tfObs0.dispose();
|
||||
tfObs1.dispose();
|
||||
tfRewards.dispose();
|
||||
tfTerminals.dispose();
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
optimizeCriticActor(){
|
||||
const {batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals} = this.getTfBatch();
|
||||
|
||||
const lossC = this.trainCritic(batch, tfActions, tfObs0, tfObs1, tfRewards, tfTerminals);
|
||||
const lossA = this.trainActor(tfObs0);
|
||||
|
||||
tfActions.dispose();
|
||||
tfObs0.dispose();
|
||||
tfObs1.dispose();
|
||||
tfRewards.dispose();
|
||||
tfTerminals.dispose();
|
||||
|
||||
return {lossC, lossA};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// This class is called from js/DDPG/index.js
|
||||
class DDPGAgent {
|
||||
|
||||
/**
|
||||
* @param env (metacar.env) Set in js/DDPG/index.js
|
||||
*/
|
||||
constructor(env, config){
|
||||
|
||||
this.stopTraining = false;
|
||||
this.env = env;
|
||||
|
||||
config = config || {};
|
||||
// Default Config
|
||||
this.config = {
|
||||
"stateSize": config.stateSize || 17,
|
||||
"nbActions": config.nbActions || 2,
|
||||
"seed": config.seed || 0,
|
||||
"batchSize": config.batchSize || 128,
|
||||
"actorLr": config.actorLr || 0.0001,
|
||||
"criticLr": config .criticLr || 0.001,
|
||||
"memorySize": config.memorySize || 30000,
|
||||
"gamma": config.gamme || 0.99,
|
||||
"noiseDecay": config.noiseDecay || 0.99,
|
||||
"rewardScale": config.rewardScale || 1,
|
||||
"nbEpochs": config.nbEpochs || 200,
|
||||
"nbEpochsCycle": config.nbEpochsCycle || 10,
|
||||
"nbTrainSteps": config.nbTrainSteps || 110,
|
||||
"tau": config.tau || 0.008,
|
||||
"initialStddev": config.initialStddev || 0.1,
|
||||
"desiredActionStddev": config.desiredActionStddev || 0.1,
|
||||
"adoptionCoefficient": config.adoptionCoefficient || 1.01,
|
||||
"actorFirstLayerSize": config.actorFirstLayerSize || 64,
|
||||
"actorSecondLayerSize": config.actorSecondLayerSize || 32,
|
||||
"criticFirstLayerSSize": config.criticFirstLayerSSize || 64,
|
||||
"criticFirstLayerASize": config.criticFirstLayerASize || 64,
|
||||
"criticSecondLayerSize": config.criticSecondLayerSize || 32,
|
||||
"maxStep": config.maxStep || 800,
|
||||
"stopOnRewardError": config.stopOnRewardError != undefined ? config.stopOnRewardError:true,
|
||||
"resetEpisode": config.resetEpisode != undefined ? config.resetEpisode:false,
|
||||
"saveDuringTraining": config.saveDuringTraining || false,
|
||||
"saveInterval": config.saveInterval || 20
|
||||
};
|
||||
this.epoch = 0;
|
||||
// From js/DDPG/noise.js
|
||||
this.noise = new AdaptiveParamNoiseSpec(this.config);
|
||||
|
||||
// Configure components.
|
||||
|
||||
// Buffer replay
|
||||
// The baseline use 1e6 but this size should be enough for this problem
|
||||
this.memory = new PrioritizedMemory(this.config.memorySize);
|
||||
// Actor and Critic are from js/DDPG/models.js
|
||||
this.actor = new Actor(this.config);
|
||||
this.critic = new Critic(this.config);
|
||||
|
||||
// Seed javascript
|
||||
Math.seedrandom(0);
|
||||
|
||||
this.rewardsList = [];
|
||||
this.epiDuration = [];
|
||||
|
||||
// DDPG
|
||||
this.ddpg = new DDPG(this.actor, this.critic, this.memory, this.noise, this.config);
|
||||
}
|
||||
|
||||
save(name){
|
||||
/*
|
||||
Save the network
|
||||
*/
|
||||
this.ddpg.critic.model.save('downloads://critic-' + name);
|
||||
this.ddpg.actor.model.save('downloads://actor-'+ name);
|
||||
}
|
||||
|
||||
async restore(folder, name){
|
||||
/*
|
||||
Restore the weights of the network
|
||||
*/
|
||||
const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
|
||||
const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
|
||||
|
||||
this.ddpg.critic = copyFromSave(critic, Critic, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
|
||||
this.ddpg.actor = copyFromSave(actor, Actor, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
|
||||
|
||||
// Define in js/DDPG/models.js
|
||||
// Init target network Q' and μ' with the same weights
|
||||
this.ddpg.actorTarget = copyModel(this.ddpg.actor, Actor);
|
||||
this.ddpg.criticTarget = copyModel(this.ddpg.critic, Critic);
|
||||
// Perturbed Actor (See parameter space noise Exploration paper)
|
||||
this.ddpg.perturbedActor = copyModel(this.ddpg.actor, Actor);
|
||||
//this.adaptivePerturbedActor = copyModel(this.actor, Actor);
|
||||
this.ddpg.setLearningOp();
|
||||
}
|
||||
|
||||
/**
|
||||
* Play one step
|
||||
*/
|
||||
play(){
|
||||
// Get the current state
|
||||
const state = this.env.getState().linear;
|
||||
// Pick an action
|
||||
const tfActions = this.ddpg.predict(tf.tensor2d([state]));
|
||||
const actions = tfActions.buffer().values;
|
||||
agent.env.step([actions[0], actions[1]]);
|
||||
tfActions.dispose();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the estimation of the Q value given the state
|
||||
* and the action
|
||||
* @param state number[]
|
||||
* @param action [a, steering]
|
||||
*/
|
||||
getQvalue(state, a){
|
||||
return this.ddpg.getQvalue(state, a);
|
||||
}
|
||||
|
||||
stop(){
|
||||
this.stopTraining = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Step into the training environement
|
||||
* @param tfPreviousStep (tf.tensor2d) Current state
|
||||
* @param mPreviousStep number[]
|
||||
* @return {done, state} One boolean and the new state
|
||||
*/
|
||||
stepTrain(tfPreviousStep, mPreviousStep){
|
||||
// Get actions
|
||||
const tfActions = this.ddpg.perturbedPrediction(tfPreviousStep);
|
||||
// Step in the environment with theses actions
|
||||
let mAcions = tfActions.buffer().values;
|
||||
let mReward = this.env.step([mAcions[0], mAcions[1]]);
|
||||
this.rewardsList.push(mReward);
|
||||
// Get the new observations
|
||||
let mState = this.env.getState().linear;
|
||||
let tfState = tf.tensor2d([mState]);
|
||||
let mDone = 0;
|
||||
if (mReward == -1 && this.config.stopOnRewardError){
|
||||
mDone = 1;
|
||||
}
|
||||
// Add the new tuple to the buffer
|
||||
this.ddpg.memory.append(mPreviousStep, [mAcions[0], mAcions[1]], mReward, mState, mDone);
|
||||
// Dispose tensors
|
||||
tfPreviousStep.dispose();
|
||||
tfActions.dispose();
|
||||
|
||||
return {mDone, mState, tfState};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimize models and log states
|
||||
*/
|
||||
_optimize(){
|
||||
this.ddpg.noise.desiredActionStddev = Math.max(0.1, this.config.noiseDecay * this.ddpg.noise.desiredActionStddev);
|
||||
let lossValuesCritic = [];
|
||||
let lossValuesActor = [];
|
||||
console.time("Training");
|
||||
for (let t=0; t < this.config.nbTrainSteps; t++){
|
||||
let {lossC, lossA} = this.ddpg.optimizeCriticActor();
|
||||
lossValuesCritic.push(lossC);
|
||||
lossValuesActor.push(lossA);
|
||||
}
|
||||
console.timeEnd("Training");
|
||||
console.log("desiredActionStddev:", this.ddpg.noise.desiredActionStddev);
|
||||
setMetric("CriticLoss", mean(lossValuesCritic));
|
||||
setMetric("ActorLoss", mean(lossValuesActor));
|
||||
}
|
||||
|
||||
/**
|
||||
* Train DDPG Agent
|
||||
*/
|
||||
async train(realTime){
|
||||
this.stopTraining = false;
|
||||
// One epoch
|
||||
for (this.epoch; this.epoch < this.config.nbEpochs; this.epoch++){
|
||||
// Perform cycles.
|
||||
this.rewardsList = [];
|
||||
this.stepList = [];
|
||||
this.distanceList = [];
|
||||
document.getElementById("trainingProgress").innerHTML = "Progression: "+this.epoch+"/"+this.config.nbEpochs+"<br>";
|
||||
for (let c=0; c < this.config.nbEpochsCycle; c++){
|
||||
if (c%10==0){ logTfMemory(); }
|
||||
let mPreviousStep = this.env.getState().linear;
|
||||
let tfPreviousStep = tf.tensor2d([mPreviousStep]);
|
||||
let step = 0;
|
||||
|
||||
console.time("LoopTime");
|
||||
for (step=0; step < this.config.maxStep; step++){
|
||||
let rel = this.stepTrain(tfPreviousStep, mPreviousStep);
|
||||
mPreviousStep = rel.mState;
|
||||
tfPreviousStep = rel.tfState;
|
||||
if (rel.mDone && this.config.stopOnRewardError){
|
||||
break;
|
||||
}
|
||||
if (this.stopTraining){
|
||||
this.env.render(true);
|
||||
return;
|
||||
}
|
||||
if (realTime && step % 10 == 0)
|
||||
await tf.nextFrame();
|
||||
}
|
||||
this.stepList.push(step);
|
||||
console.timeEnd("LoopTime");
|
||||
let distance = this.ddpg.adaptParamNoise();
|
||||
this.distanceList.push(distance[0]);
|
||||
|
||||
if (this.config.resetEpisode){
|
||||
this.env.reset();
|
||||
}
|
||||
this.env.shuffle({cars: false});
|
||||
tfPreviousStep.dispose();
|
||||
console.log("e="+ this.epoch +", c="+c);
|
||||
|
||||
await tf.nextFrame();
|
||||
}
|
||||
if (this.epoch > 5){
|
||||
this._optimize();
|
||||
}
|
||||
if (this.config.saveDuringTraining && this.epoch % this.config.saveInterval == 0 && this.epoch != 0){
|
||||
this.save("model-ddpg-traffic-epoch-"+this.epoch);
|
||||
}
|
||||
setMetric("Reward", mean(this.rewardsList));
|
||||
setMetric("EpisodeDuration", mean(this.stepList));
|
||||
setMetric("NoiseDistance", mean(this.distanceList));
|
||||
await tf.nextFrame();
|
||||
}
|
||||
|
||||
|
||||
this.env.render(true);
|
||||
}
|
||||
|
||||
};
|
||||
@@ -0,0 +1,63 @@
|
||||
let levelUrl = metacar.level.level2;
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
env.setAgentLidar({pts: 5, width: 3, height: 7, pos: -0.5})
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var agent = new DDPGAgent(env, {
|
||||
stateSize: 26
|
||||
});
|
||||
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]);
|
||||
|
||||
let it = 0;
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let reward = env.getLastReward();
|
||||
const qValue = agent.getQvalue(state.linear, [state.a, state.steering]);
|
||||
if (it % 10 == 0)
|
||||
displayScores("realtime_viewer", [qValue, state.a, state.steering], reward, ["Q(a, s)", "Acceleration", "Steering Angle"]);
|
||||
it += 1;
|
||||
});
|
||||
|
||||
env.load().then(() => {
|
||||
|
||||
env.addEvent("train [Background]", () => {
|
||||
let train = confirm("The training process takes some time and might slow this tab. Do you want to continue? \n You can also load a pre-trained model.");
|
||||
if (train){
|
||||
env.render(false);
|
||||
agent.train(false);
|
||||
}
|
||||
});
|
||||
|
||||
env.addEvent("Train [Show the training]", () => {
|
||||
env.steping(false);
|
||||
agent.train(true);
|
||||
});
|
||||
|
||||
env.addEvent("shuffle", () => {
|
||||
env.shuffle({cars: false});
|
||||
})
|
||||
|
||||
env.addEvent("play", () => {
|
||||
agent.play();
|
||||
});
|
||||
|
||||
env.addEvent("stop", () => {
|
||||
agent.stop();
|
||||
});
|
||||
|
||||
env.addEvent("reset_env");
|
||||
|
||||
env.addEvent("save", () => {
|
||||
agent.save("model-ddpg-road");
|
||||
});
|
||||
|
||||
env.addEvent("load", () => {
|
||||
agent.restore("ddpg-road", "model-ddpg-road")
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
|
||||
class Memory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.length = 0;
|
||||
this.start = 0;
|
||||
|
||||
this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param idx (number)
|
||||
*/
|
||||
getItem(idx){
|
||||
if (idx < 0 || idx >= this.length){
|
||||
console.error("Memory.getItem: idx not in range.");
|
||||
}
|
||||
return this.data[(this.start + idx) % this.maxlen]
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const arrLength = this.length;
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
return batch;
|
||||
}
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * arrLength);
|
||||
batch.obs0.push(this.obs0List[id]);
|
||||
batch.obs1.push(this.obs1List[id]);
|
||||
batch.rewards.push(this.rewardsList[id]);
|
||||
batch.actions.push(this.actionsList[id]);
|
||||
batch.terminals.push(this.terminals1List[id]);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal1){
|
||||
if (this.length < this.maxlen){
|
||||
this.length += 1;
|
||||
}
|
||||
else if (this.length == this.maxlen) {
|
||||
//this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
this.start = (this.start + 1) % this.maxlen;
|
||||
}
|
||||
else {
|
||||
console.error("Memory.append: This should never be printed");
|
||||
}
|
||||
this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
|
||||
this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
|
||||
this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
|
||||
this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
|
||||
this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Copy a model
|
||||
* @param model Actor|Critic instance
|
||||
* @param instance Actor|Critic
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function copyFromSave(model, instance, config, obs, action){
|
||||
return tf.tidy(() => {
|
||||
nModel = new instance(config);
|
||||
// action might be not required
|
||||
nModel.buildModel(obs, action);
|
||||
const weights = model.weights;
|
||||
for (let m=0; m < weights.length; m++){
|
||||
nModel.model.weights[m].val.assign(weights[m].val);
|
||||
}
|
||||
return nModel;
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Copy a model
|
||||
* @param model Actor|Critic instance
|
||||
* @param instance Actor|Critic
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function copyModel(model, instance){
|
||||
return tf.tidy(() => {
|
||||
nModel = new instance(model.config);
|
||||
// action might be not required
|
||||
nModel.buildModel(model.obs, model.action);
|
||||
const weights = model.model.weights;
|
||||
for (let m=0; m < weights.length; m++){
|
||||
nModel.model.weights[m].val.assign(weights[m].val);
|
||||
}
|
||||
return nModel;
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy the value of of the model into the perturbedModel and
|
||||
* add a random pertubation
|
||||
* @param model Actor|Critic instance
|
||||
* @param perturbedActor Actor|Critic instance
|
||||
* @param stddev (number)
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function assignAndStd(model, perturbedModel, stddev, seed){
|
||||
return tf.tidy(() => {
|
||||
const weights = model.model.trainableWeights;
|
||||
for (let m=0; m < weights.length; m++){
|
||||
let shape = perturbedModel.model.trainableWeights[m].val.shape;
|
||||
let randomTensor = tf.randomNormal(shape, 0, stddev, "float32", seed);
|
||||
let nValue = weights[m].val.add(randomTensor);
|
||||
perturbedModel.model.trainableWeights[m].val.assign(nValue);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the target models
|
||||
* @param target Actor|Critic instance
|
||||
* @param perturbedActor Actor|Critic instance
|
||||
* @param config (Object)
|
||||
* @return Copy of the model
|
||||
*/
|
||||
function targetUpdate(target, original, config){
|
||||
return tf.tidy(() => {
|
||||
const originalW = original.model.trainableWeights;
|
||||
const targetW = target.model.trainableWeights;
|
||||
|
||||
const one = tf.scalar(1);
|
||||
const tau = tf.scalar(config.tau);
|
||||
|
||||
for (let m=0; m < originalW.length; m++){
|
||||
const lastValue = target.model.trainableWeights[m].val.clone();
|
||||
let nValue = tau.mul(originalW[m].val).add(targetW[m].val.mul(one.sub(tau)));
|
||||
target.model.trainableWeights[m].val.assign(nValue);
|
||||
const diff = lastValue.sub(target.model.trainableWeights[m].val).mean().buffer().values;
|
||||
if (diff[0] == 0){
|
||||
console.warn("targetUpdate: Nothing have been changed!")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
class Actor{
|
||||
|
||||
/**
|
||||
@param config (Object)
|
||||
*/
|
||||
constructor(config) {
|
||||
this.stateSize = config.stateSize;
|
||||
this.nbActions = config.nbActions;
|
||||
this.layerNorm = config.layerNorm;
|
||||
|
||||
this.firstLayerSize = config.actorFirstLayerSize;
|
||||
this.secondLayerSize = config.actorSecondLayerSize;
|
||||
|
||||
this.seed = config.seed;
|
||||
this.config = config;
|
||||
this.obs = null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param obs tf.input
|
||||
*/
|
||||
buildModel(obs){
|
||||
this.obs = obs;
|
||||
|
||||
// First layer
|
||||
this.firstLayer = tf.layers.dense({
|
||||
units: this.firstLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
// Second layer
|
||||
this.secondLayer = tf.layers.dense({
|
||||
units: this.secondLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
// Ouput layer
|
||||
this.outputLayer = tf.layers.dense({
|
||||
units: this.nbActions,
|
||||
kernelInitializer: tf.initializers.randomUniform({
|
||||
minval: 0.003, maxval: 0.003, seed: this.seed}),
|
||||
activation: 'tanh',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
// Actor prediction
|
||||
this.predict = (tfState) => {
|
||||
return tf.tidy(() => {
|
||||
if (tfState){
|
||||
obs = tfState;
|
||||
}
|
||||
|
||||
let l1 = this.firstLayer.apply(obs);
|
||||
let l2 = this.secondLayer.apply(l1);
|
||||
|
||||
return this.outputLayer.apply(l2);
|
||||
});
|
||||
}
|
||||
const output = this.predict();
|
||||
this.model = tf.model({inputs: obs, outputs: output});
|
||||
}
|
||||
};
|
||||
|
||||
class Critic {
|
||||
|
||||
/**
|
||||
* @param config (Object)
|
||||
*/
|
||||
constructor(config) {
|
||||
this.stateSize = config.stateSize;
|
||||
this.nbActions = config.nbActions;
|
||||
this.layerNorm = config.layerNorm;
|
||||
|
||||
this.firstLayerSSize = config.criticFirstLayerSSize
|
||||
this.firstLayerASize = config.criticFirstLayerASize;
|
||||
this.secondLayerSize = config.criticSecondLayerSize;
|
||||
|
||||
this.seed = config.seed;
|
||||
this.config = config;
|
||||
this.obs = null;
|
||||
this.action = null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param obs tf.input
|
||||
* @param action tf.input
|
||||
*/
|
||||
buildModel(obs, action){
|
||||
this.obs = obs;
|
||||
this.action = action;
|
||||
|
||||
// Used to merged the two first Layer later.
|
||||
this.add = tf.layers.add();
|
||||
|
||||
// First layer
|
||||
this.firstLayerS = tf.layers.dense({
|
||||
units: this.firstLayerSSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
// First layer
|
||||
this.firstLayerA = tf.layers.dense({
|
||||
units: this.firstLayerASize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
// Second layer
|
||||
this.secondLayer = tf.layers.dense({
|
||||
units: this.secondLayerSize,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'relu',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Ouput layer
|
||||
this.outputLayer = tf.layers.dense({
|
||||
units: 1,
|
||||
kernelInitializer: tf.initializers.randomUniform({
|
||||
minval: 0.003, maxval: 0.003, seed: this.seed}),
|
||||
activation: 'linear',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Critic prediction
|
||||
this.predict = (tfState, tfActions) => {
|
||||
return tf.tidy(() => {
|
||||
if (tfState && tfActions){
|
||||
obs = tfState;
|
||||
action = tfActions;
|
||||
}
|
||||
|
||||
let l1A = this.firstLayerA.apply(action);
|
||||
let l1S = this.firstLayerS.apply(obs)
|
||||
// Merged layers
|
||||
let concat = this.add.apply([l1A, l1S])
|
||||
|
||||
let l2 = this.secondLayer.apply(concat);
|
||||
|
||||
return this.outputLayer.apply(l2);
|
||||
});
|
||||
}
|
||||
|
||||
const output = this.predict();
|
||||
this.model = tf.model({inputs: [obs, action], outputs: output});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Noise class
|
||||
* The original baseline is made of three noise
|
||||
* AdaptiveParamNoiseSpec, ActionNoise and NormalActionNoise
|
||||
* Only AdaptiveParamNoiseSpec is implemented for now
|
||||
* See "C Adapative Scaling" Page 14 in the paper.
|
||||
*/
|
||||
|
||||
class AdaptiveParamNoiseSpec {
|
||||
|
||||
/**
|
||||
* @param conf Object
|
||||
* conf.initialStddev: 0.1 default // σ
|
||||
* conf.desiredActionStddev: 0.1 default // δ
|
||||
* conf.adoptionCoefficient: 1.01 default // α
|
||||
*/
|
||||
constructor(conf){
|
||||
conf = conf || {};
|
||||
this.initialStddev = conf.initialStddev || 0.4;
|
||||
this.desiredActionStddev = conf.desiredActionStddev || 0.4;
|
||||
this.adoptionCoefficient = conf.adoptionCoefficient || 1.01;
|
||||
this.currentStddev = this.initialStddev;
|
||||
}
|
||||
|
||||
/**
|
||||
* The distance from the Adaptive scaling
|
||||
* @param distance number
|
||||
*/
|
||||
adapt(distance){
|
||||
// if d(π, _π_) > δ then σ = σ/α
|
||||
if (distance > this.desiredActionStddev){
|
||||
// Decrease σ
|
||||
this.currentStddev /= this.adoptionCoefficient;
|
||||
}
|
||||
else{
|
||||
// σ = σ*α
|
||||
// Increase σ
|
||||
this.currentStddev *= this.adoptionCoefficient;
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,240 @@
|
||||
|
||||
class PrioritizedMemory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.buffer = [];
|
||||
this.priorBuffer = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
if (batchSize > this.priorBuffer.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
_bufferBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let nElem = this.buffer.pop();
|
||||
batch.obs0.push(nElem.obs0);
|
||||
batch.obs1.push(nElem.obs1);
|
||||
batch.rewards.push(nElem.reward);
|
||||
batch.actions.push(nElem.action);
|
||||
batch.terminals.push(nElem.terminal);
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
|
||||
return batch
|
||||
}
|
||||
|
||||
_addRandomBufferBatch(batchSize, batch){
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
popBatch(batchSize){
|
||||
let originalBatchSize = batchSize;
|
||||
let priorBufferBatchSize;
|
||||
let bufferBatchSize;
|
||||
if (batchSize % 2 != 0){
|
||||
console.warn("Batch size should be a even.")
|
||||
}
|
||||
if (this.priorBuffer.length < batchSize/2){
|
||||
//console.log("get full batch from buffer");
|
||||
const batch = this._bufferBatch(batchSize);
|
||||
console.assert(batch.obs0.length == batchSize);
|
||||
return batch;
|
||||
}
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
//console.log("Get half of prior and other from buffer.");
|
||||
batchSize = batchSize / 2;
|
||||
}
|
||||
else{
|
||||
//console.log("Get all from priorBuffer");
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
this.priorBuffer.splice(id, 1);
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
this._addRandomBufferBatch(batchSize, batch);
|
||||
}
|
||||
console.assert(batch.obs0.length == originalBatchSize);
|
||||
return batch
|
||||
}
|
||||
|
||||
_insert(element, array) {
|
||||
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){
|
||||
array.unshift(element);
|
||||
return array;
|
||||
}
|
||||
array.splice(this._locationOf(element, array) + 1, 0, element);
|
||||
return array;
|
||||
}
|
||||
|
||||
_locationOf(element, array, start, end) {
|
||||
start = start || 0;
|
||||
end = end || array.length;
|
||||
|
||||
var pivot = parseInt(start + (end - start) / 2, 10);
|
||||
|
||||
if (end-start <= 1 || array[pivot] === element) return pivot;
|
||||
|
||||
if (array[pivot].cost != null && array[pivot].cost < element.cost) {
|
||||
return this._locationOf(element, array, pivot, end);
|
||||
} else {
|
||||
return this._locationOf(element, array, start, pivot);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param batch (Object) from getBatch()
|
||||
* @param cost (number) Cost associated with each row of the batch
|
||||
*/
|
||||
appendBackWithCost(batch, costs){
|
||||
for (let b=0; b < batch.obs0.length; b++){
|
||||
if (this.buffer.length == this.maxlen){
|
||||
this.buffer.shift();
|
||||
}
|
||||
this._insert({
|
||||
obs0: batch.obs0[b],
|
||||
action: batch.actions[b],
|
||||
reward: batch.rewards[b],
|
||||
obs1: batch.obs1[b],
|
||||
terminal: batch.terminals[b],
|
||||
cost: costs[b]
|
||||
}, this.buffer);
|
||||
}
|
||||
console.assert(this.buffer.length <= this.maxlen);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal){
|
||||
if (this.priorBuffer.length == this.maxlen){
|
||||
this.priorBuffer.shift();
|
||||
}
|
||||
this.priorBuffer.push({
|
||||
obs0: obs0,
|
||||
action: action,
|
||||
reward: reward,
|
||||
obs1: obs1,
|
||||
terminal: terminal,
|
||||
cost: null
|
||||
});
|
||||
console.assert(this.priorBuffer.length <= this.maxlen);
|
||||
}
|
||||
}
|
||||
/*
|
||||
var mem = new Memory(20000);
|
||||
Math.seedrandom(0);
|
||||
console.assert(mem.length == 0);
|
||||
|
||||
var array = [];
|
||||
for (let i=1; i < 40000; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
|
||||
console.assert(mem.length == 20000);
|
||||
console.assert(mem.list[0].obs0 == "obs0-20000");
|
||||
console.assert(mem.list[19999].obs0 == "obs0-39999");
|
||||
|
||||
let batch = mem.getBatch(32);
|
||||
|
||||
console.assert(batch.obs0.length == 32);
|
||||
console.assert(mem.length == 20000 - 32);
|
||||
|
||||
let costs = [];
|
||||
for (i=31; i >= 0; i--){
|
||||
costs.push(i);
|
||||
}
|
||||
mem.appendBackWithCost(batch, costs);
|
||||
|
||||
console.log(mem.list);
|
||||
|
||||
/*
|
||||
for (let i=1; i < 64; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
*/
|
||||
@@ -16,5 +16,5 @@ editor.load().then(() => {
|
||||
localStorage.setItem('mylevel.json', JSON.stringify(content));
|
||||
window.open("/test_editor.html");
|
||||
|
||||
}, {download: false, name: "level.json"});
|
||||
}, {download: true, name: "level.json"});
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
let levelUrl = metacar.level.level2;
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state, 200, 200);
|
||||
let reward = env.getLastReward();
|
||||
displayScores("realtime_viewer", [], reward, []);
|
||||
});
|
||||
|
||||
env.load();
|
||||
@@ -7,7 +7,7 @@ var env = new metacar.env("canvas", levelUrl);
|
||||
var agent = new PolicyAgent(env);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
let state = env.getState().lidar;
|
||||
displayState("realtime_viewer", state, 200, 200);
|
||||
let scores = agent.getStateValues(state);
|
||||
let reward = env.getLastReward();
|
||||
@@ -30,6 +30,6 @@ env.load().then(() => {
|
||||
env.addEvent("save", () => agent.save());
|
||||
env.addEvent("load", () => {
|
||||
document.getElementById("metacar_canvas_button_train").style.display = "none";
|
||||
agent.restore()
|
||||
agent.restore();
|
||||
});
|
||||
});
|
||||
+4
-5
@@ -179,14 +179,12 @@ class PolicyAgent {
|
||||
*/
|
||||
this.valueModel = await tf.loadModel('https://metacar-project.com/public/models/policy/value-model-policy-agent.json');
|
||||
this.policyModel = await tf.loadModel("https://metacar-project.com/public/models/policy/policy-model-policy-agent.json");
|
||||
//this.valueModel = await tf.loadModel('http://localhost:3000/public/models/policy/value-model-policy-agent.json');
|
||||
//this.policyModel = await tf.loadModel("http://localhost:3000/public/models/policy/policy-model-policy-agent.json");
|
||||
}
|
||||
|
||||
play(){
|
||||
tf.tidy(() => {
|
||||
// Get the current state
|
||||
const st = tf.tensor2d(this.env.getState(), [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
const st = tf.tensor2d(this.env.getState().lidar, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
// Predict the policy
|
||||
const softmax = this.policyModel.predict(st);
|
||||
// Get the action
|
||||
@@ -231,7 +229,8 @@ class PolicyAgent {
|
||||
console.time("Exploring");
|
||||
for (var step = 0; step < this.nb_step; step++) {
|
||||
// Get the current state
|
||||
const array_st = this.env.getState(true);
|
||||
let array_st = this.env.getState().linear;
|
||||
array_st = array_st.slice(0, array_st.length - 1);
|
||||
// Convert the state into a tensor
|
||||
//const st = tf.tensor(array_st, [this.lidarPts, this.lidarPts]).reshape([1, this.ttLidarPts]);
|
||||
const st = tf.tensor2d([array_st]);
|
||||
@@ -301,7 +300,7 @@ class PolicyAgent {
|
||||
tf_advantages.dispose();
|
||||
|
||||
// Set the agent on a new free road
|
||||
this.env.randomRoadPosition();
|
||||
this.env.shuffle({cars: false});
|
||||
//env.reset();
|
||||
// Go to the next episode
|
||||
setTimeout(() => this.train(this.env, it+1), 1);
|
||||
@@ -12,8 +12,8 @@ var agent = new QTableAgent(env, 2);
|
||||
|
||||
env.loop(() => {
|
||||
let state = env.getState();
|
||||
displayState("realtime_viewer", state, 200, 200);
|
||||
let scores = agent.getStateValues(state);
|
||||
displayState("realtime_viewer", state.lidar, 200, 200);
|
||||
let scores = agent.getStateValues(state.lidar);
|
||||
let reward = env.getLastReward();
|
||||
displayScores("realtime_viewer", scores, reward, ["Top", "Left", "Right"]);
|
||||
});
|
||||
+4
-4
@@ -54,7 +54,7 @@ class QTableAgent {
|
||||
|
||||
play(){
|
||||
// Get the current state
|
||||
let state = this.env.getState();
|
||||
let state = this.env.getState().lidar;
|
||||
state = state.toString();
|
||||
// In this state in not in the Q(s, a) function
|
||||
if (!(state in this.Q)){
|
||||
@@ -108,7 +108,7 @@ class QTableAgent {
|
||||
console.log("episode=", ep, "eps=", eps, "mean_reward", mean(mean_reward));
|
||||
}
|
||||
mean_reward = [];
|
||||
let st = this.env.getState().toString();
|
||||
let st = this.env.getState().lidar.toString();
|
||||
let act;
|
||||
let gamma = 0.99;
|
||||
let st2;
|
||||
@@ -117,7 +117,7 @@ class QTableAgent {
|
||||
act = this.pickAction(st, eps);
|
||||
let reward = this.env.step(act);
|
||||
mean_reward.push(reward);
|
||||
st2 = this.env.getState().toString();
|
||||
st2 = this.env.getState().lidar.toString();
|
||||
// Pick greedy action (eps = 0)
|
||||
act2 = this.pickAction(st2, 0.);
|
||||
this.createStateIfNotExist(st2);
|
||||
@@ -125,7 +125,7 @@ class QTableAgent {
|
||||
this.Q[st][act] = this.Q[st][act] + 0.01*(reward + (gamma*this.Q[st2][act2]) - this.Q[st][act]);
|
||||
st = st2;
|
||||
}
|
||||
this.env.randomRoadPosition();
|
||||
this.env.shuffle({cars: false});
|
||||
}
|
||||
this.env.render(true);
|
||||
for (let s=0; s < this.stateList.length; s++){
|
||||
@@ -112,4 +112,51 @@ function displayState(id, state, width, height){
|
||||
yPos += ySize;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
METRICS = {};
|
||||
|
||||
function initMetricsContainer(container, metrics){
|
||||
container = document.getElementById(container);
|
||||
for (let m=0; m < metrics.length; m++){
|
||||
nDiv = document.createElement("div");
|
||||
nDiv.id = 'metrics_'+metrics[m];
|
||||
nDiv.style.width = "250px";
|
||||
nDiv.style.height = "250px";
|
||||
nDiv.style.display = "inline-block";
|
||||
nDiv.style.marginRight = "10px";
|
||||
container.appendChild(nDiv);
|
||||
|
||||
METRICS[metrics[m]] = new CanvasJS.Chart('metrics_'+metrics[m], {
|
||||
width: 250,
|
||||
height: 250,
|
||||
animationEnabled: false,
|
||||
theme: "light2",
|
||||
title:{
|
||||
text: metrics[m],
|
||||
},
|
||||
axisY:{
|
||||
includeZero: false
|
||||
},
|
||||
data: [{
|
||||
type: "line",
|
||||
dataPoints: [{y: 0}]
|
||||
}]
|
||||
});
|
||||
METRICS[metrics[m]].render();
|
||||
}
|
||||
}
|
||||
|
||||
function setMetric(name, value){
|
||||
let chart = METRICS[name];
|
||||
let size = chart.options.data[0].dataPoints.length - 1;
|
||||
|
||||
if (chart.options.data[0].dataPoints.length > 500){
|
||||
chart.options.data[0].dataPoints = chart.options.data[0].dataPoints.slice(1, size);
|
||||
size = size - 1;
|
||||
}
|
||||
|
||||
size = chart.options.data[0].dataPoints.length - 1;
|
||||
chart.options.data[0].dataPoints.push({y: value, x: chart.options.data[0].dataPoints[size].x+1});
|
||||
chart.render();
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[32,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,26],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":64,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-road.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,64],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[26,64],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[64,32],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[32,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model7","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense22","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense22","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense23","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense23","trainable":true},"inbound_nodes":[[["dense_Dense22",0,0,{}]]]},{"name":"dense_Dense24","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense24","trainable":true},"inbound_nodes":[[["dense_Dense23",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense24",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-traffic-epoch-120.weights.bin"],"weights":[{"name":"dense_Dense22/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense22/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense23/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense23/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense24/kernel","shape":[64,2],"dtype":"float32"},{"name":"dense_Dense24/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model6","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense19","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense19","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense18","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense18","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add3","class_name":"Add","config":{"name":"add_Add3","trainable":true},"inbound_nodes":[[["dense_Dense19",0,0,{}],["dense_Dense18",0,0,{}]]]},{"name":"dense_Dense20","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense20","trainable":true},"inbound_nodes":[[["add_Add3",0,0,{}]]]},{"name":"dense_Dense21","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense21","trainable":true},"inbound_nodes":[[["dense_Dense20",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense21",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic-epoch-120.weights.bin"],"weights":[{"name":"dense_Dense19/kernel","shape":[2,128],"dtype":"float32"},{"name":"dense_Dense19/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense18/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense18/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense20/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense20/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense21/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense21/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
BIN
Binary file not shown.
@@ -2,7 +2,7 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Metacar: Q-learning demo</title>
|
||||
<title>Metacar: Discrete Actions.</title>
|
||||
<link rel="icon" href="/public/img/icon.png">
|
||||
<link rel="stylesheet" href="/public/css/general.css">
|
||||
<link href="https://fonts.googleapis.com/css?family=Tajawal" rel="stylesheet">
|
||||
@@ -25,7 +25,7 @@
|
||||
</div>
|
||||
|
||||
<div class="body_container">
|
||||
<h3>Current state (Lidar points)</h3><br>
|
||||
<h3>Current state (Lidar points)</h3><br><br>
|
||||
<div id="realtime_viewer" style="float: left"></div>
|
||||
<p>
|
||||
<b>Q-learning</b> algorithm with table. <br>
|
||||
@@ -54,8 +54,9 @@
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
<script type="text/javascript" src="/public/js/q_table_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/level0.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/q_table/q_table_agent.js"></script>
|
||||
<script type="text/javascript" src="/public/js/q_table/index.js"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -133,13 +133,16 @@ export class AssetManger {
|
||||
@line is Optional. 0 By default.
|
||||
*/
|
||||
if (line == undefined)
|
||||
line = 0;
|
||||
line = Math.min(1, Math.floor(Math.random() * 2));
|
||||
let carById = this.level.findCarById(road.cars[0]);
|
||||
if (!force && road.cars.length >= 1 && carById && carById.core.line == line){
|
||||
line = line == 0 ? 1:0;
|
||||
}
|
||||
car.line = line;
|
||||
|
||||
car.a = 0;
|
||||
car.v = 0;
|
||||
|
||||
// Set the car on the road
|
||||
car.x = road.x;
|
||||
car.y = road.y;
|
||||
@@ -158,7 +161,6 @@ export class AssetManger {
|
||||
let line_side_factor_t = line == 0 ? 0:Math.PI;
|
||||
car.x += line_side_factor*Math.round(x_m);
|
||||
car.y += line_side_factor*Math.round(y_m);
|
||||
|
||||
// (x, y position) Relatif to the map
|
||||
car.mx = Math.floor(car.x / ROADSIZE);
|
||||
car.my = Math.floor(car.y / ROADSIZE);
|
||||
|
||||
@@ -21,6 +21,7 @@ export class BasicMotionEngine extends MotionEngine {
|
||||
*/
|
||||
private rotationStep: number;
|
||||
private actions: string[];
|
||||
private maxSpeed: number = 1.0;
|
||||
|
||||
constructor(level: Level, options: BasicMotionOptions) {
|
||||
super(level);
|
||||
@@ -49,6 +50,8 @@ export class BasicMotionEngine extends MotionEngine {
|
||||
this.setUpKeyboard();
|
||||
// Setup up velocity to 0
|
||||
this.car.v = 0;
|
||||
this.car.a = 0; // Acceleration
|
||||
this.car.yaw_rate = 0;
|
||||
|
||||
this.detectInteractions();
|
||||
}
|
||||
|
||||
@@ -182,6 +182,7 @@ export class BotMotionEngine extends MotionEngine {
|
||||
let turn = false;
|
||||
let nx = this.car.mx + is_next_pos.mx;
|
||||
let ny = this.car.my + is_next_pos.my;
|
||||
|
||||
if (!this.isRoad(nx, ny)) {
|
||||
turn = true;
|
||||
this.car.optionalTurn = false;
|
||||
|
||||
+32
-10
@@ -11,6 +11,7 @@ import {
|
||||
CAR_IMG, Sprite, MAP, ROADSIZE, Container, Graphics
|
||||
} from "./global";
|
||||
import { RoadSprite } from "./asset_manager";
|
||||
import { runInThisContext } from "vm";
|
||||
|
||||
var Global_carId = 0;
|
||||
|
||||
@@ -55,6 +56,12 @@ export interface CarSprite extends PIXI.Sprite {
|
||||
optionalTurn?: boolean;
|
||||
agent?: boolean;
|
||||
v?: number;
|
||||
a?: number;
|
||||
yaw_rate?: number;
|
||||
last_a?: number;
|
||||
last_yaw_rate?: number;
|
||||
next_road?: any;
|
||||
turnedRandom?: any;
|
||||
}
|
||||
|
||||
export interface LidarChild extends PIXI.Graphics {
|
||||
@@ -62,6 +69,14 @@ export interface LidarChild extends PIXI.Graphics {
|
||||
pt?: boolean;
|
||||
}
|
||||
|
||||
export interface State {
|
||||
lidar?: number[][];
|
||||
linear?: number[];
|
||||
v?: number;
|
||||
a?: number;
|
||||
steering?: number;
|
||||
}
|
||||
|
||||
export class Car {
|
||||
|
||||
public level: Level|Editor;
|
||||
@@ -70,7 +85,6 @@ export class Car {
|
||||
public motion: any;
|
||||
|
||||
private info: CarInfo;
|
||||
public turnedRandom: any;
|
||||
|
||||
constructor(level: Level|Editor, info: CarInfo, textures: any, options:CarOptions={}) {
|
||||
/*
|
||||
@@ -162,7 +176,8 @@ export class Car {
|
||||
// TOdo: CHECK something is strange here
|
||||
this.core.haveTurned = false;
|
||||
this.core.optionalTurn = false;
|
||||
this.turnedRandom = undefined;
|
||||
this.core.turnedRandom = undefined;
|
||||
this.core.next_road = false;
|
||||
if (n_road)
|
||||
n_road.cars.push(this.core.carId);
|
||||
}
|
||||
@@ -181,19 +196,26 @@ export class Car {
|
||||
}
|
||||
}
|
||||
|
||||
getState(linear:boolean = false): number[][]|number[]{
|
||||
getState(): State {
|
||||
/*
|
||||
Get the current state of the car
|
||||
The state is the current value of each point
|
||||
of the lidar.
|
||||
*/
|
||||
if (!linear)
|
||||
return this.motion.state.map(function(arr: any) { return arr.slice(); });
|
||||
else{
|
||||
let state: number[] = [];
|
||||
this.motion.state.map((row: number[]) => { state = state.concat(row);});
|
||||
return state;
|
||||
}
|
||||
let nState: State = {};
|
||||
|
||||
let linear: number[] = [];
|
||||
this.motion.state.map((row: number[]) => { linear = linear.concat(row);});
|
||||
linear.push(this.core.v);
|
||||
|
||||
nState.linear = linear;
|
||||
nState.lidar = this.motion.state.map(function(arr: any) { return arr.slice(); });
|
||||
|
||||
nState.v = this.core.v;
|
||||
nState.a = this.core.last_a;
|
||||
nState.steering = this.core.last_yaw_rate;
|
||||
|
||||
return nState;
|
||||
}
|
||||
|
||||
step(delta: number, action:number|number[]=null){
|
||||
|
||||
@@ -10,6 +10,7 @@ import * as U from "./utils";
|
||||
export class ControlMotionEngine extends MotionEngine {
|
||||
|
||||
private actions: (string|number)[];
|
||||
private maxSpeed: number = 2.0;
|
||||
|
||||
constructor(level: Level) {
|
||||
/*
|
||||
@@ -133,6 +134,9 @@ export class ControlMotionEngine extends MotionEngine {
|
||||
Step into the environement
|
||||
@delta (Float) time since the last update
|
||||
*/
|
||||
this.car.last_a = this.car.a;
|
||||
this.car.last_yaw_rate = this.car.yaw_rate;
|
||||
|
||||
// The car lose speed over time
|
||||
if (this.car.v > 0 && this.car.a == 0)
|
||||
this.car.v = Math.max(0, this.car.v - 0.01);
|
||||
@@ -140,14 +144,14 @@ export class ControlMotionEngine extends MotionEngine {
|
||||
this.car.v = Math.min(0, this.car.v + 0.01);
|
||||
|
||||
if (this.car.a > 0 && this.car.v >= 0)
|
||||
this.car.v = Math.min(1.5, this.car.v + this.car.a*0.01);
|
||||
this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.005);
|
||||
else if (this.car.a > 0 && this.car.v < 0){
|
||||
this.car.v = Math.min(1.5, this.car.v + this.car.a*0.03);
|
||||
this.car.v = Math.min(this.maxSpeed, this.car.v + this.car.a*0.03);
|
||||
}
|
||||
if (this.car.a < 0 && this.car.v <= 0)
|
||||
this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.01);
|
||||
this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.005);
|
||||
else if (this.car.a < 0 && this.car.v > 0){
|
||||
this.car.v = Math.max(-1.5, this.car.v + this.car.a*0.03);
|
||||
this.car.v = Math.max(-this.maxSpeed, this.car.v + this.car.a*0.03);
|
||||
}
|
||||
if (this.car.yaw_rate == 0){
|
||||
this.car.x += this.car.v * Math.cos(this.car.rotation)*delta;
|
||||
|
||||
+5
-1
@@ -2,6 +2,7 @@ import {fullCity} from "./embedded/level/full_city";
|
||||
import {level1} from "./embedded/level/level_1";
|
||||
import {level0} from "./embedded/level/level_0";
|
||||
import {level2} from "./embedded/level/level_2";
|
||||
import {level3} from "./embedded/level/level_3";
|
||||
|
||||
/**
|
||||
* Object used to enumerate each
|
||||
@@ -17,6 +18,7 @@ export interface embeddedUrlI {
|
||||
fullCity: string;
|
||||
level1: string;
|
||||
level2: string;
|
||||
level3: string;
|
||||
level0: string;
|
||||
};
|
||||
|
||||
@@ -24,6 +26,7 @@ export const embeddedUrl: embeddedUrlI = {
|
||||
fullCity: "embedded://level/fullCity",
|
||||
level1: "embedded://level/level1",
|
||||
level2: "embedded://level/level2",
|
||||
level3: "embedded://level/level3",
|
||||
level0: "embedded://level/level0",
|
||||
}
|
||||
|
||||
@@ -32,6 +35,7 @@ export const embeddedContent: any = {
|
||||
fullCity: fullCity,
|
||||
level1: level1,
|
||||
level0: level0,
|
||||
level2: level2
|
||||
level2: level2,
|
||||
level3: level3
|
||||
}
|
||||
}
|
||||
+304
-99
@@ -1,114 +1,319 @@
|
||||
export const level2: any = {
|
||||
"cars": [
|
||||
"cars": [],
|
||||
"road": [
|
||||
{
|
||||
"mx": 7,
|
||||
"my": 7,
|
||||
"line": 0
|
||||
"x": 300,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"mx": 9,
|
||||
"my": 6,
|
||||
"line": 0
|
||||
"x": 300,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"mx": 8,
|
||||
"my": 6,
|
||||
"line": 1
|
||||
"x": 360,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"mx": 10,
|
||||
"my": 6,
|
||||
"line": 1
|
||||
"x": 420,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 120,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 120,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 120,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
}
|
||||
],
|
||||
"asset": [
|
||||
{
|
||||
"x": 162,
|
||||
"y": 264
|
||||
},
|
||||
{
|
||||
"x": 407,
|
||||
"y": 264
|
||||
},
|
||||
{
|
||||
"x": 259,
|
||||
"y": 126
|
||||
},
|
||||
{
|
||||
"x": 306,
|
||||
"y": 33
|
||||
},
|
||||
{
|
||||
"x": 491,
|
||||
"y": 148
|
||||
},
|
||||
{
|
||||
"x": 391,
|
||||
"y": 431
|
||||
},
|
||||
{
|
||||
"x": 608,
|
||||
"y": 35
|
||||
},
|
||||
{
|
||||
"x": 484,
|
||||
"y": 224
|
||||
},
|
||||
{
|
||||
"x": 248,
|
||||
"y": 297
|
||||
},
|
||||
{
|
||||
"x": 376,
|
||||
"y": 336
|
||||
},
|
||||
{
|
||||
"x": 78,
|
||||
"y": 429
|
||||
},
|
||||
{
|
||||
"x": 1,
|
||||
"y": 205
|
||||
},
|
||||
{
|
||||
"x": 3,
|
||||
"y": 106
|
||||
},
|
||||
{
|
||||
"x": -46,
|
||||
"y": 290
|
||||
},
|
||||
{
|
||||
"x": 622,
|
||||
"y": 154
|
||||
},
|
||||
{
|
||||
"x": 617,
|
||||
"y": 280
|
||||
}
|
||||
],
|
||||
"house": [
|
||||
{
|
||||
"x": 68,
|
||||
"y": 317
|
||||
}
|
||||
],
|
||||
"house2": [
|
||||
{
|
||||
"x": 624,
|
||||
"y": 84
|
||||
"x": 162,
|
||||
"y": 264
|
||||
},
|
||||
{
|
||||
"x": -15,
|
||||
"y": 144
|
||||
"x": 622,
|
||||
"y": 154
|
||||
}
|
||||
],
|
||||
"house3": [
|
||||
{
|
||||
"x": 618,
|
||||
"y": 231
|
||||
"x": 407,
|
||||
"y": 264
|
||||
},
|
||||
{
|
||||
"x": 13,
|
||||
"y": 441
|
||||
}
|
||||
],
|
||||
"bench": [
|
||||
{
|
||||
"x": 318,
|
||||
"y": 216
|
||||
},
|
||||
{
|
||||
"x": 422,
|
||||
"y": 36
|
||||
"x": -46,
|
||||
"y": 290
|
||||
}
|
||||
],
|
||||
"tree": [
|
||||
{
|
||||
"x": 151,
|
||||
"y": 144
|
||||
"x": 259,
|
||||
"y": 126
|
||||
},
|
||||
{
|
||||
"x": 229,
|
||||
"y": 175
|
||||
"x": 491,
|
||||
"y": 148
|
||||
},
|
||||
{
|
||||
"x": 326,
|
||||
"y": 132
|
||||
"x": 391,
|
||||
"y": 431
|
||||
},
|
||||
{
|
||||
"x": 413,
|
||||
"y": 167
|
||||
"x": 608,
|
||||
"y": 35
|
||||
},
|
||||
{
|
||||
"x": 487,
|
||||
"y": 157
|
||||
"x": 484,
|
||||
"y": 224
|
||||
},
|
||||
{
|
||||
"x": 445,
|
||||
"y": 123
|
||||
"x": 248,
|
||||
"y": 297
|
||||
},
|
||||
{
|
||||
"x": 511,
|
||||
"x": 78,
|
||||
"y": 429
|
||||
},
|
||||
{
|
||||
"x": 584,
|
||||
"y": 427
|
||||
"x": 1,
|
||||
"y": 205
|
||||
},
|
||||
{
|
||||
"x": 80,
|
||||
"y": 391
|
||||
"x": 3,
|
||||
"y": 106
|
||||
},
|
||||
{
|
||||
"x": 219,
|
||||
"y": 409
|
||||
"x": 617,
|
||||
"y": 280
|
||||
}
|
||||
],
|
||||
"bench": [
|
||||
{
|
||||
"x": 306,
|
||||
"y": 33
|
||||
},
|
||||
{
|
||||
"x": 207,
|
||||
"y": 326
|
||||
},
|
||||
{
|
||||
"x": 156,
|
||||
"y": 426
|
||||
},
|
||||
{
|
||||
"x": 162,
|
||||
"y": 13
|
||||
"x": 376,
|
||||
"y": 336
|
||||
}
|
||||
],
|
||||
"map": [
|
||||
@@ -129,11 +334,11 @@ export const level2: any = {
|
||||
0,
|
||||
"↱",
|
||||
"↔",
|
||||
"↧",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↧",
|
||||
"↔",
|
||||
"↰",
|
||||
0
|
||||
@@ -142,9 +347,35 @@ export const level2: any = {
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↳",
|
||||
"↔",
|
||||
"↧",
|
||||
"↠",
|
||||
"↲",
|
||||
0,
|
||||
"↕",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↟",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
@@ -157,7 +388,7 @@ export const level2: any = {
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
@@ -170,10 +401,10 @@ export const level2: any = {
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↧",
|
||||
"↔",
|
||||
"↥",
|
||||
"↔",
|
||||
"↔",
|
||||
"↠",
|
||||
"↲",
|
||||
0
|
||||
],
|
||||
@@ -183,43 +414,17 @@ export const level2: any = {
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↱",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔"
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
],
|
||||
"agent": {
|
||||
"mx": 2,
|
||||
"my": 4,
|
||||
"mx": 4,
|
||||
"my": 3,
|
||||
"line": 0,
|
||||
"motion": {
|
||||
"type": "BasicMotionEngine",
|
||||
|
||||
@@ -0,0 +1,816 @@
|
||||
export const level3: any = {
|
||||
"cars": [
|
||||
{
|
||||
"mx": 1,
|
||||
"my": 3,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 1,
|
||||
"my": 8,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 4,
|
||||
"my": 8,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 7,
|
||||
"my": 7,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 9,
|
||||
"my": 6,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 11,
|
||||
"my": 4,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 9,
|
||||
"my": 3,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 7,
|
||||
"my": 2,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 5,
|
||||
"my": 1,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 2,
|
||||
"my": 1,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 6,
|
||||
"my": 8,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 1,
|
||||
"my": 6,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 5,
|
||||
"my": 5,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 4,
|
||||
"my": 3,
|
||||
"line": 1
|
||||
},
|
||||
{
|
||||
"mx": 3,
|
||||
"my": 4,
|
||||
"line": 0
|
||||
},
|
||||
{
|
||||
"mx": 4,
|
||||
"my": 6,
|
||||
"line": 0
|
||||
}
|
||||
],
|
||||
"road": [
|
||||
{
|
||||
"x": 120,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 420
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 120,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 420
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 120
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 660,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 600,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 660,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 660,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 660,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 600,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 420
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 420
|
||||
},
|
||||
{
|
||||
"x": 420,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 360,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 360
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 240
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 180,
|
||||
"y": 60
|
||||
},
|
||||
{
|
||||
"x": 60,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 300,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 240,
|
||||
"y": 480
|
||||
},
|
||||
{
|
||||
"x": 600,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 540,
|
||||
"y": 180
|
||||
},
|
||||
{
|
||||
"x": 660,
|
||||
"y": 300
|
||||
},
|
||||
{
|
||||
"x": 480,
|
||||
"y": 360
|
||||
}
|
||||
],
|
||||
"asset": [
|
||||
{
|
||||
"x": 515,
|
||||
"y": 262
|
||||
},
|
||||
{
|
||||
"x": 378,
|
||||
"y": 254
|
||||
},
|
||||
{
|
||||
"x": 510,
|
||||
"y": 89
|
||||
},
|
||||
{
|
||||
"x": 626,
|
||||
"y": 86
|
||||
},
|
||||
{
|
||||
"x": 518,
|
||||
"y": 445
|
||||
},
|
||||
{
|
||||
"x": 631,
|
||||
"y": 436
|
||||
},
|
||||
{
|
||||
"x": 711,
|
||||
"y": 434
|
||||
},
|
||||
{
|
||||
"x": 243,
|
||||
"y": 295
|
||||
},
|
||||
{
|
||||
"x": 243,
|
||||
"y": 246
|
||||
},
|
||||
{
|
||||
"x": 253,
|
||||
"y": 153
|
||||
},
|
||||
{
|
||||
"x": 16,
|
||||
"y": 521
|
||||
},
|
||||
{
|
||||
"x": 129,
|
||||
"y": 214
|
||||
},
|
||||
{
|
||||
"x": 121,
|
||||
"y": 228
|
||||
},
|
||||
{
|
||||
"x": 238,
|
||||
"y": 425
|
||||
},
|
||||
{
|
||||
"x": 456,
|
||||
"y": 273
|
||||
},
|
||||
{
|
||||
"x": 584,
|
||||
"y": 531
|
||||
},
|
||||
{
|
||||
"x": 376,
|
||||
"y": 558
|
||||
},
|
||||
{
|
||||
"x": 63,
|
||||
"y": 539
|
||||
},
|
||||
{
|
||||
"x": 91,
|
||||
"y": 559
|
||||
},
|
||||
{
|
||||
"x": 4,
|
||||
"y": 515
|
||||
},
|
||||
{
|
||||
"x": -3,
|
||||
"y": 315
|
||||
},
|
||||
{
|
||||
"x": -3,
|
||||
"y": 183
|
||||
},
|
||||
{
|
||||
"x": 368,
|
||||
"y": 188
|
||||
},
|
||||
{
|
||||
"x": 500,
|
||||
"y": 18
|
||||
},
|
||||
{
|
||||
"x": 610,
|
||||
"y": 16
|
||||
},
|
||||
{
|
||||
"x": 685,
|
||||
"y": 13
|
||||
},
|
||||
{
|
||||
"x": 273,
|
||||
"y": 0
|
||||
},
|
||||
{
|
||||
"x": 738,
|
||||
"y": 359
|
||||
},
|
||||
{
|
||||
"x": 742,
|
||||
"y": 181
|
||||
},
|
||||
{
|
||||
"x": 752,
|
||||
"y": 12
|
||||
}
|
||||
],
|
||||
"house3": [
|
||||
{
|
||||
"x": 515,
|
||||
"y": 262
|
||||
},
|
||||
{
|
||||
"x": 626,
|
||||
"y": 86
|
||||
}
|
||||
],
|
||||
"house2": [
|
||||
{
|
||||
"x": 378,
|
||||
"y": 254
|
||||
},
|
||||
{
|
||||
"x": 631,
|
||||
"y": 436
|
||||
},
|
||||
{
|
||||
"x": 711,
|
||||
"y": 434
|
||||
},
|
||||
{
|
||||
"x": 752,
|
||||
"y": 12
|
||||
}
|
||||
],
|
||||
"house": [
|
||||
{
|
||||
"x": 510,
|
||||
"y": 89
|
||||
},
|
||||
{
|
||||
"x": 518,
|
||||
"y": 445
|
||||
},
|
||||
{
|
||||
"x": 742,
|
||||
"y": 181
|
||||
}
|
||||
],
|
||||
"tree": [
|
||||
{
|
||||
"x": 243,
|
||||
"y": 295
|
||||
},
|
||||
{
|
||||
"x": 243,
|
||||
"y": 246
|
||||
},
|
||||
{
|
||||
"x": 121,
|
||||
"y": 228
|
||||
},
|
||||
{
|
||||
"x": 238,
|
||||
"y": 425
|
||||
},
|
||||
{
|
||||
"x": 456,
|
||||
"y": 273
|
||||
},
|
||||
{
|
||||
"x": 584,
|
||||
"y": 531
|
||||
},
|
||||
{
|
||||
"x": 376,
|
||||
"y": 558
|
||||
},
|
||||
{
|
||||
"x": 91,
|
||||
"y": 559
|
||||
},
|
||||
{
|
||||
"x": 4,
|
||||
"y": 515
|
||||
},
|
||||
{
|
||||
"x": -3,
|
||||
"y": 315
|
||||
},
|
||||
{
|
||||
"x": -3,
|
||||
"y": 183
|
||||
},
|
||||
{
|
||||
"x": 368,
|
||||
"y": 188
|
||||
},
|
||||
{
|
||||
"x": 500,
|
||||
"y": 18
|
||||
},
|
||||
{
|
||||
"x": 610,
|
||||
"y": 16
|
||||
},
|
||||
{
|
||||
"x": 685,
|
||||
"y": 13
|
||||
},
|
||||
{
|
||||
"x": 273,
|
||||
"y": 0
|
||||
},
|
||||
{
|
||||
"x": 738,
|
||||
"y": 359
|
||||
}
|
||||
],
|
||||
"bench": [
|
||||
{
|
||||
"x": 253,
|
||||
"y": 153
|
||||
}
|
||||
],
|
||||
"map": [
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↱",
|
||||
"↔",
|
||||
"↠",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↰",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↱",
|
||||
"↠",
|
||||
"↰",
|
||||
0,
|
||||
"↳",
|
||||
"↔",
|
||||
"↠",
|
||||
"↔",
|
||||
"↰",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↟",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↟",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
"↳",
|
||||
"↔",
|
||||
"↲",
|
||||
0,
|
||||
"↱",
|
||||
"↠",
|
||||
"↔",
|
||||
"↔",
|
||||
"↲",
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
"↕",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
"↳",
|
||||
"↔",
|
||||
"↔",
|
||||
"↔",
|
||||
"↠",
|
||||
"↔",
|
||||
"↲",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
],
|
||||
"agent": {
|
||||
"mx": 3,
|
||||
"my": 8,
|
||||
"line": 0,
|
||||
"motion": {
|
||||
"type": "BasicMotionEngine",
|
||||
"options": {
|
||||
"rotationStep": 0.5,
|
||||
"actions": [
|
||||
"UP",
|
||||
"LEFT",
|
||||
"RIGHT",
|
||||
"DOWN",
|
||||
"WAIT"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+7
-8
@@ -105,17 +105,12 @@ export class Level extends World {
|
||||
*/
|
||||
this.agent.reset();
|
||||
for (var c = 0; c < this.cars.length; c++) {
|
||||
//this.cars[c].reset();
|
||||
this.cars[c].reset();
|
||||
}
|
||||
}
|
||||
|
||||
setReward(agent_col: any, on_road: any, action: any){
|
||||
/*
|
||||
TODO: Let's the reward define in the agent class
|
||||
*/
|
||||
let reward = -0.1;
|
||||
if (action == 0 || this.agent.core.v == 1)
|
||||
reward += 0.5;
|
||||
let reward = 0 + Math.max(0., this.agent.core.v) / this.agent.motion.maxSpeed;
|
||||
if (agent_col.length > 0){
|
||||
reward = -1;
|
||||
}
|
||||
@@ -133,12 +128,16 @@ export class Level extends World {
|
||||
}
|
||||
|
||||
|
||||
step(delta: number, action:number|number[]=null){
|
||||
step(delta: number, action:number|number[]=null, auto: boolean = true){
|
||||
/*
|
||||
Process one step into the environement
|
||||
@delta (Float) time since the last update
|
||||
@action: (Integer) The action to take (can be null if no action)
|
||||
*/
|
||||
if (auto && !this.steping){
|
||||
return;
|
||||
}
|
||||
|
||||
// Go through all cars to move each one
|
||||
for (var c = 0; c < this.cars.length; c++) {
|
||||
if (this.cars[c].lidar && !this.cars[c].core.agent) // If this car can move
|
||||
|
||||
+41
-14
@@ -8,7 +8,7 @@ import {UIEvent} from "./ui_event";
|
||||
import * as U from "./utils";
|
||||
import { BasicMotionEngine, BasicMotionOptions } from "./basic_motion_engine";
|
||||
import { ControlMotionEngine } from "./control_motion_engine";
|
||||
import { LidarInfoI } from "./car";
|
||||
import { LidarInfoI, State } from "./car";
|
||||
|
||||
/**
|
||||
* @local Chooce whether to load a file from the computer.
|
||||
@@ -18,6 +18,15 @@ export interface eventLoadOptions {
|
||||
local: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* @cars Shuffle the position of the other cars (true default)
|
||||
* @agent Shuffle the position of the agent (true default)
|
||||
*/
|
||||
export interface shuffleConfig{
|
||||
agent: boolean;
|
||||
cars: boolean;
|
||||
}
|
||||
|
||||
export class MetaCar {
|
||||
|
||||
private level: Level;
|
||||
@@ -132,6 +141,14 @@ export class MetaCar {
|
||||
this.level.render(val);
|
||||
}
|
||||
|
||||
/**
|
||||
* Choose wheter the environment should step automaticly
|
||||
* @param val True or False
|
||||
*/
|
||||
public steping(val: boolean){
|
||||
this.level.setSteping(val);
|
||||
}
|
||||
|
||||
/**
|
||||
* Usefull method to save/download a string as file.
|
||||
* @content The content of the file
|
||||
@@ -157,8 +174,8 @@ export class MetaCar {
|
||||
* The size of the state depends of the size of the Lidar.
|
||||
* @return The state as a 2D Array or 1D Array (linear:true)
|
||||
*/
|
||||
public getState(linear:boolean = false): number[][]|number[]{
|
||||
return this.level.agent.getState(linear);
|
||||
public getState(): State{
|
||||
return this.level.agent.getState();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -167,7 +184,7 @@ export class MetaCar {
|
||||
@return Reward value
|
||||
*/
|
||||
public step(action: number|number[]): number{
|
||||
return this.level.step(1, action);
|
||||
return this.level.step(1, action, false);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -181,22 +198,32 @@ export class MetaCar {
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the agent on a new random road on the map.
|
||||
* Shuffle the position of the agent and the others
|
||||
* cars.
|
||||
*/
|
||||
randomRoadPosition(): void{
|
||||
shuffle(config: shuffleConfig): void{
|
||||
/*
|
||||
This position
|
||||
*/
|
||||
let roads = this.level.getRoads();
|
||||
let keys = Object.keys(roads);
|
||||
keys.sort(function() {return Math.random()-0.5;});
|
||||
for (let k in keys){
|
||||
let road = roads[keys[k]];
|
||||
if (road.cars.length == 0){
|
||||
road.setCarPosition(this.level.agent.core);
|
||||
break;
|
||||
config = config || {cars: true, agent: true};
|
||||
config.cars = config.cars != undefined ? config.cars:true;
|
||||
config.agent = config.agent != undefined ? config.agent:true;
|
||||
|
||||
if (config.agent) {
|
||||
let roads = this.level.getRoads();
|
||||
let keys = Object.keys(roads);
|
||||
keys.sort(function() {return Math.random()-0.5;});
|
||||
for (let k in keys){
|
||||
let road = roads[keys[k]];
|
||||
if (road.cars.length == 0){
|
||||
road.setCarPosition(this.level.agent.core);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (config.cars){
|
||||
this.level.shuffleCarsPositions();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -61,6 +61,7 @@ export class UIEvent {
|
||||
// Listen the event
|
||||
button.addEventListener("click", () => {
|
||||
this.level.render(false);
|
||||
this.level.setSteping(false);
|
||||
if (fc) fc();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ export class World {
|
||||
protected loop: any; // Loop method called for each render
|
||||
protected canvasId: string; // Id of the target canvas
|
||||
protected cars: Car[] = [];
|
||||
protected steping: boolean = true;
|
||||
|
||||
constructor(levelContent: LevelInfo, canvasId: string) {
|
||||
/*
|
||||
@@ -124,9 +125,11 @@ export class World {
|
||||
render(val: boolean){
|
||||
if (val){
|
||||
this.app.ticker.start();
|
||||
this.steping = true;
|
||||
}
|
||||
else{
|
||||
this.app.ticker.stop();
|
||||
this.steping = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,4 +147,28 @@ export class World {
|
||||
return this.app.renderer.plugins.interaction.mouse.global;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop stepping in the environment automaticly
|
||||
*/
|
||||
public setSteping(val: boolean): void{
|
||||
this.steping = val;
|
||||
}
|
||||
|
||||
public shuffleCarsPositions(){
|
||||
for (let c = 0; c < this.cars.length; c++){
|
||||
|
||||
let roads = this.roads;
|
||||
let keys = Object.keys(roads);
|
||||
keys.sort(function() {return Math.random()-0.5;});
|
||||
for (let k in keys){
|
||||
let road = roads[keys[k]];
|
||||
if (road.cars.length == 0){
|
||||
road.setCarPosition(this.cars[c].core);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user