mirror of
https://github.com/wassname/metacar.git
synced 2026-09-11 12:20:12 +08:00
WIP: Implement DDPG
This commit is contained in:
Thibault Neveu
@@ -37,11 +37,17 @@
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.11.6"> </script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.7.1/pixi.min.js"></script>
|
||||
<script src="/dist/metacar.min.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/utils.js"></script>
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
<script type="text/javascript" src="/public/js/level2.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/DDPG/models.js"></script>
|
||||
<script type="text/javascript" src="/public/js/DDPG/memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/DDPG/noise.js"></script>
|
||||
<script type="text/javascript" src="/public/js/DDPG/ddpg.js"></script>
|
||||
<script type="text/javascript" src="/public/js/DDPG/index.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// This class is called from js/DDPG/index.js
|
||||
class DDPG {
|
||||
|
||||
constructor(){
|
||||
// Default Config
|
||||
this.config = {
|
||||
"stateSize": 25,
|
||||
"nbActions": 2,
|
||||
"layerNom": true,
|
||||
"normalizeObservations": true,
|
||||
"seed": 0,
|
||||
"criticL2Reg": 0.01,
|
||||
"batchSize": 64,
|
||||
"actorLr": 0.0001,
|
||||
"criticLr": 0.001,
|
||||
"gamma": 0.99,
|
||||
"rewardScale": 1,
|
||||
"nbEpochs": 500,
|
||||
"nbEpochsCycle": 800,
|
||||
"nbTrainSteps": 50,
|
||||
"nbRolloutStep": 100
|
||||
};
|
||||
// Inputs
|
||||
const obsInput = tf.input({shape: [this.config.stateSize]});
|
||||
const actionInput = tf.input({shape: [this.config.nbActions]});
|
||||
|
||||
// From js/DDPG/noise.js
|
||||
this.paramNoise = new AdaptiveParamNoiseSpec();
|
||||
// Buffer replay
|
||||
// The baseline use 1e6 but this size should be enough
|
||||
this.memory = new Memory(1000);
|
||||
// Actor and Critic are from js/DDPG/models.js
|
||||
this.actor = new Actor(
|
||||
this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed);
|
||||
this.critic = new Critic(
|
||||
this.config.stateSize, this.config.nbActions, this.config.layerNom, this.config.seed);
|
||||
this.actor.buildModel(obsInput);
|
||||
this.critic.buildModel(obsInput, actionInput);
|
||||
}
|
||||
};
|
||||
@@ -1,5 +1,8 @@
|
||||
let levelUrl = metacar.level.level2;
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var ddpg = new DDPG();
|
||||
|
||||
var env = new metacar.env("canvas", levelUrl);
|
||||
|
||||
env.setAgentMotion(metacar.motion.ControlMotion, {});
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
class Memory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.length = 0;
|
||||
this.start = 0;
|
||||
|
||||
this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param idx (number)
|
||||
*/
|
||||
getItem(idx){
|
||||
if (idx < 0 || idx >= this.length){
|
||||
console.error("Memory.getItem: idx not in range.");
|
||||
}
|
||||
return this.data[(this.start + idx) % this.maxlen]
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const arrLength = this.obs0List.length;
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals1': [],
|
||||
};
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * arrLength);
|
||||
batch.obs0.push(this.obs0List[id]);
|
||||
batch.obs1.push(this.obs1List[id]);
|
||||
batch.rewards.push(this.rewardsList[id]);
|
||||
batch.actions.push(this.actionsList[id]);
|
||||
batch.terminals1.push(this.terminals1List[id]);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal1){
|
||||
if (this.length < this.maxlen){
|
||||
this.length += 1;
|
||||
}
|
||||
else if (this.length == this.maxlen) {
|
||||
this.start = (this.start + 1) % this.maxlen;
|
||||
}
|
||||
else {
|
||||
console.error("Memory.append: This should never be printed");
|
||||
}
|
||||
this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
|
||||
this.obs1List[(this.start + this.length - 1) % this.maxlen] = action;
|
||||
this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
|
||||
this.actionsList[(this.start + this.length - 1) % this.maxlen] = obs1;
|
||||
this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
|
||||
class Actor{
|
||||
|
||||
/**
|
||||
* @param stateSize(number)
|
||||
* @param nbActions (number)
|
||||
* @param layerNorm (boolean)
|
||||
* @param seed (number)
|
||||
*/
|
||||
constructor(stateSize, nbActions, layerNorm, seed) {
|
||||
this.stateSize = stateSize;
|
||||
this.nbActions = nbActions;
|
||||
this.layerNorm = layerNorm;
|
||||
this.seed = seed;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param obs tf.input
|
||||
*/
|
||||
buildModel(obs){
|
||||
this.firstLayerBatchNorm = null;
|
||||
this.secondLayerBatchNorm = null;
|
||||
|
||||
this.relu = tf.layers.thresholdedReLU();
|
||||
|
||||
// First layer with BatchNormalization
|
||||
this.firstLayer = tf.layers.dense({
|
||||
inputShape: this.stateSize,
|
||||
units: 64,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
if (this.layerNorm){
|
||||
// WARNING: BatchNormalization instead of layerNormalization
|
||||
this.firstLayerBatchNorm = tf.layers.batchNormalization({
|
||||
scale: true,
|
||||
center: true
|
||||
});
|
||||
}
|
||||
|
||||
// Second layer with BatchNormalization
|
||||
this.secondLayer = tf.layers.dense({
|
||||
inputShape: 64,
|
||||
units: 64,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
if (this.layerNorm){
|
||||
// WARNING: BatchNormalization instead of layerNormalization
|
||||
this.secondLayerBatchNorm = tf.layers.batchNormalization({
|
||||
scale: true,
|
||||
center: true
|
||||
});
|
||||
}
|
||||
|
||||
// Ouput layer
|
||||
this.outputLayer = tf.layers.dense({
|
||||
inputShape: 64,
|
||||
units: this.nbActions,
|
||||
kernelInitializer: tf.initializers.randomUniform({
|
||||
minval: 0.003, maxval: 0.003, seed: this.seed}),
|
||||
activation: 'tanh',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Actor prediction
|
||||
const predict = () => {
|
||||
return tf.tidy(() => {
|
||||
let l1 = this.firstLayer.apply(obs);
|
||||
if (this.firstLayerBatchNorm){
|
||||
l1 = this.firstLayerBatchNorm.apply(l1);
|
||||
}
|
||||
//l1 = this.relu.apply(l1);
|
||||
let l2 = this.secondLayer.apply(l1);
|
||||
if (this.secondLayerBatchNorm){
|
||||
l2 = this.secondLayerBatchNorm.apply(l2);
|
||||
}
|
||||
//l2 = this.relu.apply(l2);
|
||||
return this.outputLayer.apply(l2);
|
||||
});
|
||||
}
|
||||
const output = predict();
|
||||
this.model = tf.model({inputs: this.obs, outputs: output});
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
class Critic {
|
||||
|
||||
/**
|
||||
* @param stateSize(number)
|
||||
* @param nbActions (number)
|
||||
* @param layerNorm (boolean)
|
||||
* @param seed (number)
|
||||
*/
|
||||
constructor(stateSize, nbActions, layerNorm, seed) {
|
||||
this.stateSize = stateSize;
|
||||
this.nbActions = nbActions;
|
||||
this.layerNorm = layerNorm;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param obs tf.input
|
||||
* @param action tf.input
|
||||
*/
|
||||
buildModel(obs, action){
|
||||
this.firstLayerBatchNorm = null;
|
||||
this.secondLayerBatchNorm = null;
|
||||
|
||||
this.relu = tf.layers.thresholdedReLU();
|
||||
|
||||
// First layer with BatchNormalization
|
||||
this.firstLayer = tf.layers.dense({
|
||||
inputShape: this.stateSize,
|
||||
units: 64,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
if (this.layerNorm){
|
||||
// WARNING: BatchNormalization instead of layerNormalization
|
||||
this.firstLayerBatchNorm = tf.layers.batchNormalization({
|
||||
scale: true,
|
||||
center: true
|
||||
});
|
||||
}
|
||||
|
||||
// Second layer with BatchNormalization
|
||||
this.secondLayer = tf.layers.dense({
|
||||
inputShape: 64 + this.nbActions, // Previous layer + action
|
||||
units: 64,
|
||||
kernelInitializer: tf.initializers.glorotUniform({seed: this.seed}),
|
||||
activation: 'linear', // relu is add later
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
if (this.layerNorm){
|
||||
// WARNING: BatchNormalization instead of layerNormalization
|
||||
this.secondLayerBatchNorm = tf.layers.batchNormalization({
|
||||
scale: true,
|
||||
center: true
|
||||
});
|
||||
}
|
||||
|
||||
// Ouput layer
|
||||
this.outputLayer = tf.layers.dense({
|
||||
inputShape: 64,
|
||||
units: 1,
|
||||
kernelInitializer: tf.initializers.randomUniform({
|
||||
minval: 0.003, maxval: 0.003, seed: this.seed}),
|
||||
activation: 'tanh',
|
||||
useBias: true,
|
||||
biasInitializer: "zeros"
|
||||
});
|
||||
|
||||
// Actor prediction
|
||||
const predict = () => {
|
||||
return tf.tidy(() => {
|
||||
let l1 = this.firstLayer.apply(obs);
|
||||
l1 = l1.concat(action);
|
||||
if (this.firstLayerBatchNorm){
|
||||
l1 = this.firstLayerBatchNorm.apply(l1);
|
||||
}
|
||||
//l1 = this.relu(l1);
|
||||
let l2 = this.secondLayer.apply(l1);
|
||||
if (this.secondLayerBatchNorm){
|
||||
l2 = this.secondLayerBatchNorm.apply(l2);
|
||||
}
|
||||
//l2 = this.relu.apply(l2);
|
||||
return this.outputLayer.apply(l2);
|
||||
});
|
||||
}
|
||||
const output = predict();
|
||||
this.model = tf.model({inputs: this.obs, outputs: output});
|
||||
}
|
||||
|
||||
};
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Noise class
|
||||
* The original baseline is made of three noise
|
||||
* AdaptiveParamNoiseSpec, ActionNoise and NormalActionNoise
|
||||
* Only AdaptiveParamNoiseSpec is implemented for now
|
||||
* See "C Adapative Scaling" Page 14 in the paper.
|
||||
*/
|
||||
|
||||
class AdaptiveParamNoiseSpec{
|
||||
|
||||
/**
|
||||
* @param conf Object
|
||||
* conf.initialStddev: 0.1 default // σ
|
||||
* conf.desiredActionStddev: 0.1 default // δ
|
||||
* conf.adoptionCoefficient: 1.01 default // α
|
||||
*/
|
||||
constructor(conf){
|
||||
conf = conf || {};
|
||||
this.initialStddev = conf.initialStddev || 0.1;
|
||||
this.desiredActionStddev = conf.initialStddev || 0.1;
|
||||
this.adoptionCoefficient = conf.adoptionCoefficient || 1.01;
|
||||
this.currentStddev = conf.initialStddev;
|
||||
}
|
||||
|
||||
/**
|
||||
* The distance from the Adaptive scaling
|
||||
* @param distance number
|
||||
*/
|
||||
adapt(distance){
|
||||
// if d(π, _π_) > δ then σ = σ/α
|
||||
if (distance > this.desiredActionStddev){
|
||||
// Decrease σ
|
||||
this.currentStddev /= this.adoptionCoefficient;
|
||||
}
|
||||
else{
|
||||
// σ = σ*α
|
||||
// Increase σ
|
||||
this.currentStddev *= this.adoptionCoefficient;
|
||||
}
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user