mirror of
https://github.com/wassname/metacar.git
synced 2026-09-09 11:26:47 +08:00
DDPG finished
This commit is contained in:
Thibault Neveu
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -32,7 +32,6 @@
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
<li><b>Prioritized Experience Replay</b>: <a href="https://arxiv.org/abs/1511.05952">paper</a> </li>
|
||||
</ul>
|
||||
<br>
|
||||
You can use the <b>arrow keys</b> to control the car by yourself.<br><br>
|
||||
@@ -58,6 +57,7 @@
|
||||
|
||||
<script type="text/javascript" src="/public/js/ddpg/models.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/prioritized_memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/noise.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg_agent.js"></script>
|
||||
|
||||
@@ -32,7 +32,6 @@
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
<li><b>Prioritized Experience Replay</b>: <a href="https://arxiv.org/abs/1511.05952">paper</a> </li>
|
||||
</ul>
|
||||
<br>
|
||||
You can use the <b>arrow keys</b> to control the car by yourself.<br><br>
|
||||
@@ -57,6 +56,7 @@
|
||||
<script type="text/javascript" src="/public/js/viewer.js"></script>
|
||||
|
||||
<script type="text/javascript" src="/public/js/ddpg/models.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/prioritized_memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/memory.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/noise.js"></script>
|
||||
<script type="text/javascript" src="/public/js/ddpg/ddpg.js"></script>
|
||||
|
||||
@@ -71,7 +71,6 @@
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
<li><b>Prioritized Experience Replay</b>: <a href="https://arxiv.org/abs/1511.05952">paper</a> </li>
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
@@ -90,7 +89,6 @@
|
||||
<ul style="margin:0;">
|
||||
<li><b>Deep Deterministic Policy Gradients (DDPG): <a href="https://arxiv.org/abs/1509.02971">paper</a></b> </li>
|
||||
<li><b>Parameter Space Noise for Exploration</b>: <a href="https://blog.openai.com/better-exploration-with-parameter-noise/">paper</a> </li>
|
||||
<li><b>Prioritized Experience Replay</b>: <a href="https://arxiv.org/abs/1511.05952">paper</a> </li>
|
||||
</ul>
|
||||
<p>
|
||||
The control is based on two continuous values for the throttle and steering angle of the car.
|
||||
|
||||
@@ -63,8 +63,7 @@ env.load().then(() => {
|
||||
});
|
||||
|
||||
env.addEvent("load", () => {
|
||||
//agent.restore("ddpg-traffic", "model-ddpg-traffic")
|
||||
agent.restore("four", "model-ddpg-traffic-epoch-120");
|
||||
agent.restore("ddpg-traffic", "model-ddpg-traffic-epoch-120");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -179,6 +179,7 @@ class DDPG {
|
||||
return erros.mean();
|
||||
}, true, this.criticWeights);
|
||||
|
||||
// For experience Replay
|
||||
this.memory.appendBackWithCost(batch, costs);
|
||||
|
||||
const loss = criticLoss.buffer().values[0];
|
||||
|
||||
@@ -25,7 +25,7 @@ class DDPGAgent {
|
||||
"nbEpochs": config.nbEpochs || 200,
|
||||
"nbEpochsCycle": config.nbEpochsCycle || 10,
|
||||
"nbTrainSteps": config.nbTrainSteps || 110,
|
||||
"tau": config.tau || 0.01,
|
||||
"tau": config.tau || 0.008,
|
||||
"initialStddev": config.initialStddev || 0.1,
|
||||
"desiredActionStddev": config.desiredActionStddev || 0.1,
|
||||
"adoptionCoefficient": config.adoptionCoefficient || 1.01,
|
||||
@@ -48,7 +48,7 @@ class DDPGAgent {
|
||||
|
||||
// Buffer replay
|
||||
// The baseline use 1e6 but this size should be enough for this problem
|
||||
this.memory = new Memory(this.config.memorySize);
|
||||
this.memory = new PrioritizedMemory(this.config.memorySize);
|
||||
// Actor and Critic are from js/DDPG/models.js
|
||||
this.actor = new Actor(this.config);
|
||||
this.critic = new Critic(this.config);
|
||||
@@ -75,8 +75,8 @@ class DDPGAgent {
|
||||
/*
|
||||
Restore the weights of the network
|
||||
*/
|
||||
const critic = await tf.loadModel('http://localhost:3000/public/models/'+folder+'/critic-'+name+'.json');
|
||||
const actor = await tf.loadModel("http://localhost:3000/public/models/"+folder+"/actor-"+name+".json");
|
||||
const critic = await tf.loadModel('https://metacar-project.com/public/models/'+folder+'/critic-'+name+'.json');
|
||||
const actor = await tf.loadModel("https://metacar-project.com/public/models/"+folder+"/actor-"+name+".json");
|
||||
|
||||
this.ddpg.critic = copyFromSave(critic, Critic, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
|
||||
this.ddpg.actor = copyFromSave(actor, Actor, this.config, this.ddpg.obsInput, this.ddpg.actionInput);
|
||||
|
||||
@@ -7,10 +7,7 @@ env.setAgentLidar({pts: 5, width: 3, height: 7, pos: -0.5})
|
||||
|
||||
// js/DDPG/ddpg.js
|
||||
var agent = new DDPGAgent(env, {
|
||||
stateSize: 26,
|
||||
resetEpisode: true,
|
||||
saveDuringTraining: true,
|
||||
saveInterval: 10,
|
||||
stateSize: 26
|
||||
});
|
||||
|
||||
initMetricsContainer("statContainer", ["Reward", "ActorLoss", "CriticLoss", "EpisodeDuration", "NoiseDistance"]);
|
||||
|
||||
@@ -6,8 +6,24 @@ class Memory {
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.buffer = [];
|
||||
this.priorBuffer = [];
|
||||
this.length = 0;
|
||||
this.start = 0;
|
||||
|
||||
this.obs0List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.obs1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.rewardsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.actionsList = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
this.terminals1List = Array.apply(null, Array(maxlen)).map(Number.prototype.valueOf, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param idx (number)
|
||||
*/
|
||||
getItem(idx){
|
||||
if (idx < 0 || idx >= this.length){
|
||||
console.error("Memory.getItem: idx not in range.");
|
||||
}
|
||||
return this.data[(this.start + idx) % this.maxlen]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -16,92 +32,7 @@ class Memory {
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
if (batchSize > this.priorBuffer.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
_bufferBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let nElem = this.buffer.pop();
|
||||
batch.obs0.push(nElem.obs0);
|
||||
batch.obs1.push(nElem.obs1);
|
||||
batch.rewards.push(nElem.reward);
|
||||
batch.actions.push(nElem.action);
|
||||
batch.terminals.push(nElem.terminal);
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
|
||||
return batch
|
||||
}
|
||||
|
||||
_addRandomBufferBatch(batchSize, batch){
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
popBatch(batchSize){
|
||||
let originalBatchSize = batchSize;
|
||||
let priorBufferBatchSize;
|
||||
let bufferBatchSize;
|
||||
if (batchSize % 2 != 0){
|
||||
console.warn("Batch size should be a even.")
|
||||
}
|
||||
if (this.priorBuffer.length < batchSize/2){
|
||||
//console.log("get full batch from buffer");
|
||||
const batch = this._bufferBatch(batchSize);
|
||||
console.assert(batch.obs0.length == batchSize);
|
||||
return batch;
|
||||
}
|
||||
const arrLength = this.length;
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
@@ -110,80 +41,19 @@ class Memory {
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
//console.log("Get half of prior and other from buffer.");
|
||||
batchSize = batchSize / 2;
|
||||
}
|
||||
else{
|
||||
//console.log("Get all from priorBuffer");
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
this.priorBuffer.splice(id, 1);
|
||||
let id = Math.floor(Math.random() * arrLength);
|
||||
batch.obs0.push(this.obs0List[id]);
|
||||
batch.obs1.push(this.obs1List[id]);
|
||||
batch.rewards.push(this.rewardsList[id]);
|
||||
batch.actions.push(this.actionsList[id]);
|
||||
batch.terminals.push(this.terminals1List[id]);
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
this._addRandomBufferBatch(batchSize, batch);
|
||||
}
|
||||
console.assert(batch.obs0.length == originalBatchSize);
|
||||
return batch
|
||||
}
|
||||
|
||||
_insert(element, array) {
|
||||
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){
|
||||
array.unshift(element);
|
||||
return array;
|
||||
}
|
||||
array.splice(this._locationOf(element, array) + 1, 0, element);
|
||||
return array;
|
||||
}
|
||||
|
||||
_locationOf(element, array, start, end) {
|
||||
start = start || 0;
|
||||
end = end || array.length;
|
||||
|
||||
var pivot = parseInt(start + (end - start) / 2, 10);
|
||||
|
||||
if (end-start <= 1 || array[pivot] === element) return pivot;
|
||||
|
||||
if (array[pivot].cost != null && array[pivot].cost < element.cost) {
|
||||
return this._locationOf(element, array, pivot, end);
|
||||
} else {
|
||||
return this._locationOf(element, array, start, pivot);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param batch (Object) from getBatch()
|
||||
* @param cost (number) Cost associated with each row of the batch
|
||||
*/
|
||||
appendBackWithCost(batch, costs){
|
||||
for (let b=0; b < batch.obs0.length; b++){
|
||||
if (this.buffer.length == this.maxlen){
|
||||
this.buffer.shift();
|
||||
}
|
||||
this._insert({
|
||||
obs0: batch.obs0[b],
|
||||
action: batch.actions[b],
|
||||
reward: batch.rewards[b],
|
||||
obs1: batch.obs1[b],
|
||||
terminal: batch.terminals[b],
|
||||
cost: costs[b]
|
||||
}, this.buffer);
|
||||
}
|
||||
console.assert(this.buffer.length <= this.maxlen);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
@@ -191,50 +61,23 @@ class Memory {
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal){
|
||||
if (this.priorBuffer.length == this.maxlen){
|
||||
this.priorBuffer.shift();
|
||||
append(obs0, action, reward, obs1, terminal1){
|
||||
if (this.length < this.maxlen){
|
||||
this.length += 1;
|
||||
}
|
||||
this.priorBuffer.push({
|
||||
obs0: obs0,
|
||||
action: action,
|
||||
reward: reward,
|
||||
obs1: obs1,
|
||||
terminal: terminal,
|
||||
cost: null
|
||||
});
|
||||
console.assert(this.priorBuffer.length <= this.maxlen);
|
||||
else if (this.length == this.maxlen) {
|
||||
//this.obs0List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.obs1List[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
//this.actionsList[(this.start + this.length - 1) % this.maxlen].dispose();
|
||||
this.start = (this.start + 1) % this.maxlen;
|
||||
}
|
||||
else {
|
||||
console.error("Memory.append: This should never be printed");
|
||||
}
|
||||
this.obs0List[(this.start + this.length - 1) % this.maxlen] = obs0;
|
||||
this.obs1List[(this.start + this.length - 1) % this.maxlen] = obs1;
|
||||
this.rewardsList[(this.start + this.length - 1) % this.maxlen] = reward;
|
||||
this.actionsList[(this.start + this.length - 1) % this.maxlen] = action;
|
||||
this.terminals1List[(this.start + this.length - 1) % this.maxlen] = terminal1;
|
||||
}
|
||||
}
|
||||
/*
|
||||
var mem = new Memory(20000);
|
||||
Math.seedrandom(0);
|
||||
console.assert(mem.length == 0);
|
||||
|
||||
var array = [];
|
||||
for (let i=1; i < 40000; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
|
||||
console.assert(mem.length == 20000);
|
||||
console.assert(mem.list[0].obs0 == "obs0-20000");
|
||||
console.assert(mem.list[19999].obs0 == "obs0-39999");
|
||||
|
||||
let batch = mem.getBatch(32);
|
||||
|
||||
console.assert(batch.obs0.length == 32);
|
||||
console.assert(mem.length == 20000 - 32);
|
||||
|
||||
let costs = [];
|
||||
for (i=31; i >= 0; i--){
|
||||
costs.push(i);
|
||||
}
|
||||
mem.appendBackWithCost(batch, costs);
|
||||
|
||||
console.log(mem.list);
|
||||
|
||||
/*
|
||||
for (let i=1; i < 64; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
*/
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
|
||||
class PrioritizedMemory {
|
||||
|
||||
/**
|
||||
* @param maxlen (number) Buffer limit
|
||||
*/
|
||||
constructor(maxlen){
|
||||
this.maxlen = maxlen;
|
||||
this.buffer = [];
|
||||
this.priorBuffer = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
getBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
if (batchSize > this.priorBuffer.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
_bufferBatch(batchSize){
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let nElem = this.buffer.pop();
|
||||
batch.obs0.push(nElem.obs0);
|
||||
batch.obs1.push(nElem.obs1);
|
||||
batch.rewards.push(nElem.reward);
|
||||
batch.actions.push(nElem.action);
|
||||
batch.terminals.push(nElem.terminal);
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize/2; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
|
||||
return batch
|
||||
}
|
||||
|
||||
_addRandomBufferBatch(batchSize, batch){
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.buffer.length);
|
||||
batch.obs0.push(this.buffer[id].obs0);
|
||||
batch.obs1.push(this.buffer[id].obs1);
|
||||
batch.rewards.push(this.buffer[id].reward);
|
||||
batch.actions.push(this.buffer[id].action);
|
||||
batch.terminals.push(this.buffer[id].terminal);
|
||||
this.buffer.splice(id, 1);
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a batch
|
||||
* @param batchSize (number)
|
||||
* @return batch []
|
||||
*/
|
||||
popBatch(batchSize){
|
||||
let originalBatchSize = batchSize;
|
||||
let priorBufferBatchSize;
|
||||
let bufferBatchSize;
|
||||
if (batchSize % 2 != 0){
|
||||
console.warn("Batch size should be a even.")
|
||||
}
|
||||
if (this.priorBuffer.length < batchSize/2){
|
||||
//console.log("get full batch from buffer");
|
||||
const batch = this._bufferBatch(batchSize);
|
||||
console.assert(batch.obs0.length == batchSize);
|
||||
return batch;
|
||||
}
|
||||
const batch = {
|
||||
'obs0': [],
|
||||
'obs1': [],
|
||||
'rewards': [],
|
||||
'actions': [],
|
||||
'terminals': [],
|
||||
};
|
||||
if (batchSize > this.length){
|
||||
console.warn("The size of the replay buffer is < to the batchSize. Return empty batch.");
|
||||
return batch;
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
//console.log("Get half of prior and other from buffer.");
|
||||
batchSize = batchSize / 2;
|
||||
}
|
||||
else{
|
||||
//console.log("Get all from priorBuffer");
|
||||
}
|
||||
|
||||
for (let b=0; b < batchSize; b++){
|
||||
let id = Math.floor(Math.random() * this.priorBuffer.length);
|
||||
batch.obs0.push(this.priorBuffer[id].obs0);
|
||||
batch.obs1.push(this.priorBuffer[id].obs1);
|
||||
batch.rewards.push(this.priorBuffer[id].reward);
|
||||
batch.actions.push(this.priorBuffer[id].action);
|
||||
batch.terminals.push(this.priorBuffer[id].terminal);
|
||||
this.priorBuffer.splice(id, 1);
|
||||
}
|
||||
|
||||
if (this.buffer.length > 0){
|
||||
this._addRandomBufferBatch(batchSize, batch);
|
||||
}
|
||||
console.assert(batch.obs0.length == originalBatchSize);
|
||||
return batch
|
||||
}
|
||||
|
||||
_insert(element, array) {
|
||||
if (array.length == 0 || element.cost < array[0].cost || array[0].cost == null){
|
||||
array.unshift(element);
|
||||
return array;
|
||||
}
|
||||
array.splice(this._locationOf(element, array) + 1, 0, element);
|
||||
return array;
|
||||
}
|
||||
|
||||
_locationOf(element, array, start, end) {
|
||||
start = start || 0;
|
||||
end = end || array.length;
|
||||
|
||||
var pivot = parseInt(start + (end - start) / 2, 10);
|
||||
|
||||
if (end-start <= 1 || array[pivot] === element) return pivot;
|
||||
|
||||
if (array[pivot].cost != null && array[pivot].cost < element.cost) {
|
||||
return this._locationOf(element, array, pivot, end);
|
||||
} else {
|
||||
return this._locationOf(element, array, start, pivot);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param batch (Object) from getBatch()
|
||||
* @param cost (number) Cost associated with each row of the batch
|
||||
*/
|
||||
appendBackWithCost(batch, costs){
|
||||
for (let b=0; b < batch.obs0.length; b++){
|
||||
if (this.buffer.length == this.maxlen){
|
||||
this.buffer.shift();
|
||||
}
|
||||
this._insert({
|
||||
obs0: batch.obs0[b],
|
||||
action: batch.actions[b],
|
||||
reward: batch.rewards[b],
|
||||
obs1: batch.obs1[b],
|
||||
terminal: batch.terminals[b],
|
||||
cost: costs[b]
|
||||
}, this.buffer);
|
||||
}
|
||||
console.assert(this.buffer.length <= this.maxlen);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param obs0 []
|
||||
* @param action (number)
|
||||
* @param reward (number)
|
||||
* @param obs1 []
|
||||
* @param terminal1 (boolean)
|
||||
*/
|
||||
append(obs0, action, reward, obs1, terminal){
|
||||
if (this.priorBuffer.length == this.maxlen){
|
||||
this.priorBuffer.shift();
|
||||
}
|
||||
this.priorBuffer.push({
|
||||
obs0: obs0,
|
||||
action: action,
|
||||
reward: reward,
|
||||
obs1: obs1,
|
||||
terminal: terminal,
|
||||
cost: null
|
||||
});
|
||||
console.assert(this.priorBuffer.length <= this.maxlen);
|
||||
}
|
||||
}
|
||||
/*
|
||||
var mem = new Memory(20000);
|
||||
Math.seedrandom(0);
|
||||
console.assert(mem.length == 0);
|
||||
|
||||
var array = [];
|
||||
for (let i=1; i < 40000; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
|
||||
console.assert(mem.length == 20000);
|
||||
console.assert(mem.list[0].obs0 == "obs0-20000");
|
||||
console.assert(mem.list[19999].obs0 == "obs0-39999");
|
||||
|
||||
let batch = mem.getBatch(32);
|
||||
|
||||
console.assert(batch.obs0.length == 32);
|
||||
console.assert(mem.length == 20000 - 32);
|
||||
|
||||
let costs = [];
|
||||
for (i=31; i >= 0; i--){
|
||||
costs.push(i);
|
||||
}
|
||||
mem.appendBackWithCost(batch, costs);
|
||||
|
||||
console.log(mem.list);
|
||||
|
||||
/*
|
||||
for (let i=1; i < 64; i++){
|
||||
mem.append("obs0-"+i, "action-"+i, "reward-"+i, "obs1-"+i, "terminal-"+i);
|
||||
}
|
||||
*/
|
||||
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model7","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense22","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense22","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense23","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense23","trainable":true},"inbound_nodes":[[["dense_Dense22",0,0,{}]]]},{"name":"dense_Dense24","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense24","trainable":true},"inbound_nodes":[[["dense_Dense23",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense24",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-traffic-epoch-120.weights.bin"],"weights":[{"name":"dense_Dense22/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense22/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense23/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense23/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense24/kernel","shape":[64,2],"dtype":"float32"},{"name":"dense_Dense24/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model1","layers":[{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense1","class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"dense_Dense2","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true},"inbound_nodes":[[["dense_Dense1",0,0,{}]]]},{"name":"dense_Dense3","class_name":"Dense","config":{"units":2,"activation":"tanh","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true},"inbound_nodes":[[["dense_Dense2",0,0,{}]]]}],"input_layers":[["input1",0,0]],"output_layers":[["dense_Dense3",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./actor-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[64,2],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[2],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model6","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense19","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense19","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense18","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense18","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add3","class_name":"Add","config":{"name":"add_Add3","trainable":true},"inbound_nodes":[[["dense_Dense19",0,0,{}],["dense_Dense18",0,0,{}]]]},{"name":"dense_Dense20","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense20","trainable":true},"inbound_nodes":[[["add_Add3",0,0,{}]]]},{"name":"dense_Dense21","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense21","trainable":true},"inbound_nodes":[[["dense_Dense20",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense21",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic-epoch-120.weights.bin"],"weights":[{"name":"dense_Dense19/kernel","shape":[2,128],"dtype":"float32"},{"name":"dense_Dense19/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense18/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense18/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense20/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense20/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense21/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense21/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
BIN
Binary file not shown.
@@ -1 +0,0 @@
|
||||
{"modelTopology":{"class_name":"Model","config":{"name":"model2","layers":[{"name":"input2","class_name":"InputLayer","config":{"batch_input_shape":[null,2],"dtype":"float32","sparse":false,"name":"input2"},"inbound_nodes":[]},{"name":"input1","class_name":"InputLayer","config":{"batch_input_shape":[null,50],"dtype":"float32","sparse":false,"name":"input1"},"inbound_nodes":[]},{"name":"dense_Dense5","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense5","trainable":true},"inbound_nodes":[[["input2",0,0,{}]]]},{"name":"dense_Dense4","class_name":"Dense","config":{"units":128,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true},"inbound_nodes":[[["input1",0,0,{}]]]},{"name":"add_Add1","class_name":"Add","config":{"name":"add_Add1","trainable":true},"inbound_nodes":[[["dense_Dense5",0,0,{}],["dense_Dense4",0,0,{}]]]},{"name":"dense_Dense6","class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"uniform","seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense6","trainable":true},"inbound_nodes":[[["add_Add1",0,0,{}]]]},{"name":"dense_Dense7","class_name":"Dense","config":{"units":1,"activation":"linear","use_bias":true,"kernel_initializer":{"class_name":"RandomUniform","config":{"minval":0.003,"maxval":0.003,"seed":0}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense7","trainable":true},"inbound_nodes":[[["dense_Dense6",0,0,{}]]]}],"input_layers":[["input1",0,0],["input2",0,0]],"output_layers":[["dense_Dense7",0,0]]},"keras_version":"tfjs-layers 0.6.6","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./critic-model-ddpg-traffic.weights.bin"],"weights":[{"name":"dense_Dense5/kernel","shape":[2,128],"dtype":"float32"},{"name":"dense_Dense5/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[50,128],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense6/kernel","shape":[128,64],"dtype":"float32"},{"name":"dense_Dense6/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense7/kernel","shape":[64,1],"dtype":"float32"},{"name":"dense_Dense7/bias","shape":[1],"dtype":"float32"}]}]}
|
||||
Binary file not shown.
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user