mirror of
https://github.com/wassname/rl_2d_walker.js.git
synced 2026-09-24 13:50:30 +08:00
90 lines
2.1 KiB
JavaScript
90 lines
2.1 KiB
JavaScript
|
|
function Agent(opt, world) {
|
|
this.walker = new Walker(world.world)
|
|
this.options = opt
|
|
|
|
this.world = world
|
|
this.frequency = 20
|
|
this.reward = 0
|
|
this.loaded = false
|
|
|
|
this.loss = 0
|
|
this.timer = 0
|
|
this.timerFrequency = 60 / this.frequency
|
|
this.resetFrequency = 30 / this.frequency
|
|
|
|
if (this.options.dynamicallyLoaded !== true) {
|
|
this.init(world.brains.actor.newConfiguration(), null)
|
|
}
|
|
|
|
};
|
|
|
|
Agent.prototype.init = function (actor, critic) {
|
|
var actions = this.walker.joints.length
|
|
var temporal = 1
|
|
var states = this.walker.bodies.length * 7
|
|
|
|
var input = window.neurojs.Agent.getInputDimension(states, actions, temporal)
|
|
|
|
this.brain = new window.neurojs.Agent({
|
|
|
|
actor: actor,
|
|
critic: critic,
|
|
|
|
states: states,
|
|
actions: actions,
|
|
|
|
algorithm: 'ddpg',
|
|
|
|
temporalWindow: temporal,
|
|
|
|
discount: 0.97,
|
|
rate: 0.004,
|
|
theta: 0.05, // progressive copy
|
|
alpha: 0.1, // advantage learning
|
|
|
|
experience: 75e3,
|
|
// buffer: window.neurojs.Buffers.UniformReplayBuffer,
|
|
|
|
learningPerTick: 40,
|
|
startLearningAt: 900,
|
|
|
|
|
|
|
|
})
|
|
|
|
// this.world.brains.shared.add('actor', this.brain.algorithm.actor)
|
|
this.world.brains.shared.add('critic', this.brain.algorithm.critic)
|
|
|
|
this.actions = actions
|
|
this.loaded = true
|
|
};
|
|
|
|
Agent.prototype.step = function () {
|
|
if (!this.loaded) {
|
|
return
|
|
}
|
|
|
|
this.timer++
|
|
|
|
if (this.timer % this.timerFrequency === 0) {
|
|
|
|
// reward from last step
|
|
this.reward = this.walker.reward
|
|
// this.reward = this.walker.score * 10
|
|
// // punish for using energy
|
|
// this.reward -= this.walker.joints.map(j=>j.GetJointSpeed()).reduce((sum,speed)=>sum+speed**2)
|
|
// // console.log(this.reward)
|
|
// this.walker.score = 0
|
|
|
|
// train
|
|
this.loss = this.brain.learn(this.reward)
|
|
this.action = this.brain.policy(this.walker.getState())
|
|
}
|
|
if (this.action) {
|
|
this.walker.simulationStep(this.action)
|
|
}
|
|
|
|
return this.timer % this.timerFrequency === 0
|
|
};
|