mirror of
https://github.com/wassname/ray.git
synced 2026-07-07 05:35:39 +08:00
Eric Liang
ecb811c26e
* minimal apex checkin * cleanup dqn options * actor utils * Sun Feb 25 17:39:54 PST 2018 * update * compression refactor * fix * add test * fix models * Sun Feb 25 21:46:27 PST 2018 * Wed Feb 28 10:26:34 PST 2018 * Wed Feb 28 10:28:09 PST 2018 * Wed Feb 28 10:42:59 PST 2018 * refactor * Wed Feb 28 11:17:19 PST 2018 * Wed Feb 28 11:42:08 PST 2018 * Wed Feb 28 11:42:13 PST 2018 * Wed Feb 28 11:59:02 PST 2018 * Wed Feb 28 11:59:58 PST 2018 * Wed Feb 28 12:00:08 PST 2018 * Wed Feb 28 12:02:19 PST 2018 * Wed Feb 28 13:44:31 PST 2018 * Wed Feb 28 17:01:20 PST 2018 * Sat Mar 3 14:55:59 PST 2018 * make optimizer construction explicit * Sat Mar 3 18:23:08 PST 2018 * Sat Mar 3 18:24:28 PST 2018 * Sat Mar 3 18:49:28 PST 2018 * Sat Mar 3 18:50:42 PST 2018 * Sat Mar 3 18:56:10 PST 2018
62 lines
2.3 KiB
Python
62 lines
2.3 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import ray
|
|
from ray.rllib.optimizers.optimizer import Optimizer
|
|
from ray.rllib.utils.timer import TimerStat
|
|
|
|
|
|
class AsyncOptimizer(Optimizer):
|
|
"""An asynchronous RL optimizer, e.g. for implementing A3C.
|
|
|
|
This optimizer asynchronously pulls and applies gradients from remote
|
|
evaluators, sending updated weights back as needed. This pipelines the
|
|
gradient computations on the remote workers.
|
|
"""
|
|
def _init(self, grads_per_step=100, batch_size=10):
|
|
self.apply_timer = TimerStat()
|
|
self.wait_timer = TimerStat()
|
|
self.dispatch_timer = TimerStat()
|
|
self.grads_per_step = grads_per_step
|
|
self.batch_size = batch_size
|
|
|
|
def step(self):
|
|
weights = ray.put(self.local_evaluator.get_weights())
|
|
gradient_queue = []
|
|
num_gradients = 0
|
|
|
|
# Kick off the first wave of async tasks
|
|
for e in self.remote_evaluators:
|
|
e.set_weights.remote(weights)
|
|
fut = e.compute_gradients.remote(e.sample.remote())
|
|
gradient_queue.append((fut, e))
|
|
num_gradients += 1
|
|
|
|
# Note: can't use wait: https://github.com/ray-project/ray/issues/1128
|
|
while gradient_queue:
|
|
with self.wait_timer:
|
|
fut, e = gradient_queue.pop(0)
|
|
gradient = ray.get(fut)
|
|
|
|
if gradient is not None:
|
|
with self.apply_timer:
|
|
self.local_evaluator.apply_gradients(gradient)
|
|
|
|
if num_gradients < self.grads_per_step:
|
|
with self.dispatch_timer:
|
|
e.set_weights.remote(self.local_evaluator.get_weights())
|
|
fut = e.compute_gradients.remote(e.sample.remote())
|
|
gradient_queue.append((fut, e))
|
|
num_gradients += 1
|
|
|
|
self.num_steps_sampled += self.grads_per_step * self.batch_size
|
|
self.num_steps_trained += self.grads_per_step * self.batch_size
|
|
|
|
def stats(self):
|
|
return dict(Optimizer.stats(), **{
|
|
"wait_time_ms": round(1000 * self.wait_timer.mean, 3),
|
|
"apply_time_ms": round(1000 * self.apply_timer.mean, 3),
|
|
"dispatch_time_ms": round(1000 * self.dispatch_timer.mean, 3),
|
|
})
|