mirror of
https://github.com/wassname/ray.git
synced 2026-07-22 13:00:49 +08:00
Eric LiangandGitHub
9ea57c2a93
Rename AsyncSamplesOptimizer -> AsyncReplayOptimizer Add AsyncSamplesOptimizer that implements the IMPALA architecture integrate V-trace with a3c policy graph audit V-trace integration benchmark compare vs A3C and with V-trace on/off PongNoFrameskip-v4 on IMPALA scaling from 16 to 128 workers, solving Pong in <10 min. For reference, solving this env takes ~40 minutes for Ape-X and several hours for A3C.
62 lines
2.1 KiB
Python
62 lines
2.1 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
from ray.rllib.agents.ddpg.ddpg import DDPGAgent, DEFAULT_CONFIG as DDPG_CONFIG
|
|
from ray.rllib.utils import merge_dicts
|
|
from ray.tune.trial import Resources
|
|
|
|
APEX_DDPG_DEFAULT_CONFIG = merge_dicts(
|
|
DDPG_CONFIG,
|
|
{
|
|
"optimizer_class": "AsyncReplayOptimizer",
|
|
"optimizer": merge_dicts(
|
|
DDPG_CONFIG["optimizer"], {
|
|
"max_weight_sync_delay": 400,
|
|
"num_replay_buffer_shards": 4,
|
|
"debug": False
|
|
}),
|
|
"n_step": 3,
|
|
"gpu": False,
|
|
"num_workers": 32,
|
|
"buffer_size": 2000000,
|
|
"learning_starts": 50000,
|
|
"train_batch_size": 512,
|
|
"sample_batch_size": 50,
|
|
"max_weight_sync_delay": 400,
|
|
"target_network_update_freq": 500000,
|
|
"timesteps_per_iteration": 25000,
|
|
"per_worker_exploration": True,
|
|
"worker_side_prioritization": True,
|
|
"min_iter_time_s": 30,
|
|
},
|
|
)
|
|
|
|
|
|
class ApexDDPGAgent(DDPGAgent):
|
|
"""DDPG variant that uses the Ape-X distributed policy optimizer.
|
|
|
|
By default, this is configured for a large single node (32 cores). For
|
|
running in a large cluster, increase the `num_workers` config var.
|
|
"""
|
|
|
|
_agent_name = "APEX_DDPG"
|
|
_default_config = APEX_DDPG_DEFAULT_CONFIG
|
|
|
|
@classmethod
|
|
def default_resource_request(cls, config):
|
|
cf = merge_dicts(cls._default_config, config)
|
|
return Resources(
|
|
cpu=1 + cf["optimizer"]["num_replay_buffer_shards"],
|
|
gpu=cf["gpu"] and 1 or 0,
|
|
extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"],
|
|
extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
|
|
|
|
def update_target_if_needed(self):
|
|
# Ape-X updates based on num steps trained, not sampled
|
|
if self.optimizer.num_steps_trained - self.last_target_update_ts > \
|
|
self.config["target_network_update_freq"]:
|
|
self.local_evaluator.for_policy(lambda p: p.update_target())
|
|
self.last_target_update_ts = self.optimizer.num_steps_trained
|
|
self.num_target_updates += 1
|