[tune] [rllib] Automatically determine RLlib resources and add queueing mechanism for autoscaling (#1848)

python/ray/rllib/a3c/a3c.py

@@ -13,6 +13,7 @@ from ray.rllib.utils import FilterManager
 from ray.rllib.a3c.a3c_evaluator import A3CEvaluator, RemoteA3CEvaluator, \
     GPURemoteA3CEvaluator
 from ray.tune.result import TrainingResult
+from ray.tune.trial import Resources
 
 
 DEFAULT_CONFIG = {
@@ -68,6 +69,14 @@ class A3CAgent(Agent):
     _default_config = DEFAULT_CONFIG
     _allow_unknown_subkeys = ["model", "optimizer", "env_config"]
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(
+            cpu=1, gpu=0,
+            extra_cpu=cf["num_workers"],
+            extra_gpu=cf["use_gpu_for_workers"] and cf["num_workers"] or 0)
+
     def _init(self):
         self.local_evaluator = A3CEvaluator(
             self.registry, self.env_creator, self.config, self.logdir,

python/ray/rllib/agent.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 
 import logging
 import numpy as np
+import json
 import os
 import pickle
 
@@ -62,6 +63,14 @@ class Agent(Trainable):
     _allow_unknown_configs = False
     _allow_unknown_subkeys = []
 
+    @classmethod
+    def resource_help(cls, config):
+        return (
+            "\n\nYou can adjust the resource requests of RLlib agents by "
+            "setting `num_workers` and other configs. See the "
+            "DEFAULT_CONFIG defined by each agent for more info.\n\n"
+            "The config of this agent is: " + json.dumps(config))
+
     def __init__(
             self, config=None, env=None, registry=None,
             logger_creator=None):

python/ray/rllib/bc/bc.py

@@ -8,16 +8,19 @@ from ray.rllib.bc.bc_evaluator import BCEvaluator, GPURemoteBCEvaluator, \
     RemoteBCEvaluator
 from ray.rllib.optimizers import AsyncOptimizer
 from ray.tune.result import TrainingResult
+from ray.tune.trial import Resources
 
 DEFAULT_CONFIG = {
     # Number of workers (excluding master)
-    "num_workers": 4,
+    "num_workers": 1,
     # Size of rollout batch
     "batch_size": 100,
     # Max global norm for each gradient calculated by worker
     "grad_clip": 40.0,
     # Learning rate
     "lr": 0.0001,
+    # Whether to use a GPU for local optimization.
+    "gpu": False,
     # Whether to place workers on GPUs
     "use_gpu_for_workers": False,
     # Model and preprocessor options
@@ -46,6 +49,18 @@ class BCAgent(Agent):
     _default_config = DEFAULT_CONFIG
     _allow_unknown_configs = True
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        if cf["use_gpu_for_workers"]:
+            num_gpus_per_worker = 1
+        else:
+            num_gpus_per_worker = 0
+        return Resources(
+            cpu=1, gpu=cf["gpu"] and 1 or 0,
+            extra_cpu=cf["num_workers"],
+            extra_gpu=num_gpus_per_worker * cf["num_workers"])
+
     def _init(self):
         self.local_evaluator = BCEvaluator(
             self.registry, self.env_creator, self.config, self.logdir)

python/ray/rllib/dqn/apex.py

@@ -3,6 +3,7 @@ from __future__ import division
 from __future__ import print_function
 
 from ray.rllib.dqn.dqn import DQNAgent, DEFAULT_CONFIG as DQN_CONFIG
+from ray.tune.trial import Resources
 
 APEX_DEFAULT_CONFIG = dict(DQN_CONFIG, **dict(
     optimizer_class="ApexOptimizer",
@@ -12,6 +13,7 @@ APEX_DEFAULT_CONFIG = dict(DQN_CONFIG, **dict(
         debug=False,
     )),
     n_step=3,
+    gpu=True,
     num_workers=32,
     buffer_size=2000000,
     learning_starts=50000,
@@ -35,6 +37,15 @@ class ApexAgent(DQNAgent):
     _agent_name = "APEX"
     _default_config = APEX_DEFAULT_CONFIG
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(
+            cpu=1 + cf["optimizer_config"]["num_replay_buffer_shards"],
+            gpu=cf["gpu"] and 1 or 0,
+            extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"],
+            extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
+
     def update_target_if_needed(self):
         # Ape-X updates based on num steps trained, not sampled
         if self.optimizer.num_steps_trained - self.last_target_update_ts > \

python/ray/rllib/dqn/dqn.py

@@ -13,6 +13,7 @@ from ray.rllib import optimizers
 from ray.rllib.dqn.dqn_evaluator import DQNEvaluator
 from ray.rllib.agent import Agent
 from ray.tune.result import TrainingResult
+from ray.tune.trial import Resources
 
 
 OPTIMIZER_SHARED_CONFIGS = [
@@ -100,14 +101,16 @@ DEFAULT_CONFIG = dict(
     },
 
     # === Parallelism ===
+    # Whether to use a GPU for local optimization.
+    gpu=False,
     # Number of workers for collecting samples with. This only makes sense
     # to increase if your environment is particularly slow to sample, or if
     # you're using the Async or Ape-X optimizers.
     num_workers=0,
     # Whether to allocate GPUs for workers (if > 0).
     num_gpus_per_worker=0,
-    # Whether to reserve CPUs for workers (if not None).
-    num_cpus_per_worker=None,
+    # Whether to allocate CPUs for workers (if > 0).
+    num_cpus_per_worker=1,
     # Optimizer class to use.
     optimizer_class="LocalSyncReplayOptimizer",
     # Config to pass to the optimizer.
@@ -124,6 +127,14 @@ class DQNAgent(Agent):
         "model", "optimizer", "tf_session_args", "env_config"]
     _default_config = DEFAULT_CONFIG
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(
+            cpu=1, gpu=cf["gpu"] and 1 or 0,
+            extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"],
+            extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
+
     def _init(self):
         self.local_evaluator = DQNEvaluator(
             self.registry, self.env_creator, self.config, self.logdir, 0)

python/ray/rllib/es/es.py

@@ -13,6 +13,7 @@ import time
 
 import ray
 from ray.rllib import agent
+from ray.tune.trial import Resources
 
 from ray.rllib.es import optimizers
 from ray.rllib.es import policies
@@ -138,6 +139,11 @@ class ESAgent(agent.Agent):
     _default_config = DEFAULT_CONFIG
     _allow_unknown_subkeys = ["env_config"]
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(cpu=1, gpu=0, extra_cpu=cf["num_workers"])
+
     def _init(self):
         policy_params = {
             "action_noise_std": 0.01

python/ray/rllib/optimizers/apex_optimizer.py

@@ -18,7 +18,7 @@ import ray
 from ray.rllib.optimizers.policy_optimizer import PolicyOptimizer
 from ray.rllib.optimizers.replay_buffer import PrioritizedReplayBuffer
 from ray.rllib.optimizers.sample_batch import SampleBatch
-from ray.rllib.utils.actors import TaskPool
+from ray.rllib.utils.actors import TaskPool, create_colocated
 from ray.rllib.utils.timer import TimerStat
 from ray.rllib.utils.window_stat import WindowStat
 
@@ -163,15 +163,12 @@ class ApexOptimizer(PolicyOptimizer):
         self.learner = LearnerThread(self.local_evaluator)
         self.learner.start()
 
-        # TODO(ekl) use create_colocated() for these actors once
-        # https://github.com/ray-project/ray/issues/1734 is fixed
-        self.replay_actors = [
-            ReplayActor.remote(
-                num_replay_buffer_shards, learning_starts, buffer_size,
-                train_batch_size, prioritized_replay_alpha,
-                prioritized_replay_beta, prioritized_replay_eps, clip_rewards)
-            for _ in range(num_replay_buffer_shards)
-        ]
+        self.replay_actors = create_colocated(
+            ReplayActor,
+            [num_replay_buffer_shards, learning_starts, buffer_size,
+             train_batch_size, prioritized_replay_alpha,
+             prioritized_replay_beta, prioritized_replay_eps, clip_rewards],
+            num_replay_buffer_shards)
         assert len(self.remote_evaluators) > 0
 
         # Stats

python/ray/rllib/pg/pg.py

@@ -9,6 +9,8 @@ from ray.rllib.optimizers import LocalSyncOptimizer
 from ray.rllib.pg.pg_evaluator import PGEvaluator
 from ray.rllib.agent import Agent
 from ray.tune.result import TrainingResult
+from ray.tune.trial import Resources
+
 
 DEFAULT_CONFIG = {
     # Number of workers (excluding master)
@@ -41,6 +43,11 @@ class PGAgent(Agent):
     _agent_name = "PG"
     _default_config = DEFAULT_CONFIG
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(cpu=1, gpu=0, extra_cpu=cf["num_workers"])
+
     def _init(self):
         self.optimizer = LocalSyncOptimizer.make(
             evaluator_cls=PGEvaluator,

python/ray/rllib/ppo/ppo.py

@@ -12,6 +12,7 @@ from tensorflow.python import debug as tf_debug
 
 import ray
 from ray.tune.result import TrainingResult
+from ray.tune.trial import Resources
 from ray.rllib.agent import Agent
 from ray.rllib.utils import FilterManager
 from ray.rllib.ppo.ppo_evaluator import PPOEvaluator
@@ -69,8 +70,10 @@ DEFAULT_CONFIG = {
     "min_steps_per_task": 200,
     # Number of actors used to collect the rollouts
     "num_workers": 5,
-    # Resource requirements for remote actors
-    "worker_resources": {"num_cpus": None},
+    # Whether to allocate GPUs for workers (if > 0).
+    "num_gpus_per_worker": 0,
+    # Whether to allocate CPUs for workers (if > 0).
+    "num_cpus_per_worker": 1,
     # Dump TensorFlow timeline after this many SGD minibatches
     "full_trace_nth_sgd_batch": -1,
     # Whether to profile data loading
@@ -89,17 +92,26 @@ DEFAULT_CONFIG = {
 
 class PPOAgent(Agent):
     _agent_name = "PPO"
-    _allow_unknown_subkeys = ["model", "tf_session_args", "env_config",
-                              "worker_resources"]
+    _allow_unknown_subkeys = ["model", "tf_session_args", "env_config"]
     _default_config = DEFAULT_CONFIG
 
+    @classmethod
+    def default_resource_request(cls, config):
+        cf = dict(cls._default_config, **config)
+        return Resources(
+            cpu=1,
+            gpu=len([d for d in cf["devices"] if "gpu" in d.lower()]),
+            extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"],
+            extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
+
     def _init(self):
         self.global_step = 0
         self.kl_coeff = self.config["kl_coeff"]
         self.local_evaluator = PPOEvaluator(
             self.registry, self.env_creator, self.config, self.logdir, False)
         RemotePPOEvaluator = ray.remote(
-            **self.config["worker_resources"])(PPOEvaluator)
+            num_cpus=self.config["num_cpus_per_worker"],
+            num_gpus=self.config["num_gpus_per_worker"])(PPOEvaluator)
         self.remote_evaluators = [
             RemotePPOEvaluator.remote(
                 self.registry, self.env_creator, self.config, self.logdir,

python/ray/rllib/train.py

@@ -34,16 +34,22 @@ parser.add_argument(
     "--redis-address", default=None, type=str,
     help="The Redis address of the cluster.")
 parser.add_argument(
-    "--num-cpus", default=None, type=int,
-    help="Number of CPUs to allocate to Ray.")
+    "--ray-num-cpus", default=None, type=int,
+    help="--num-cpus to pass to Ray. This only has an affect in local mode.")
 parser.add_argument(
-    "--num-gpus", default=None, type=int,
-    help="Number of GPUs to allocate to Ray.")
+    "--ray-num-gpus", default=None, type=int,
+    help="--num-gpus to pass to Ray. This only has an affect in local mode.")
 parser.add_argument(
     "--experiment-name", default="default", type=str,
     help="Name of the subdirectory under `local_dir` to put results in.")
 parser.add_argument(
     "--env", default=None, type=str, help="The gym environment to use.")
+parser.add_argument(
+    "--queue-trials", action='store_true',
+    help=(
+        "Whether to queue trials when the cluster does not currently have "
+        "enough resources to launch one. This should be set to True when "
+        "running on an autoscaling cluster to enable automatic scale-up."))
 parser.add_argument(
     "-f", "--config-file", default=None, type=str,
     help="If specified, use config options from this file. Note that this "
@@ -62,7 +68,9 @@ if __name__ == "__main__":
                 "run": args.run,
                 "checkpoint_freq": args.checkpoint_freq,
                 "local_dir": args.local_dir,
-                "trial_resources": resources_to_json(args.trial_resources),
+                "trial_resources": (
+                    args.trial_resources and
+                    resources_to_json(args.trial_resources)),
                 "stop": args.stop,
                 "config": dict(args.config, env=args.env),
                 "restore": args.restore,
@@ -79,5 +87,7 @@ if __name__ == "__main__":
 
     ray.init(
         redis_address=args.redis_address,
-        num_cpus=args.num_cpus, num_gpus=args.num_gpus)
-    run_experiments(experiments, scheduler=_make_scheduler(args))
+        num_cpus=args.ray_num_cpus, num_gpus=args.ray_num_gpus)
+    run_experiments(
+        experiments, scheduler=_make_scheduler(args),
+        queue_trials=args.queue_trials)

python/ray/rllib/tuned_examples/cartpole-grid-search-example.yaml

@@ -4,9 +4,6 @@ cartpole-ppo:
     stop:
         episode_reward_mean: 200
         time_total_s: 180
-    trial_resources:
-        cpu: 1
-        extra_cpu: 1
     config:
         num_workers: 2
         num_sgd_iter:

python/ray/rllib/tuned_examples/hopper-ppo.yaml

@@ -1,8 +1,4 @@
 hopper-ppo:
     env: Hopper-v1
     run: PPO
-    trial_resources:
-        cpu: 1
-        gpu: 4
-        extra_cpu: 64
     config: {"gamma": 0.995, "kl_coeff": 1.0, "num_sgd_iter": 20, "sgd_stepsize": .0001, "sgd_batchsize": 32768, "devices": ["/gpu:0", "/gpu:1", "/gpu:2", "/gpu:3"], "tf_session_args": {"device_count": {"GPU": 4}, "log_device_placement": false, "allow_soft_placement": true}, "timesteps_per_batch": 160000, "num_workers": 64}

python/ray/rllib/tuned_examples/humanoid-es.yaml

@@ -1,9 +1,6 @@
 humanoid-es:
     env: Humanoid-v1
     run: ES
-    trial_resources:
-        cpu: 1
-        extra_cpu: 100
     stop:
         episode_reward_mean: 6000
     config:

python/ray/rllib/tuned_examples/humanoid-ppo-gae.yaml

@@ -3,9 +3,5 @@ humanoid-ppo-gae:
     run: PPO
     stop:
         episode_reward_mean: 6000
-    trial_resources:
-        cpu: 1
-        gpu: 4
-        extra_cpu: 64
     config: {"lambda": 0.95, "clip_param": 0.2, "kl_coeff": 1.0, "num_sgd_iter": 20, "sgd_stepsize": .0001, "sgd_batchsize": 32768, "horizon": 5000, "devices": ["/gpu:0", "/gpu:1", "/gpu:2", "/gpu:3"], "tf_session_args": {"device_count": {"GPU": 4}, "log_device_placement": false, "allow_soft_placement": true}, "timesteps_per_batch": 320000, "num_workers": 64, "model": {"free_log_std": true}, "write_logs": false}
 

python/ray/rllib/tuned_examples/humanoid-ppo.yaml

@@ -3,8 +3,4 @@ humanoid-ppo:
     run: PPO
     stop:
        episode_reward_mean: 6000
-    trial_resources:
-       cpu: 1
-       gpu: 4
-       extra_cpu: 64
     config: {"kl_coeff": 1.0, "num_sgd_iter": 20, "sgd_stepsize": .0001, "sgd_batchsize": 32768, "devices": ["/gpu:0", "/gpu:1", "/gpu:2", "/gpu:3"], "tf_session_args": {"device_count": {"GPU": 4}, "log_device_placement": false, "allow_soft_placement": true}, "timesteps_per_batch": 320000, "num_workers": 64, "model": {"free_log_std": true}, "use_gae": false}

python/ray/rllib/tuned_examples/hyperband-cartpole.yaml

@@ -5,9 +5,6 @@ cartpole-ppo:
     stop:
         episode_reward_mean: 200
         time_total_s: 180
-    trial_resources:
-        cpu: 1
-        extra_cpu: 1
     config:
         num_workers: 1
         num_sgd_iter:

python/ray/rllib/tuned_examples/pendulum-ppo.yaml

@@ -2,9 +2,6 @@
 pendulum-ppo:
     env: Pendulum-v0
     run: PPO
-    trial_resources:
-        cpu: 1
-        extra_cpu: 4
     config:
         timesteps_per_batch: 2048
         num_workers: 4

python/ray/rllib/tuned_examples/pong-a3c-pytorch.yaml

@@ -1,9 +1,6 @@
 pong-a3c-pytorch-cnn:
     env: PongDeterministic-v4
     run: A3C
-    trial_resources:
-        cpu: 1
-        extra_cpu: 16
     config:
         num_workers: 16
         batch_size: 20

python/ray/rllib/tuned_examples/pong-a3c.yaml

@@ -1,9 +1,6 @@
 pong-a3c:
     env: PongDeterministic-v4
     run: A3C
-    trial_resources:
-        cpu: 1
-        extra_cpu: 16
     config:
         num_workers: 16
         batch_size: 20

python/ray/rllib/tuned_examples/pong-apex.yaml

@@ -4,11 +4,6 @@
 pong-apex:
     env: PongNoFrameskip-v4
     run: APEX
-    trial_resources:
-        cpu: 1
-        gpu: 1
-        extra_cpu:
-            eval: 4 + spec.config.num_workers
     config:
         target_network_update_freq: 50000
         num_workers: 32

python/ray/rllib/tuned_examples/pong-ppo.yaml

@@ -8,10 +8,6 @@
 pong-deterministic-ppo:
     env: PongDeterministic-v4
     run: PPO
-    trial_resources:
-        cpu: 1
-        gpu: 1
-        extra_cpu: 4
     stop:
         episode_reward_mean: 21
     config:

python/ray/rllib/tuned_examples/regression_tests/cartpole-a3c.yaml

@@ -4,8 +4,6 @@ cartpole-a3c:
     stop:
         episode_reward_mean: 200
         time_total_s: 600
-    trial_resources:
-        cpu: 2
     config:
         num_workers: 4
         gamma: 0.95

python/ray/rllib/tuned_examples/regression_tests/cartpole-dqn.yaml

@@ -4,8 +4,6 @@ cartpole-dqn:
     stop:
         episode_reward_mean: 200
         time_total_s: 600
-    trial_resources:
-        cpu: 1
     config:
         n_step: 3
         gamma: 0.95

python/ray/rllib/tuned_examples/regression_tests/cartpole-es.yaml

@@ -4,8 +4,6 @@ cartpole-es:
     stop:
         episode_reward_mean: 200
         time_total_s: 300
-    trial_resources:
-        cpu: 2
     config:
         num_workers: 2
         noise_size: 25000000

python/ray/rllib/tuned_examples/regression_tests/cartpole-ppo.yaml

@@ -4,7 +4,5 @@ cartpole-ppo:
     stop:
         episode_reward_mean: 200
         time_total_s: 300
-    trial_resources:
-        cpu: 1
     config:
         num_workers: 1

python/ray/rllib/tuned_examples/walker2d-ppo.yaml

@@ -1,8 +1,4 @@
 walker2d-v1-ppo:
     env: Walker2d-v1
     run: PPO
-    trial_resources:
-        cpu: 1
-        gpu: 4
-        extra_cpu: 64
     config: {"kl_coeff": 1.0, "num_sgd_iter": 20, "sgd_stepsize": .0001, "sgd_batchsize": 32768, "devices": ["/gpu:0", "/gpu:1", "/gpu:2", "/gpu:3"], "tf_session_args": {"device_count": {"GPU": 4}, "log_device_placement": false, "allow_soft_placement": true}, "timesteps_per_batch": 320000, "num_workers": 64}