[rllib] clarify train batch size for PPO (#2793)

It's possible to configure PPO in a way that ends up discarding most of the samples (they are treated as "stragglers"). Add a warning when this happens, and raise an exception if the waste is particularly egregious.

doc/source/rllib-config.svg

@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 651.4671916010499 211.1784776902887" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l651.46716 0l0 211.17848l-651.46716 0l0 -211.17848z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l651.46716 0l0 211.17848l-651.46716 0z" fill-rule="evenodd"/><path fill="#cfe2f3" d="m282.34647 62.354332l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m282.34647 62.354332l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path fill="#cfe2f3" d="m297.09088 77.21757l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m297.09088 77.21757l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path fill="#cfe2f3" d="m311.83533 92.0808l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m311.83533 92.0808l160.0 0l0 112.15748l-160.0 0z" fill-rule="evenodd"/><path fill="#000000" d="m326.3909 197.57953l0 -13.359375l5.046875 0q1.328125 0 2.03125 0.125q0.96875 0.171875 1.640625 0.640625q0.671875 0.453125 1.078125 1.28125q0.40625 0.828125 0.40625 1.828125q0 1.703125 -1.09375 2.890625q-1.078125 1.171875 -3.921875 1.171875l-3.421875 0l0 5.421875l-1.765625 0zm1.765625 -7.0l3.453125 0q1.71875 0 2.4375 -0.640625q0.71875 -0.640625 0.71875 -1.796875q0 -0.84375 -0.421875 -1.4375q-0.421875 -0.59375 -1.125 -0.78125q-0.4375 -0.125 -1.640625 -0.125l-3.421875 0l0 4.78125zm9.865448 2.15625q0 -2.6875 1.484375 -3.96875q1.25 -1.078125 3.046875 -1.078125q2.0 0 3.265625 1.3125q1.265625 1.296875 1.265625 3.609375q0 1.859375 -0.5625 2.9375q-0.5625 1.0625 -1.640625 1.65625q-1.0625 0.59375 -2.328125 0.59375q-2.03125 0 -3.28125 -1.296875q-1.25 -1.3125 -1.25 -3.765625zm1.6875 0q0 1.859375 0.796875 2.796875q0.8125 0.921875 2.046875 0.921875q1.21875 0 2.03125 -0.921875q0.8125 -0.9375 0.8125 -2.84375q0 -1.796875 -0.8125 -2.71875q-0.8125 -0.921875 -2.03125 -0.921875q-1.234375 0 -2.046875 0.921875q-0.796875 0.90625 -0.796875 2.765625zm9.250702 4.84375l0 -13.359375l1.640625 0l0 13.359375l-1.640625 0zm4.1917114 -11.46875l0 -1.890625l1.640625 0l0 1.890625l-1.640625 0zm0 11.46875l0 -9.671875l1.640625 0l0 9.671875l-1.640625 0zm10.457306 -3.546875l1.609375 0.21875q-0.265625 1.65625 -1.359375 2.609375q-1.078125 0.9375 -2.671875 0.9375q-1.984375 0 -3.1875 -1.296875q-1.203125 -1.296875 -1.203125 -3.71875q0 -1.578125 0.515625 -2.75q0.515625 -1.171875 1.578125 -1.75q1.0625 -0.59375 2.3125 -0.59375q1.578125 0 2.578125 0.796875q1.0 0.796875 1.28125 2.265625l-1.59375 0.234375q-0.234375 -0.96875 -0.8125 -1.453125q-0.578125 -0.5 -1.390625 -0.5q-1.234375 0 -2.015625 0.890625q-0.78125 0.890625 -0.78125 2.8125q0 1.953125 0.75 2.84375q0.75 0.875 1.953125 0.875q0.96875 0 1.609375 -0.59375q0.65625 -0.59375 0.828125 -1.828125zm2.9375 7.265625l-0.1875 -1.53125q0.546875 0.140625 0.9375 0.140625q0.546875 0 0.875 -0.1875q0.328125 -0.171875 0.546875 -0.5q0.15625 -0.25 0.5 -1.21875q0.046875 -0.140625 0.140625 -0.40625l-3.671875 -9.6875l1.765625 0l2.015625 5.59375q0.390625 1.078125 0.703125 2.25q0.28125 -1.125 0.671875 -2.203125l2.078125 -5.640625l1.640625 0l-3.6875 9.828125q-0.59375 1.609375 -0.921875 2.203125q-0.4375 0.8125 -1.0 1.1875q-0.5625 0.375 -1.34375 0.375q-0.484375 0 -1.0625 -0.203125zm14.823944 -3.71875l0 -13.359375l9.65625 0l0 1.578125l-7.875 0l0 4.09375l7.375 0l0 1.5625l-7.375 0l0 4.546875l8.1875 0l0 1.578125l-9.96875 0zm14.896698 0l-3.6875 -9.671875l1.734375 0l2.078125 5.796875q0.328125 0.9375 0.625 1.9375q0.203125 -0.765625 0.609375 -1.828125l2.140625 -5.90625l1.6875 0l-3.65625 9.671875l-1.53125 0zm12.953125 -1.1875q-0.921875 0.765625 -1.765625 1.09375q-0.828125 0.3125 -1.796875 0.3125q-1.59375 0 -2.453125 -0.78125q-0.859375 -0.78125 -0.859375 -1.984375q0 -0.71875 0.328125 -1.296875q0.328125 -0.59375 0.84375 -0.9375q0.53125 -0.359375 1.1875 -0.546875q0.46875 -0.125 1.453125 -0.25q1.984375 -0.234375 2.921875 -0.5625q0.015625 -0.34375 0.015625 -0.421875q0 -1.0 -0.46875 -1.421875q-0.625 -0.546875 -1.875 -0.546875q-1.15625 0 -1.703125 0.40625q-0.546875 0.40625 -0.8125 1.421875l-1.609375 -0.21875q0.21875 -1.015625 0.71875 -1.640625q0.5 -0.640625 1.453125 -0.984375q0.953125 -0.34375 2.1875 -0.34375q1.25 0 2.015625 0.296875q0.78125 0.28125 1.140625 0.734375q0.375 0.4375 0.515625 1.109375q0.078125 0.421875 0.078125 1.515625l0 2.1875q0 2.28125 0.109375 2.890625q0.109375 0.59375 0.40625 1.15625l-1.703125 0q-0.265625 -0.515625 -0.328125 -1.1875zm-0.140625 -3.671875q-0.890625 0.375 -2.671875 0.625q-1.015625 0.140625 -1.4375 0.328125q-0.421875 0.1875 -0.65625 0.53125q-0.21875 0.34375 -0.21875 0.78125q0 0.65625 0.5 1.09375q0.5 0.4375 1.453125 0.4375q0.9375 0 1.671875 -0.40625q0.75 -0.421875 1.0Line truncated

doc/source/rllib-training.rst

@@ -50,10 +50,13 @@ In an example below, we train A2C by specifying 8 workers through the config fla
     python ray/python/ray/rllib/train.py --env=PongDeterministic-v4 \
         --run=A2C --config '{"num_workers": 8, "monitor": true}'
 
+.. image:: rllib-config.svg
+
 Specifying Resources
 ~~~~~~~~~~~~~~~~~~~~
 
 You can control the degree of parallelism used by setting the ``num_workers`` hyperparameter for most agents. Many agents also provide a ``num_gpus`` or ``gpu`` option. In addition, you can allocate a fraction of a GPU by setting ``gpu_fraction: f``. For example, with DQN you can pack five agents onto one GPU by setting ``gpu_fraction: 0.2``. Note that fractional GPU support requires enabling the experimental Xray backend by setting the environment variable ``RAY_USE_XRAY=1``.
+>>>>>>> 01b030bd57f014386aa5e4c67a2e069938528abb
 
 Evaluating Trained Agents
 ~~~~~~~~~~~~~~~~~~~~~~~~~

python/ray/rllib/agents/agent.py

@@ -26,6 +26,9 @@ COMMON_CONFIG = {
     "num_workers": 2,
     # Default sample batch size
     "sample_batch_size": 200,
+    # Training batch size, if applicable. Should be >= sample_batch_size.
+    # Samples batches will be concatenated together to this size for training.
+    "train_batch_size": 200,
     # Whether to rollout "complete_episodes" or "truncate_episodes"
     "batch_mode": "truncate_episodes",
     # Whether to use a background thread for sampling (slightly off-policy)

python/ray/rllib/agents/es/es.py

@@ -30,7 +30,7 @@ DEFAULT_CONFIG = {
     "l2_coeff": 0.005,
     "noise_stdev": 0.02,
     "episodes_per_batch": 1000,
-    "timesteps_per_batch": 10000,
+    "train_batch_size": 10000,
     "eval_prob": 0.003,
     "return_proc_mode": "centered_rank",
     "num_workers": 10,
@@ -213,8 +213,7 @@ class ESAgent(Agent):
         # Use the actors to do rollouts, note that we pass in the ID of the
         # policy weights.
         results, num_episodes, num_timesteps = self._collect_results(
-            theta_id, config["episodes_per_batch"],
-            config["timesteps_per_batch"])
+            theta_id, config["episodes_per_batch"], config["train_batch_size"])
 
         all_noise_indices = []
         all_training_returns = []

python/ray/rllib/agents/ppo/ppo.py

@@ -20,18 +20,20 @@ DEFAULT_CONFIG = with_common_config({
     "lambda": 1.0,
     # Initial coefficient for KL divergence
     "kl_coeff": 0.2,
+    # Size of batches collected from each worker
+    "sample_batch_size": 200,
     # Number of timesteps collected for each SGD round
-    "timesteps_per_batch": 4000,
+    "train_batch_size": 4000,
+    # Total SGD batch size across all devices for SGD (multi-gpu only)
+    "sgd_minibatch_size": 128,
     # Number of SGD iterations in each outer loop
     "num_sgd_iter": 30,
     # Stepsize of SGD
-    "sgd_stepsize": 5e-5,
+    "lr": 5e-5,
     # Learning rate schedule
     "lr_schedule": None,
     # Share layers for value function
     "vf_share_layers": False,
-    # Total SGD batch size across all devices for SGD (multi-gpu only)
-    "sgd_batchsize": 128,
     # Coefficient of the value function loss
     "vf_loss_coeff": 1.0,
     # Coefficient of the entropy regularizer
@@ -79,6 +81,17 @@ class PPOAgent(Agent):
             extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
 
     def _init(self):
+        waste_ratio = (
+            self.config["sample_batch_size"] * self.config["num_workers"] /
+            self.config["train_batch_size"])
+        if waste_ratio > 1:
+            msg = ("sample_batch_size * num_workers >> train_batch_size. "
+                   "This means that many steps will be discarded. Consider "
+                   "reducing sample_batch_size, or increase train_batch_size.")
+            if waste_ratio > 1.5:
+                raise ValueError(msg)
+            else:
+                print("Warning: " + msg)
         self.local_evaluator = self.make_local_evaluator(
             self.env_creator, self._policy_graph)
         self.remote_evaluators = self.make_remote_evaluators(
@@ -90,15 +103,15 @@ class PPOAgent(Agent):
             self.optimizer = SyncSamplesOptimizer(
                 self.local_evaluator, self.remote_evaluators, {
                     "num_sgd_iter": self.config["num_sgd_iter"],
-                    "timesteps_per_batch": self.config["timesteps_per_batch"]
+                    "train_batch_size": self.config["train_batch_size"]
                 })
         else:
             self.optimizer = LocalMultiGPUOptimizer(
                 self.local_evaluator, self.remote_evaluators, {
-                    "sgd_batch_size": self.config["sgd_batchsize"],
+                    "sgd_batch_size": self.config["sgd_minibatch_size"],
                     "num_sgd_iter": self.config["num_sgd_iter"],
                     "num_gpus": self.config["num_gpus"],
-                    "timesteps_per_batch": self.config["timesteps_per_batch"],
+                    "train_batch_size": self.config["train_batch_size"],
                     "standardize_fields": ["advantages"],
                 })
 

python/ray/rllib/agents/ppo/ppo_policy_graph.py

@@ -191,7 +191,7 @@ class PPOPolicyGraph(LearningRateSchedule, TFPolicyGraph):
             vf_loss_coeff=self.config["vf_loss_coeff"],
             use_gae=self.config["use_gae"])
 
-        LearningRateSchedule.__init__(self, self.config["sgd_stepsize"],
+        LearningRateSchedule.__init__(self, self.config["lr"],
                                       self.config["lr_schedule"])
         TFPolicyGraph.__init__(
             self,

python/ray/rllib/agents/ppo/rollout.py

@@ -6,7 +6,7 @@ import ray
 from ray.rllib.evaluation.sample_batch import SampleBatch
 
 
-def collect_samples(agents, timesteps_per_batch):
+def collect_samples(agents, train_batch_size):
     num_timesteps_so_far = 0
     trajectories = []
     # This variable maps the object IDs of trajectories that are currently
@@ -19,7 +19,7 @@ def collect_samples(agents, timesteps_per_batch):
         fut_sample = agent.sample.remote()
         agent_dict[fut_sample] = agent
 
-    while num_timesteps_so_far < timesteps_per_batch:
+    while num_timesteps_so_far < train_batch_size:
         # TODO(pcm): Make wait support arbitrary iterators and remove the
         # conversion to list here.
         [fut_sample], _ = ray.wait(list(agent_dict))

python/ray/rllib/examples/legacy_multiagent/multiagent_mountaincar.py

@@ -41,7 +41,7 @@ if __name__ == '__main__':
     num_cpus = 4
     ray.init(num_cpus=num_cpus, redirect_output=True)
     config["num_workers"] = num_cpus
-    config["timesteps_per_batch"] = 10
+    config["train_batch_size"] = 1000
     config["num_sgd_iter"] = 10
     config["gamma"] = 0.999
     config["horizon"] = horizon

python/ray/rllib/examples/legacy_multiagent/multiagent_pendulum.py

@@ -41,8 +41,8 @@ if __name__ == '__main__':
     num_cpus = 4
     ray.init(num_cpus=num_cpus, redirect_output=True)
     config["num_workers"] = num_cpus
-    config["timesteps_per_batch"] = 10
-    config["sgd_batchsize"] = 10
+    config["train_batch_size"] = 1000
+    config["sgd_minibatch_size"] = 10
     config["num_sgd_iter"] = 10
     config["gamma"] = 0.999
     config["horizon"] = horizon

python/ray/rllib/optimizers/multi_gpu_optimizer.py

@@ -33,12 +33,12 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
     def _init(self,
               sgd_batch_size=128,
               num_sgd_iter=10,
-              timesteps_per_batch=1024,
+              train_batch_size=1024,
               num_gpus=0,
               standardize_fields=[]):
         self.batch_size = sgd_batch_size
         self.num_sgd_iter = num_sgd_iter
-        self.timesteps_per_batch = timesteps_per_batch
+        self.train_batch_size = train_batch_size
         if not num_gpus:
             self.devices = ["/cpu:0"]
         else:
@@ -99,7 +99,7 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
                 # TODO(rliaw): remove when refactoring
                 from ray.rllib.agents.ppo.rollout import collect_samples
                 samples = collect_samples(self.remote_evaluators,
-                                          self.timesteps_per_batch)
+                                          self.train_batch_size)
             else:
                 samples = self.local_evaluator.sample()
             self._check_not_multiagent(samples)

python/ray/rllib/optimizers/sync_samples_optimizer.py

@@ -17,13 +17,13 @@ class SyncSamplesOptimizer(PolicyOptimizer):
     model weights are then broadcast to all remote evaluators.
     """
 
-    def _init(self, num_sgd_iter=1, timesteps_per_batch=1):
+    def _init(self, num_sgd_iter=1, train_batch_size=1):
         self.update_weights_timer = TimerStat()
         self.sample_timer = TimerStat()
         self.grad_timer = TimerStat()
         self.throughput = RunningStat()
         self.num_sgd_iter = num_sgd_iter
-        self.timesteps_per_batch = timesteps_per_batch
+        self.train_batch_size = train_batch_size
         self.learner_stats = {}
 
     def step(self):
@@ -35,7 +35,7 @@ class SyncSamplesOptimizer(PolicyOptimizer):
 
         with self.sample_timer:
             samples = []
-            while sum(s.count for s in samples) < self.timesteps_per_batch:
+            while sum(s.count for s in samples) < self.train_batch_size:
                 if self.remote_evaluators:
                     samples.extend(
                         ray.get([

python/ray/rllib/test/test_checkpoint_restore.py

@@ -22,7 +22,7 @@ ray.init(num_cpus=10)
 CONFIGS = {
     "ES": {
         "episodes_per_batch": 10,
-        "timesteps_per_batch": 100,
+        "train_batch_size": 100,
         "num_workers": 2
     },
     "DQN": {},
@@ -40,7 +40,7 @@ CONFIGS = {
     },
     "PPO": {
         "num_sgd_iter": 5,
-        "timesteps_per_batch": 1000,
+        "train_batch_size": 1000,
         "num_workers": 2
     },
     "A3C": {

python/ray/rllib/test/test_supported_spaces.py

@@ -107,15 +107,16 @@ class ModelSupportedSpaces(unittest.TestCase):
             "PPO", {
                 "num_workers": 1,
                 "num_sgd_iter": 1,
-                "timesteps_per_batch": 1,
-                "sgd_batchsize": 1
+                "train_batch_size": 10,
+                "sample_batch_size": 10,
+                "sgd_minibatch_size": 1
             }, stats)
         check_support(
             "ES", {
                 "num_workers": 1,
                 "noise_size": 10000000,
                 "episodes_per_batch": 1,
-                "timesteps_per_batch": 1
+                "train_batch_size": 1
             }, stats)
         check_support(
             "ARS", {

python/ray/rllib/tuned_examples/atari-ppo.yaml

@@ -13,9 +13,9 @@ atari-ppo:
         kl_coeff: 0.5
         clip_param: 0.1
         entropy_coeff: 0.01
-        timesteps_per_batch: 5000
+        train_batch_size: 5000
         sample_batch_size: 500
-        sgd_batchsize: 500
+        sgd_minibatch_size: 500
         num_sgd_iter: 10
         num_workers: 10
         num_envs_per_worker: 5

python/ray/rllib/tuned_examples/cartpole-grid-search-example.yaml

@@ -8,5 +8,5 @@ cartpole-ppo:
         num_workers: 2
         num_sgd_iter:
             grid_search: [1, 4]
-        sgd_batchsize:
+        sgd_minibatch_size:
             grid_search: [128, 256, 512]

python/ray/rllib/tuned_examples/hopper-ppo.yaml

@@ -5,8 +5,8 @@ hopper-ppo:
         gamma: 0.995
         kl_coeff: 1.0
         num_sgd_iter: 20
-        sgd_stepsize: .0001
-        sgd_batchsize: 32768
-        timesteps_per_batch: 160000
+        lr: .0001
+        sgd_minibatch_size: 32768
+        train_batch_size: 160000
         num_workers: 64
         num_gpus: 4

python/ray/rllib/tuned_examples/humanoid-ppo-gae.yaml

@@ -9,10 +9,10 @@ humanoid-ppo-gae:
         clip_param: 0.2
         kl_coeff: 1.0
         num_sgd_iter: 20
-        sgd_stepsize: .0001
-        sgd_batchsize: 32768
+        lr: .0001
+        sgd_minibatch_size: 32768
         horizon: 5000
-        timesteps_per_batch: 320000
+        train_batch_size: 320000
         model:
             free_log_std: true
         num_workers: 64

python/ray/rllib/tuned_examples/humanoid-ppo.yaml

@@ -7,9 +7,9 @@ humanoid-ppo:
         gamma: 0.995
         kl_coeff: 1.0
         num_sgd_iter: 20
-        sgd_stepsize: .0001
-        sgd_batchsize: 32768
-        timesteps_per_batch: 320000
+        lr: .0001
+        sgd_minibatch_size: 32768
+        train_batch_size: 320000
         model:
             free_log_std: true
         use_gae: false

python/ray/rllib/tuned_examples/hyperband-cartpole.yaml

@@ -9,5 +9,5 @@ cartpole-ppo:
         num_workers: 1
         num_sgd_iter:
             grid_search: [1, 4]
-        sgd_batchsize:
+        sgd_minibatch_size:
             grid_search: [128, 256, 512]

python/ray/rllib/tuned_examples/pendulum-ppo.yaml

@@ -3,12 +3,12 @@ pendulum-ppo:
     env: Pendulum-v0
     run: PPO
     config:
-        timesteps_per_batch: 2048
+        train_batch_size: 2048
         num_workers: 4
         lambda: 0.1
         gamma: 0.95
-        sgd_stepsize: 0.0003
-        sgd_batchsize: 64
+        lr: 0.0003
+        sgd_minibatch_size: 64
         num_sgd_iter: 10
         model:
             fcnet_hiddens: [64, 64]

python/ray/rllib/tuned_examples/regression_tests/pendulum-ppo.yaml

@@ -6,12 +6,12 @@ pendulum-ppo:
         # expect -140 within 300-500k steps
         timesteps_total: 600000
     config:
-        timesteps_per_batch: 2048
+        train_batch_size: 2048
         num_workers: 4
         lambda: 0.1
         gamma: 0.95
-        sgd_stepsize: 0.0003
-        sgd_batchsize: 64
+        lr: 0.0003
+        sgd_minibatch_size: 64
         num_sgd_iter: 10
         model:
             fcnet_hiddens: [64, 64]

python/ray/rllib/tuned_examples/walker2d-ppo.yaml

@@ -4,8 +4,8 @@ walker2d-v1-ppo:
     config:
         kl_coeff: 1.0
         num_sgd_iter: 20
-        sgd_stepsize: .0001
-        sgd_batchsize: 32768
-        timesteps_per_batch: 320000
+        lr: .0001
+        sgd_minibatch_size: 32768
+        train_batch_size: 320000
         num_workers: 64
         num_gpus: 4

python/ray/tune/examples/pbt_ppo_example.py

@@ -21,8 +21,8 @@ if __name__ == "__main__":
     # Postprocess the perturbed config to ensure it's still valid
     def explore(config):
         # ensure we collect enough timesteps to do sgd
-        if config["timesteps_per_batch"] < config["sgd_batchsize"] * 2:
-            config["timesteps_per_batch"] = config["sgd_batchsize"] * 2
+        if config["train_batch_size"] < config["sgd_minibatch_size"] * 2:
+            config["train_batch_size"] = config["sgd_minibatch_size"] * 2
         # ensure we run at least one sgd iter
         if config["num_sgd_iter"] < 1:
             config["num_sgd_iter"] = 1
@@ -37,10 +37,10 @@ if __name__ == "__main__":
         hyperparam_mutations={
             "lambda": lambda: random.uniform(0.9, 1.0),
             "clip_param": lambda: random.uniform(0.01, 0.5),
-            "sgd_stepsize": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],
+            "lr": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],
             "num_sgd_iter": lambda: random.randint(1, 30),
-            "sgd_batchsize": lambda: random.randint(128, 16384),
-            "timesteps_per_batch": lambda: random.randint(2000, 160000),
+            "sgd_minibatch_size": lambda: random.randint(128, 16384),
+            "train_batch_size": lambda: random.randint(2000, 160000),
         },
         custom_explore_fn=explore)
 
@@ -61,13 +61,13 @@ if __name__ == "__main__":
                     # These params are tuned from a fixed starting value.
                     "lambda": 0.95,
                     "clip_param": 0.2,
-                    "sgd_stepsize": 1e-4,
+                    "lr": 1e-4,
                     # These params start off randomly drawn from a set.
                     "num_sgd_iter":
                         lambda spec: random.choice([10, 20, 30]),
-                    "sgd_batchsize":
+                    "sgd_minibatch_size":
                         lambda spec: random.choice([128, 512, 2048]),
-                    "timesteps_per_batch":
+                    "train_batch_size":
                         lambda spec: random.choice([10000, 20000, 40000])
                 },
             },

test/jenkins_tests/run_multi_node_tests.sh

@@ -30,7 +30,7 @@ docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
-    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "model": {"free_log_std": true}}'
+    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"free_log_std": true}}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
@@ -51,21 +51,21 @@ docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env CartPole-v1 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
-    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "use_gae": false}'
+    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "use_gae": false}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
     --env Pendulum-v0 \
     --run ES \
     --stop '{"training_iteration": 2}' \
-    --config '{"stepsize": 0.01, "episodes_per_batch": 20, "timesteps_per_batch": 100, "num_workers": 2}'
+    --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
     --env Pong-v0 \
     --run ES \
     --stop '{"training_iteration": 2}' \
-    --config '{"stepsize": 0.01, "episodes_per_batch": 20, "timesteps_per_batch": 100, "num_workers": 2}'
+    --config '{"stepsize": 0.01, "episodes_per_batch": 20, "train_batch_size": 100, "num_workers": 2}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
@@ -105,7 +105,7 @@ docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env FrozenLake-v0 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
-    --config '{"num_sgd_iter": 10, "sgd_batchsize": 64, "timesteps_per_batch": 1000, "num_workers": 1}'
+    --config '{"num_sgd_iter": 10, "sgd_minibatch_size": 64, "train_batch_size": 1000, "num_workers": 1}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
@@ -119,7 +119,7 @@ docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env MontezumaRevenge-v0 \
     --run PPO \
     --stop '{"training_iteration": 2}' \
-    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}}'
+    --config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "lr": 1e-4, "sgd_minibatch_size": 64, "train_batch_size": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}}'
 
 docker run  -e "RAY_USE_XRAY=1" --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \