diff --git a/rllib/BUILD b/rllib/BUILD index ab62877a3..af1ccbcba 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -110,7 +110,7 @@ py_test( py_test( name = "test_ppo", tags = ["agents_dir"], - size = "medium", + size = "large", srcs = ["agents/ppo/tests/test_ppo.py", "agents/ppo/tests/test.py"] # TODO(sven): Move down once PR 6889 merged ) diff --git a/rllib/agents/ppo/ppo.py b/rllib/agents/ppo/ppo.py index f8bc143dc..4df6a3362 100644 --- a/rllib/agents/ppo/ppo.py +++ b/rllib/agents/ppo/ppo.py @@ -67,6 +67,9 @@ DEFAULT_CONFIG = with_common_config({ # usually slower, but you might want to try it if you run into issues with # the default optimizer. "simple_optimizer": False, + # Whether to fake GPUs (using CPUs). + # Set this to True for debugging on non-GPU machines (set `num_gpus` > 0). + "_fake_gpus": False, # Use PyTorch as framework? "use_pytorch": False }) @@ -92,7 +95,8 @@ def choose_policy_optimizer(workers, config): num_envs_per_worker=config["num_envs_per_worker"], train_batch_size=config["train_batch_size"], standardize_fields=["advantages"], - shuffle_sequences=config["shuffle_sequences"]) + shuffle_sequences=config["shuffle_sequences"], + _fake_gpus=config["_fake_gpus"]) def update_kl(trainer, fetches): diff --git a/rllib/agents/ppo/tests/test_ppo.py b/rllib/agents/ppo/tests/test_ppo.py index 7dcb42985..e93be0131 100644 --- a/rllib/agents/ppo/tests/test_ppo.py +++ b/rllib/agents/ppo/tests/test_ppo.py @@ -46,6 +46,33 @@ class TestPPO(unittest.TestCase): for i in range(num_iterations): trainer.train() + def test_ppo_fake_multi_gpu_learning(self): + """Test whether PPOTrainer can learn CartPole w/ faked multi-GPU.""" + config = ppo.DEFAULT_CONFIG.copy() + # Fake GPU setup. + config["num_gpus"] = 2 + config["_fake_gpus"] = True + # Mimick tuned_example for PPO CartPole. + config["num_workers"] = 1 + config["lr"] = 0.0003 + config["observation_filter"] = "MeanStdFilter" + config["num_sgd_iter"] = 6 + config["vf_share_layers"] = True + config["vf_loss_coeff"] = 0.01 + config["model"]["fcnet_hiddens"] = [32] + config["model"]["fcnet_activation"] = "linear" + + trainer = ppo.PPOTrainer(config=config, env="CartPole-v0") + num_iterations = 200 + learnt = False + for i in range(num_iterations): + results = trainer.train() + if results["episode_reward_mean"] > 150: + learnt = True + break + print(results) + assert learnt, "PPO multi-GPU (with fake-GPUs) did not learn CartPole!" + def test_ppo_exploration_setup(self): """Tests, whether PPO runs with different exploration setups.""" config = ppo.DEFAULT_CONFIG.copy() diff --git a/rllib/optimizers/multi_gpu_optimizer.py b/rllib/optimizers/multi_gpu_optimizer.py index 9df3d18d8..cf33f44b0 100644 --- a/rllib/optimizers/multi_gpu_optimizer.py +++ b/rllib/optimizers/multi_gpu_optimizer.py @@ -29,8 +29,9 @@ class LocalMultiGPUOptimizer(PolicyOptimizer): A number of SGD passes are then taken over the in-memory data. For more details, see `multi_gpu_impl.LocalSyncParallelOptimizer`. - This optimizer is Tensorflow-specific and require the underlying - Policy to be a TFPolicy instance that support `.copy()`. + This optimizer is Tensorflow-specific and requires the underlying + Policy to be a TFPolicy instance that implements the `copy()` method + for multi-GPU tower generation. Note that all replicas of the TFPolicy will merge their extra_compute_grad and apply_grad feed_dicts and fetches. This @@ -46,7 +47,8 @@ class LocalMultiGPUOptimizer(PolicyOptimizer): train_batch_size=1024, num_gpus=0, standardize_fields=[], - shuffle_sequences=True): + shuffle_sequences=True, + _fake_gpus=False): """Initialize a synchronous multi-gpu optimizer. Arguments: @@ -62,6 +64,9 @@ class LocalMultiGPUOptimizer(PolicyOptimizer): to normalize shuffle_sequences (bool): whether to shuffle the train batch prior to SGD to break up correlations + _fake_gpus (bool): Whether to use fake-GPUs (CPUs) instead of + actual GPUs (should only be used for testing on non-GPU + machines). """ PolicyOptimizer.__init__(self, workers) @@ -71,12 +76,16 @@ class LocalMultiGPUOptimizer(PolicyOptimizer): self.rollout_fragment_length = rollout_fragment_length self.train_batch_size = train_batch_size self.shuffle_sequences = shuffle_sequences + + # Collect actual devices to use. if not num_gpus: - self.devices = ["/cpu:0"] - else: - self.devices = [ - "/gpu:{}".format(i) for i in range(int(math.ceil(num_gpus))) - ] + _fake_gpus = True + num_gpus = 1 + type_ = "cpu" if _fake_gpus else "gpu" + self.devices = [ + "/{}:{}".format(type_, i) for i in range(int(math.ceil(num_gpus))) + ] + self.batch_size = int(sgd_batch_size / len(self.devices)) * len( self.devices) assert self.batch_size % len(self.devices) == 0 diff --git a/rllib/policy/dynamic_tf_policy.py b/rllib/policy/dynamic_tf_policy.py index 945e8a10a..9853f066d 100644 --- a/rllib/policy/dynamic_tf_policy.py +++ b/rllib/policy/dynamic_tf_policy.py @@ -112,6 +112,8 @@ class DynamicTFPolicy(TFPolicy): prev_actions = existing_inputs[SampleBatch.PREV_ACTIONS] prev_rewards = existing_inputs[SampleBatch.PREV_REWARDS] action_input = existing_inputs[SampleBatch.ACTIONS] + explore = existing_inputs["is_exploring"] + timestep = existing_inputs["timestep"] else: obs = tf.placeholder( tf.float32, @@ -123,8 +125,9 @@ class DynamicTFPolicy(TFPolicy): action_space, "prev_action") prev_rewards = tf.placeholder( tf.float32, [None], name="prev_reward") - - explore = tf.placeholder_with_default(False, (), name="is_exploring") + explore = tf.placeholder_with_default( + True, (), name="is_exploring") + timestep = tf.placeholder(tf.int32, (), name="timestep") self._input_dict = { SampleBatch.CUR_OBS: obs, @@ -175,8 +178,6 @@ class DynamicTFPolicy(TFPolicy): for s in self.model.get_initial_state() ] - timestep = tf.placeholder(tf.int32, (), name="timestep") - # Fully customized action generation (e.g., custom policy). if action_sampler_fn: sampled_action, sampled_action_logp = action_sampler_fn( @@ -281,8 +282,9 @@ class DynamicTFPolicy(TFPolicy): existing_inputs[len(self._loss_inputs) + i])) if rnn_inputs: rnn_inputs.append(("seq_lens", existing_inputs[-1])) - input_dict = OrderedDict([(k, existing_inputs[i]) for i, ( - k, _) in enumerate(self._loss_inputs)] + rnn_inputs) + input_dict = OrderedDict([("is_exploring", self._is_exploring), ( + "timestep", self._timestep)] + [(k, existing_inputs[i]) for i, ( + k, _) in enumerate(self._loss_inputs)] + rnn_inputs) instance = self.__class__( self.observation_space, self.action_space, diff --git a/rllib/tuned_examples/regression_tests/cartpole-ppo-tf-multi-gpu.yaml b/rllib/tuned_examples/regression_tests/cartpole-ppo-tf-multi-gpu.yaml new file mode 100644 index 000000000..386612557 --- /dev/null +++ b/rllib/tuned_examples/regression_tests/cartpole-ppo-tf-multi-gpu.yaml @@ -0,0 +1,20 @@ +cartpole-ppo-tf-multi-gpu: + env: CartPole-v0 + run: PPO + stop: + episode_reward_mean: 150 + timesteps_total: 100000 + config: + gamma: 0.99 + lr: 0.0003 + num_workers: 1 + observation_filter: MeanStdFilter + num_sgd_iter: 6 + vf_share_layers: true + vf_loss_coeff: 0.01 + model: + fcnet_hiddens: [32] + fcnet_activation: linear + # Use fake-GPU setup to prove towers are working and learning. + num_gpus: 6 + _fake_gpus: true