From 4b278c36fcdeda1a519f43104ed881746f3a863b Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Wed, 9 Sep 2020 17:33:21 +0200 Subject: [PATCH] [RLlib] Behavioral Cloning (from MARWIL). (#10619) --- doc/source/rllib-algorithms.rst | 35 ++- rllib/BUILD | 10 + rllib/agents/marwil/__init__.py | 3 + rllib/agents/marwil/bc.py | 29 +++ rllib/agents/marwil/marwil.py | 20 +- rllib/agents/marwil/tests/test_bc.py | 65 ++++++ rllib/agents/marwil/tests/test_marwil.py | 2 + rllib/agents/registry.py | 200 +++++++++--------- rllib/agents/trainer.py | 7 + rllib/agents/trainer_template.py | 19 +- rllib/evaluation/sampler.py | 5 +- rllib/examples/unity3d_env_local.py | 8 +- rllib/offline/json_reader.py | 2 +- rllib/tuned_examples/marwil/cartpole-bc.yaml | 20 ++ .../marwil/cartpole-marwil.yaml | 4 +- 15 files changed, 301 insertions(+), 128 deletions(-) create mode 100644 rllib/agents/marwil/bc.py create mode 100644 rllib/agents/marwil/tests/test_bc.py create mode 100644 rllib/tuned_examples/marwil/cartpole-bc.yaml diff --git a/doc/source/rllib-algorithms.rst b/doc/source/rllib-algorithms.rst index 7e6553e29..476c0f4c9 100644 --- a/doc/source/rllib-algorithms.rst +++ b/doc/source/rllib-algorithms.rst @@ -13,6 +13,7 @@ Algorithm Frameworks Discrete Actions Continuous Actions Multi- =================== ========== ======================= ================== =========== ============================================================= `A2C, A3C`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_, `+LSTM auto-wrapping`_, `+Transformer`_, `+autoreg`_ `ARS`_ tf + torch **Yes** **Yes** No +`BC`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_ `ES`_ tf + torch **Yes** **Yes** No `DDPG`_, `TD3`_ tf + torch No **Yes** **Yes** `APEX-DDPG`_ tf + torch No **Yes** **Yes** @@ -547,10 +548,15 @@ Tuned examples: `Humanoid-v1 `__ `[implementation] `__ MARWIL is a hybrid imitation learning and policy gradient algorithm suitable for training on batched historical data. When the ``beta`` hyperparameter is set to zero, the MARWIL objective reduces to vanilla imitation learning. MARWIL requires the `offline datasets API `__ to be used. +`[paper] `__ +`[implementation] `__ + +MARWIL is a hybrid imitation learning and policy gradient algorithm suitable for training on batched historical data. +When the ``beta`` hyperparameter is set to zero, the MARWIL objective reduces to vanilla imitation learning (see `BC`_). +MARWIL requires the `offline datasets API `__ to be used. Tuned examples: `CartPole-v0 `__ @@ -562,6 +568,29 @@ Tuned examples: `CartPole-v0 `__ +`[implementation] `__ + +Our behavioral cloning implementation is directly derived from our `MARWIL`_ implementation, +with the only difference being the ``beta`` parameter force-set to 0.0. This makes +BC try to match the behavior policy, which generated the offline data, disregarding any resulting rewards. +BC requires the `offline datasets API `__ to be used. + +Tuned examples: `CartPole-v0 `__ + +**BC-specific configs** (see also `common configs `__): + +.. literalinclude:: ../../rllib/agents/marwil/bc.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + + Contextual Bandits (contrib/bandits) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/rllib/BUILD b/rllib/BUILD index 7087beb20..21be09591 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -468,6 +468,16 @@ py_test( srcs = ["agents/marwil/tests/test_marwil.py"] ) +# BCTrainer (sub-type of MARWIL) +py_test( + name = "test_bc", + tags = ["agents_dir"], + size = "medium", + # Include the json data file. + data = ["tests/data/cartpole/large.json"], + srcs = ["agents/marwil/tests/test_bc.py"] +) + # MAMLTrainer py_test( name = "test_maml", diff --git a/rllib/agents/marwil/__init__.py b/rllib/agents/marwil/__init__.py index f901cf072..5b66c96f1 100644 --- a/rllib/agents/marwil/__init__.py +++ b/rllib/agents/marwil/__init__.py @@ -1,8 +1,11 @@ +from ray.rllib.agents.marwil.bc import BCTrainer, BC_DEFAULT_CONFIG from ray.rllib.agents.marwil.marwil import MARWILTrainer, DEFAULT_CONFIG from ray.rllib.agents.marwil.marwil_tf_policy import MARWILTFPolicy from ray.rllib.agents.marwil.marwil_torch_policy import MARWILTorchPolicy __all__ = [ + "BCTrainer", + "BC_DEFAULT_CONFIG", "DEFAULT_CONFIG", "MARWILTFPolicy", "MARWILTorchPolicy", diff --git a/rllib/agents/marwil/bc.py b/rllib/agents/marwil/bc.py new file mode 100644 index 000000000..81f8afce5 --- /dev/null +++ b/rllib/agents/marwil/bc.py @@ -0,0 +1,29 @@ +"""Behavioral Cloning (derived from MARWIL). + +Simply uses the MARWIL agent with beta force-set to 0.0. +""" +from ray.rllib.agents.marwil.marwil import MARWILTrainer, \ + DEFAULT_CONFIG as MARWIL_CONFIG +from ray.rllib.utils.typing import TrainerConfigDict + +# yapf: disable +# __sphinx_doc_begin__ +BC_DEFAULT_CONFIG = MARWILTrainer.merge_trainer_configs( + MARWIL_CONFIG, { + "beta": 0.0, + }) +# __sphinx_doc_end__ +# yapf: enable + + +def validate_config(config: TrainerConfigDict): + if config["beta"] != 0.0: + raise ValueError( + "For behavioral cloning, `beta` parameter must be 0.0!") + + +BCTrainer = MARWILTrainer.with_updates( + name="BC", + default_config=BC_DEFAULT_CONFIG, + validate_config=validate_config, +) diff --git a/rllib/agents/marwil/marwil.py b/rllib/agents/marwil/marwil.py index a196015e0..e68c61dc9 100644 --- a/rllib/agents/marwil/marwil.py +++ b/rllib/agents/marwil/marwil.py @@ -16,22 +16,22 @@ DEFAULT_CONFIG = with_common_config({ # Use importance sampling estimators for reward "input_evaluation": ["is", "wis"], - # Scaling of advantages in exponential terms - # When beta is 0, MARWIL is reduced to imitation learning + # Scaling of advantages in exponential terms. + # When beta is 0.0, MARWIL is reduced to imitation learning. "beta": 1.0, - # Balancing value estimation loss and policy optimization loss + # Balancing value estimation loss and policy optimization loss. "vf_coeff": 1.0, - # Whether to calculate cumulative rewards + # Whether to calculate cumulative rewards. "postprocess_inputs": True, - # Whether to rollout "complete_episodes" or "truncate_episodes" + # Whether to rollout "complete_episodes" or "truncate_episodes". "batch_mode": "complete_episodes", - # Learning rate for adam optimizer + # Learning rate for adam optimizer. "lr": 1e-4, - # Number of timesteps collected for each SGD round + # Number of timesteps collected for each SGD round. "train_batch_size": 2000, - # Number of steps max to keep in the batch replay buffer + # Number of steps max to keep in the batch replay buffer. "replay_buffer_size": 100000, - # Number of steps to read before learning starts + # Number of steps to read before learning starts. "learning_starts": 0, # === Parallelism === "num_workers": 0, @@ -45,8 +45,6 @@ def get_policy_class(config): from ray.rllib.agents.marwil.marwil_torch_policy import \ MARWILTorchPolicy return MARWILTorchPolicy - else: - return MARWILTFPolicy def execution_plan(workers, config): diff --git a/rllib/agents/marwil/tests/test_bc.py b/rllib/agents/marwil/tests/test_bc.py new file mode 100644 index 000000000..31a9b3818 --- /dev/null +++ b/rllib/agents/marwil/tests/test_bc.py @@ -0,0 +1,65 @@ +import os +from pathlib import Path +import unittest + +import ray +import ray.rllib.agents.marwil as marwil +from ray.rllib.utils.framework import try_import_tf +from ray.rllib.utils.test_utils import check_compute_single_action, \ + framework_iterator + +tf1, tf, tfv = try_import_tf() + + +class TestBC(unittest.TestCase): + @classmethod + def setUpClass(cls): + ray.init() + + @classmethod + def tearDownClass(cls): + ray.shutdown() + + def test_bc_compilation_and_learning_from_offline_file(self): + """Test whether a BCTrainer can be built with all frameworks. + + And learns from a historic-data file. + """ + rllib_dir = Path(__file__).parent.parent.parent.parent + print("rllib dir={}".format(rllib_dir)) + data_file = os.path.join(rllib_dir, "tests/data/cartpole/large.json") + print("data_file={} exists={}".format(data_file, + os.path.isfile(data_file))) + + config = marwil.BC_DEFAULT_CONFIG.copy() + config["num_workers"] = 0 # Run locally. + config["evaluation_num_workers"] = 1 + config["evaluation_interval"] = 1 + # Evaluate on actual environment. + config["evaluation_config"] = {"input": "sampler"} + # Learn from offline data. + config["input"] = [data_file] + num_iterations = 300 + + # Test for all frameworks. + for _ in framework_iterator(config, frameworks=("tf", "torch")): + trainer = marwil.BCTrainer(config=config, env="CartPole-v0") + for i in range(num_iterations): + eval_results = trainer.train()["evaluation"] + print("iter={} R={}".format( + i, eval_results["episode_reward_mean"])) + # Learn until some reward is reached on an actual live env. + if eval_results["episode_reward_mean"] > 60.0: + print("learnt!") + break + + check_compute_single_action( + trainer, include_prev_action_reward=True) + + trainer.stop() + + +if __name__ == "__main__": + import pytest + import sys + sys.exit(pytest.main(["-v", __file__])) diff --git a/rllib/agents/marwil/tests/test_marwil.py b/rllib/agents/marwil/tests/test_marwil.py index 49a223da1..b390ada22 100644 --- a/rllib/agents/marwil/tests/test_marwil.py +++ b/rllib/agents/marwil/tests/test_marwil.py @@ -35,7 +35,9 @@ class TestMARWIL(unittest.TestCase): config["num_workers"] = 0 # Run locally. config["evaluation_num_workers"] = 1 config["evaluation_interval"] = 1 + # Evaluate on actual environment. config["evaluation_config"] = {"input": "sampler"} + # Learn from offline data. config["input"] = [data_file] num_iterations = 300 diff --git a/rllib/agents/registry.py b/rllib/agents/registry.py index 2f46106c7..001f921d4 100644 --- a/rllib/agents/registry.py +++ b/rllib/agents/registry.py @@ -5,69 +5,9 @@ import traceback from ray.rllib.contrib.registry import CONTRIBUTED_ALGORITHMS -def _import_sac(): - from ray.rllib.agents import sac - return sac.SACTrainer - - -def _import_appo(): - from ray.rllib.agents import ppo - return ppo.APPOTrainer - - -def _import_ddppo(): - from ray.rllib.agents import ppo - return ppo.DDPPOTrainer - - -def _import_qmix(): - from ray.rllib.agents import qmix - return qmix.QMixTrainer - - -def _import_ddpg(): - from ray.rllib.agents import ddpg - return ddpg.DDPGTrainer - - -def _import_apex_ddpg(): - from ray.rllib.agents import ddpg - return ddpg.ApexDDPGTrainer - - -def _import_td3(): - from ray.rllib.agents import ddpg - return ddpg.TD3Trainer - - -def _import_ppo(): - from ray.rllib.agents import ppo - return ppo.PPOTrainer - - -def _import_es(): - from ray.rllib.agents import es - return es.ESTrainer - - -def _import_ars(): - from ray.rllib.agents import ars - return ars.ARSTrainer - - -def _import_dqn(): - from ray.rllib.agents import dqn - return dqn.DQNTrainer - - -def _import_simple_q(): - from ray.rllib.agents import dqn - return dqn.SimpleQTrainer - - -def _import_apex(): - from ray.rllib.agents import dqn - return dqn.ApexTrainer +def _import_a2c(): + from ray.rllib.agents import a3c + return a3c.A2CTrainer def _import_a3c(): @@ -75,34 +15,44 @@ def _import_a3c(): return a3c.A3CTrainer -def _import_a2c(): - from ray.rllib.agents import a3c - return a3c.A2CTrainer +def _import_apex(): + from ray.rllib.agents import dqn + return dqn.ApexTrainer -def _import_pg(): - from ray.rllib.agents import pg - return pg.PGTrainer +def _import_apex_ddpg(): + from ray.rllib.agents import ddpg + return ddpg.ApexDDPGTrainer -def _import_impala(): - from ray.rllib.agents import impala - return impala.ImpalaTrainer +def _import_appo(): + from ray.rllib.agents import ppo + return ppo.APPOTrainer -def _import_marwil(): +def _import_ars(): + from ray.rllib.agents import ars + return ars.ARSTrainer + + +def _import_bc(): from ray.rllib.agents import marwil - return marwil.MARWILTrainer + return marwil.BCTrainer -def _import_maml(): - from ray.rllib.agents import maml - return maml.MAMLTrainer +def _import_ddpg(): + from ray.rllib.agents import ddpg + return ddpg.DDPGTrainer -def _import_mbmpo(): - from ray.rllib.agents import mbmpo - return mbmpo.MBMPOTrainer +def _import_ddppo(): + from ray.rllib.agents import ppo + return ppo.DDPPOTrainer + + +def _import_dqn(): + from ray.rllib.agents import dqn + return dqn.DQNTrainer def _import_dreamer(): @@ -110,28 +60,84 @@ def _import_dreamer(): return dreamer.DREAMERTrainer +def _import_es(): + from ray.rllib.agents import es + return es.ESTrainer + + +def _import_impala(): + from ray.rllib.agents import impala + return impala.ImpalaTrainer + + +def _import_maml(): + from ray.rllib.agents import maml + return maml.MAMLTrainer + + +def _import_marwil(): + from ray.rllib.agents import marwil + return marwil.MARWILTrainer + + +def _import_mbmpo(): + from ray.rllib.agents import mbmpo + return mbmpo.MBMPOTrainer + + +def _import_pg(): + from ray.rllib.agents import pg + return pg.PGTrainer + + +def _import_ppo(): + from ray.rllib.agents import ppo + return ppo.PPOTrainer + + +def _import_qmix(): + from ray.rllib.agents import qmix + return qmix.QMixTrainer + + +def _import_sac(): + from ray.rllib.agents import sac + return sac.SACTrainer + + +def _import_simple_q(): + from ray.rllib.agents import dqn + return dqn.SimpleQTrainer + + +def _import_td3(): + from ray.rllib.agents import ddpg + return ddpg.TD3Trainer + + ALGORITHMS = { - "SAC": _import_sac, - "DDPG": _import_ddpg, - "APEX_DDPG": _import_apex_ddpg, - "TD3": _import_td3, - "PPO": _import_ppo, - "ES": _import_es, - "ARS": _import_ars, - "DQN": _import_dqn, - "SimpleQ": _import_simple_q, - "APEX": _import_apex, - "A3C": _import_a3c, "A2C": _import_a2c, - "PG": _import_pg, - "IMPALA": _import_impala, - "QMIX": _import_qmix, + "A3C": _import_a3c, + "APEX": _import_apex, + "APEX_DDPG": _import_apex_ddpg, "APPO": _import_appo, + "ARS": _import_ars, + "BC": _import_bc, + "ES": _import_es, + "DDPG": _import_ddpg, "DDPPO": _import_ddppo, - "MARWIL": _import_marwil, - "MAML": _import_maml, - "MBMPO": _import_mbmpo, + "DQN": _import_dqn, "DREAMER": _import_dreamer, + "IMPALA": _import_impala, + "MAML": _import_maml, + "MARWIL": _import_marwil, + "MBMPO": _import_mbmpo, + "PG": _import_pg, + "PPO": _import_ppo, + "QMIX": _import_qmix, + "SAC": _import_sac, + "SimpleQ": _import_simple_q, + "TD3": _import_td3, } diff --git a/rllib/agents/trainer.py b/rllib/agents/trainer.py index 40107668c..92e252e52 100644 --- a/rllib/agents/trainer.py +++ b/rllib/agents/trainer.py @@ -1160,6 +1160,13 @@ class Trainer(Trainable): if "optimizer" in state: self.optimizer.restore(state["optimizer"]) + @staticmethod + def with_updates(**overrides) -> Type["Trainer"]: + raise NotImplementedError( + "`with_updates` may only be called on Trainer sub-classes " + "that were generated via the `ray.rllib.agents.trainer_template." + "build_trainer()` function!") + def _register_if_needed(self, env_object: Union[str, EnvType]): if isinstance(env_object, str): return env_object diff --git a/rllib/agents/trainer_template.py b/rllib/agents/trainer_template.py index 1461d46b9..2241621d9 100644 --- a/rllib/agents/trainer_template.py +++ b/rllib/agents/trainer_template.py @@ -139,27 +139,30 @@ def build_trainer( if before_evaluate_fn: before_evaluate_fn(self) + @override(Trainer) def __getstate__(self): state = Trainer.__getstate__(self) state["train_exec_impl"] = ( self.train_exec_impl.shared_metrics.get().save()) return state + @override(Trainer) def __setstate__(self, state): Trainer.__setstate__(self, state) self.train_exec_impl.shared_metrics.get().restore( state["train_exec_impl"]) - def with_updates(**overrides): - """Build a copy of this trainer with the specified overrides. + @staticmethod + @override(Trainer) + def with_updates(**overrides) -> Type[Trainer]: + """Build a copy of this trainer with the specified overrides. - Arguments: - overrides (dict): use this to override any of the arguments - originally passed to build_trainer() for this policy. - """ - return build_trainer(**dict(original_kwargs, **overrides)) + Keyword Args: + overrides (dict): use this to override any of the arguments + originally passed to build_trainer() for this policy. + """ + return build_trainer(**dict(original_kwargs, **overrides)) - trainer_cls.with_updates = staticmethod(with_updates) trainer_cls.__name__ = name trainer_cls.__qualname__ = name return trainer_cls diff --git a/rllib/evaluation/sampler.py b/rllib/evaluation/sampler.py index be957e1fe..b3755d8e3 100644 --- a/rllib/evaluation/sampler.py +++ b/rllib/evaluation/sampler.py @@ -1093,7 +1093,10 @@ def _process_observations_w_trajectory_view_api( # Invoke the step callback after the step is logged to the episode callbacks.on_episode_step( - worker=worker, base_env=base_env, episode=episode) + worker=worker, + base_env=base_env, + episode=episode, + env_index=env_id) # Cut the batch if ... # - all-agents-done and not packing multiple episodes into one diff --git a/rllib/examples/unity3d_env_local.py b/rllib/examples/unity3d_env_local.py index 60eace4a8..1c4c7c45b 100644 --- a/rllib/examples/unity3d_env_local.py +++ b/rllib/examples/unity3d_env_local.py @@ -117,18 +117,18 @@ if __name__ == "__main__": config["exploration_config"] = { "type": "Curiosity", "eta": 0.1, - "lr": tune.grid_search([0.0003, 0.001]), + "lr": 0.001, # No actual feature net: map directly from observations to feature # vector (linearly). "feature_net_config": { - "fcnet_hiddens": tune.grid_search([[], [256]]), + "fcnet_hiddens": [], "fcnet_activation": "relu", }, "sub_exploration": { "type": "StochasticSampling", }, - "forward_net_activation": tune.grid_search(["relu", "swish"]), - "inverse_net_activation": tune.grid_search(["relu", "swish"]), + "forward_net_activation": "relu", + "inverse_net_activation": "relu", } stop = { diff --git a/rllib/offline/json_reader.py b/rllib/offline/json_reader.py index 1229bdd07..e6315f561 100644 --- a/rllib/offline/json_reader.py +++ b/rllib/offline/json_reader.py @@ -93,7 +93,7 @@ class JsonReader(InputReader): return SampleBatch.concat_samples(out) else: # TODO(ekl) this is trickier since the alignments between agent - # trajectories in the episode are not available any more. + # trajectories in the episode are not available any more. raise NotImplementedError( "Postprocessing of multi-agent data not implemented yet.") diff --git a/rllib/tuned_examples/marwil/cartpole-bc.yaml b/rllib/tuned_examples/marwil/cartpole-bc.yaml new file mode 100644 index 000000000..c0c0af0da --- /dev/null +++ b/rllib/tuned_examples/marwil/cartpole-bc.yaml @@ -0,0 +1,20 @@ +# To generate training data, first run: +# $ ./train.py --run=PPO --env=CartPole-v0 \ +# --stop='{"timesteps_total": 50000}' \ +# --config='{"output": "/tmp/out", "batch_mode": "complete_episodes"}' +cartpole-bc: + env: CartPole-v0 + run: BC + stop: + timesteps_total: 500000 + config: + # Works for both torch and tf. + framework: tf + # In order to evaluate on an actual environment, use these following + # settings: + evaluation_num_workers: 1 + evaluation_interval: 1 + evaluation_config: + input: sampler + # The historic (offline) data file from the PPO run (at the top). + input: /tmp/out diff --git a/rllib/tuned_examples/marwil/cartpole-marwil.yaml b/rllib/tuned_examples/marwil/cartpole-marwil.yaml index 6e9643778..06759ef95 100644 --- a/rllib/tuned_examples/marwil/cartpole-marwil.yaml +++ b/rllib/tuned_examples/marwil/cartpole-marwil.yaml @@ -16,8 +16,6 @@ cartpole-marwil: evaluation_interval: 1 evaluation_config: input: sampler - # Compare IL (beta=0) vs MARWIL. - beta: - grid_search: [0, 1] + beta: 1.0 # Compare to behavior cloning (beta=0.0). # The historic (offline) data file from the PPO run (at the top). input: /tmp/out