wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-12 12:20:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[RLlib] Behavioral Cloning (from MARWIL). (#10619)

Browse Source
This commit is contained in:
Sven Mika
2020-09-09 17:33:21 +02:00
committed by GitHub
parent afa0216280
commit 4b278c36fc
15 changed files with 301 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/rllib-algorithms.rst
+32 -3
View File
@@ -13,6 +13,7 @@ Algorithm Frameworks Discrete Actions Continuous Actions Multi-
=================== ========== ======================= ================== =========== =============================================================
`A2C, A3C`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_, `+LSTM auto-wrapping`_, `+Transformer`_, `+autoreg`_
`ARS`_ tf + torch **Yes** **Yes** No
`BC`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_
`ES`_ tf + torch **Yes** **Yes** No
`DDPG`_, `TD3`_ tf + torch No **Yes** **Yes**
`APEX-DDPG`_ tf + torch No **Yes** **Yes**
@@ -547,10 +548,15 @@ Tuned examples: `Humanoid-v1 <https://github.com/ray-project/ray/blob/master/rll
.. _marwil:
Advantage Re-Weighted Imitation Learning (MARWIL)
-------------------------------------------------
Monotonic Advantage Re-Weighted Imitation Learning (MARWIL)
-----------------------------------------------------------
|pytorch| |tensorflow|
`[paper] <http://papers.nips.cc/paper/7866-exponentially-weighted-imitation-learning-for-batched-historical-data>`__ `[implementation] <https://github.com/ray-project/ray/blob/master/rllib/agents/marwil/marwil.py>`__ MARWIL is a hybrid imitation learning and policy gradient algorithm suitable for training on batched historical data. When the ``beta`` hyperparameter is set to zero, the MARWIL objective reduces to vanilla imitation learning. MARWIL requires the `offline datasets API <rllib-offline.html>`__ to be used.
`[paper] <http://papers.nips.cc/paper/7866-exponentially-weighted-imitation-learning-for-batched-historical-data>`__
`[implementation] <https://github.com/ray-project/ray/blob/master/rllib/agents/marwil/marwil.py>`__
MARWIL is a hybrid imitation learning and policy gradient algorithm suitable for training on batched historical data.
When the ``beta`` hyperparameter is set to zero, the MARWIL objective reduces to vanilla imitation learning (see `BC`_).
MARWIL requires the `offline datasets API <rllib-offline.html>`__ to be used.
Tuned examples: `CartPole-v0 <https://github.com/ray-project/ray/blob/master/rllib/tuned_examples/marwil/cartpole-marwil.yaml>`__
@@ -562,6 +568,29 @@ Tuned examples: `CartPole-v0 <https://github.com/ray-project/ray/blob/master/rll
:end-before: __sphinx_doc_end__
.. _bc:
Behavior Cloning (BC; derived from MARWIL implementation)
---------------------------------------------------------
|pytorch| |tensorflow|
`[paper] <http://papers.nips.cc/paper/7866-exponentially-weighted-imitation-learning-for-batched-historical-data>`__
`[implementation] <https://github.com/ray-project/ray/blob/master/rllib/agents/marwil/bc.py>`__
Our behavioral cloning implementation is directly derived from our `MARWIL`_ implementation,
with the only difference being the ``beta`` parameter force-set to 0.0. This makes
BC try to match the behavior policy, which generated the offline data, disregarding any resulting rewards.
BC requires the `offline datasets API <rllib-offline.html>`__ to be used.
Tuned examples: `CartPole-v0 <https://github.com/ray-project/ray/blob/master/rllib/tuned_examples/marwil/cartpole-bc.yaml>`__
**BC-specific configs** (see also `common configs <rllib-training.html#common-parameters>`__):
.. literalinclude:: ../../rllib/agents/marwil/bc.py
:language: python
:start-after: __sphinx_doc_begin__
:end-before: __sphinx_doc_end__
Contextual Bandits (contrib/bandits)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
rllib/BUILD
+10
View File
@@ -468,6 +468,16 @@ py_test(
srcs = ["agents/marwil/tests/test_marwil.py"]
)
# BCTrainer (sub-type of MARWIL)
py_test(
name = "test_bc",
tags = ["agents_dir"],
size = "medium",
# Include the json data file.
data = ["tests/data/cartpole/large.json"],
srcs = ["agents/marwil/tests/test_bc.py"]
)
# MAMLTrainer
py_test(
name = "test_maml",
rllib/agents/marwil/__init__.py
+3
View File
@@ -1,8 +1,11 @@
from ray.rllib.agents.marwil.bc import BCTrainer, BC_DEFAULT_CONFIG
from ray.rllib.agents.marwil.marwil import MARWILTrainer, DEFAULT_CONFIG
from ray.rllib.agents.marwil.marwil_tf_policy import MARWILTFPolicy
from ray.rllib.agents.marwil.marwil_torch_policy import MARWILTorchPolicy
__all__ = [
"BCTrainer",
"BC_DEFAULT_CONFIG",
"DEFAULT_CONFIG",
"MARWILTFPolicy",
"MARWILTorchPolicy",
rllib/agents/marwil/bc.py
+29
View File
@@ -0,0 +1,29 @@
"""Behavioral Cloning (derived from MARWIL).
Simply uses the MARWIL agent with beta force-set to 0.0.
"""
from ray.rllib.agents.marwil.marwil import MARWILTrainer, \
DEFAULT_CONFIG as MARWIL_CONFIG
from ray.rllib.utils.typing import TrainerConfigDict
# yapf: disable
# __sphinx_doc_begin__
BC_DEFAULT_CONFIG = MARWILTrainer.merge_trainer_configs(
MARWIL_CONFIG, {
"beta": 0.0,
})
# __sphinx_doc_end__
# yapf: enable
def validate_config(config: TrainerConfigDict):
if config["beta"] != 0.0:
raise ValueError(
"For behavioral cloning, `beta` parameter must be 0.0!")
BCTrainer = MARWILTrainer.with_updates(
name="BC",
default_config=BC_DEFAULT_CONFIG,
validate_config=validate_config,
)
rllib/agents/marwil/marwil.py
+9 -11
View File
@@ -16,22 +16,22 @@ DEFAULT_CONFIG = with_common_config({
# Use importance sampling estimators for reward
"input_evaluation": ["is", "wis"],
# Scaling of advantages in exponential terms
# When beta is 0, MARWIL is reduced to imitation learning
# Scaling of advantages in exponential terms.
# When beta is 0.0, MARWIL is reduced to imitation learning.
"beta": 1.0,
# Balancing value estimation loss and policy optimization loss
# Balancing value estimation loss and policy optimization loss.
"vf_coeff": 1.0,
# Whether to calculate cumulative rewards
# Whether to calculate cumulative rewards.
"postprocess_inputs": True,
# Whether to rollout "complete_episodes" or "truncate_episodes"
# Whether to rollout "complete_episodes" or "truncate_episodes".
"batch_mode": "complete_episodes",
# Learning rate for adam optimizer
# Learning rate for adam optimizer.
"lr": 1e-4,
# Number of timesteps collected for each SGD round
# Number of timesteps collected for each SGD round.
"train_batch_size": 2000,
# Number of steps max to keep in the batch replay buffer
# Number of steps max to keep in the batch replay buffer.
"replay_buffer_size": 100000,
# Number of steps to read before learning starts
# Number of steps to read before learning starts.
"learning_starts": 0,
# === Parallelism ===
"num_workers": 0,
@@ -45,8 +45,6 @@ def get_policy_class(config):
from ray.rllib.agents.marwil.marwil_torch_policy import \
MARWILTorchPolicy
return MARWILTorchPolicy
else:
return MARWILTFPolicy
def execution_plan(workers, config):
rllib/agents/marwil/tests/test_bc.py
+65
View File
@@ -0,0 +1,65 @@
import os
from pathlib import Path
import unittest
import ray
import ray.rllib.agents.marwil as marwil
from ray.rllib.utils.framework import try_import_tf
from ray.rllib.utils.test_utils import check_compute_single_action, \
framework_iterator
tf1, tf, tfv = try_import_tf()
class TestBC(unittest.TestCase):
@classmethod
def setUpClass(cls):
ray.init()
@classmethod
def tearDownClass(cls):
ray.shutdown()
def test_bc_compilation_and_learning_from_offline_file(self):
"""Test whether a BCTrainer can be built with all frameworks.
And learns from a historic-data file.
"""
rllib_dir = Path(__file__).parent.parent.parent.parent
print("rllib dir={}".format(rllib_dir))
data_file = os.path.join(rllib_dir, "tests/data/cartpole/large.json")
print("data_file={} exists={}".format(data_file,
os.path.isfile(data_file)))
config = marwil.BC_DEFAULT_CONFIG.copy()
config["num_workers"] = 0 # Run locally.
config["evaluation_num_workers"] = 1
config["evaluation_interval"] = 1
# Evaluate on actual environment.
config["evaluation_config"] = {"input": "sampler"}
# Learn from offline data.
config["input"] = [data_file]
num_iterations = 300
# Test for all frameworks.
for _ in framework_iterator(config, frameworks=("tf", "torch")):
trainer = marwil.BCTrainer(config=config, env="CartPole-v0")
for i in range(num_iterations):
eval_results = trainer.train()["evaluation"]
print("iter={} R={}".format(
i, eval_results["episode_reward_mean"]))
# Learn until some reward is reached on an actual live env.
if eval_results["episode_reward_mean"] > 60.0:
print("learnt!")
break
check_compute_single_action(
trainer, include_prev_action_reward=True)
trainer.stop()
if __name__ == "__main__":
import pytest
import sys
sys.exit(pytest.main(["-v", __file__]))
rllib/agents/marwil/tests/test_marwil.py
+2
View File
@@ -35,7 +35,9 @@ class TestMARWIL(unittest.TestCase):
config["num_workers"] = 0 # Run locally.
config["evaluation_num_workers"] = 1
config["evaluation_interval"] = 1
# Evaluate on actual environment.
config["evaluation_config"] = {"input": "sampler"}
# Learn from offline data.
config["input"] = [data_file]
num_iterations = 300
rllib/agents/registry.py
+103 -97
View File
@@ -5,69 +5,9 @@ import traceback
from ray.rllib.contrib.registry import CONTRIBUTED_ALGORITHMS
def _import_sac():
from ray.rllib.agents import sac
return sac.SACTrainer
def _import_appo():
from ray.rllib.agents import ppo
return ppo.APPOTrainer
def _import_ddppo():
from ray.rllib.agents import ppo
return ppo.DDPPOTrainer
def _import_qmix():
from ray.rllib.agents import qmix
return qmix.QMixTrainer
def _import_ddpg():
from ray.rllib.agents import ddpg
return ddpg.DDPGTrainer
def _import_apex_ddpg():
from ray.rllib.agents import ddpg
return ddpg.ApexDDPGTrainer
def _import_td3():
from ray.rllib.agents import ddpg
return ddpg.TD3Trainer
def _import_ppo():
from ray.rllib.agents import ppo
return ppo.PPOTrainer
def _import_es():
from ray.rllib.agents import es
return es.ESTrainer
def _import_ars():
from ray.rllib.agents import ars
return ars.ARSTrainer
def _import_dqn():
from ray.rllib.agents import dqn
return dqn.DQNTrainer
def _import_simple_q():
from ray.rllib.agents import dqn
return dqn.SimpleQTrainer
def _import_apex():
from ray.rllib.agents import dqn
return dqn.ApexTrainer
def _import_a2c():
from ray.rllib.agents import a3c
return a3c.A2CTrainer
def _import_a3c():
@@ -75,34 +15,44 @@ def _import_a3c():
return a3c.A3CTrainer
def _import_a2c():
from ray.rllib.agents import a3c
return a3c.A2CTrainer
def _import_apex():
from ray.rllib.agents import dqn
return dqn.ApexTrainer
def _import_pg():
from ray.rllib.agents import pg
return pg.PGTrainer
def _import_apex_ddpg():
from ray.rllib.agents import ddpg
return ddpg.ApexDDPGTrainer
def _import_impala():
from ray.rllib.agents import impala
return impala.ImpalaTrainer
def _import_appo():
from ray.rllib.agents import ppo
return ppo.APPOTrainer
def _import_marwil():
def _import_ars():
from ray.rllib.agents import ars
return ars.ARSTrainer
def _import_bc():
from ray.rllib.agents import marwil
return marwil.MARWILTrainer
return marwil.BCTrainer
def _import_maml():
from ray.rllib.agents import maml
return maml.MAMLTrainer
def _import_ddpg():
from ray.rllib.agents import ddpg
return ddpg.DDPGTrainer
def _import_mbmpo():
from ray.rllib.agents import mbmpo
return mbmpo.MBMPOTrainer
def _import_ddppo():
from ray.rllib.agents import ppo
return ppo.DDPPOTrainer
def _import_dqn():
from ray.rllib.agents import dqn
return dqn.DQNTrainer
def _import_dreamer():
@@ -110,28 +60,84 @@ def _import_dreamer():
return dreamer.DREAMERTrainer
def _import_es():
from ray.rllib.agents import es
return es.ESTrainer
def _import_impala():
from ray.rllib.agents import impala
return impala.ImpalaTrainer
def _import_maml():
from ray.rllib.agents import maml
return maml.MAMLTrainer
def _import_marwil():
from ray.rllib.agents import marwil
return marwil.MARWILTrainer
def _import_mbmpo():
from ray.rllib.agents import mbmpo
return mbmpo.MBMPOTrainer
def _import_pg():
from ray.rllib.agents import pg
return pg.PGTrainer
def _import_ppo():
from ray.rllib.agents import ppo
return ppo.PPOTrainer
def _import_qmix():
from ray.rllib.agents import qmix
return qmix.QMixTrainer
def _import_sac():
from ray.rllib.agents import sac
return sac.SACTrainer
def _import_simple_q():
from ray.rllib.agents import dqn
return dqn.SimpleQTrainer
def _import_td3():
from ray.rllib.agents import ddpg
return ddpg.TD3Trainer
ALGORITHMS = {
"SAC": _import_sac,
"DDPG": _import_ddpg,
"APEX_DDPG": _import_apex_ddpg,
"TD3": _import_td3,
"PPO": _import_ppo,
"ES": _import_es,
"ARS": _import_ars,
"DQN": _import_dqn,
"SimpleQ": _import_simple_q,
"APEX": _import_apex,
"A3C": _import_a3c,
"A2C": _import_a2c,
"PG": _import_pg,
"IMPALA": _import_impala,
"QMIX": _import_qmix,
"A3C": _import_a3c,
"APEX": _import_apex,
"APEX_DDPG": _import_apex_ddpg,
"APPO": _import_appo,
"ARS": _import_ars,
"BC": _import_bc,
"ES": _import_es,
"DDPG": _import_ddpg,
"DDPPO": _import_ddppo,
"MARWIL": _import_marwil,
"MAML": _import_maml,
"MBMPO": _import_mbmpo,
"DQN": _import_dqn,
"DREAMER": _import_dreamer,
"IMPALA": _import_impala,
"MAML": _import_maml,
"MARWIL": _import_marwil,
"MBMPO": _import_mbmpo,
"PG": _import_pg,
"PPO": _import_ppo,
"QMIX": _import_qmix,
"SAC": _import_sac,
"SimpleQ": _import_simple_q,
"TD3": _import_td3,
}
rllib/agents/trainer.py
+7
View File
@@ -1160,6 +1160,13 @@ class Trainer(Trainable):
if "optimizer" in state:
self.optimizer.restore(state["optimizer"])
@staticmethod
def with_updates(**overrides) -> Type["Trainer"]:
raise NotImplementedError(
"`with_updates` may only be called on Trainer sub-classes "
"that were generated via the `ray.rllib.agents.trainer_template."
"build_trainer()` function!")
def _register_if_needed(self, env_object: Union[str, EnvType]):
if isinstance(env_object, str):
return env_object
rllib/agents/trainer_template.py
+11 -8
View File
@@ -139,27 +139,30 @@ def build_trainer(
if before_evaluate_fn:
before_evaluate_fn(self)
@override(Trainer)
def __getstate__(self):
state = Trainer.__getstate__(self)
state["train_exec_impl"] = (
self.train_exec_impl.shared_metrics.get().save())
return state
@override(Trainer)
def __setstate__(self, state):
Trainer.__setstate__(self, state)
self.train_exec_impl.shared_metrics.get().restore(
state["train_exec_impl"])
def with_updates(**overrides):
"""Build a copy of this trainer with the specified overrides.
@staticmethod
@override(Trainer)
def with_updates(**overrides) -> Type[Trainer]:
"""Build a copy of this trainer with the specified overrides.
Arguments:
overrides (dict): use this to override any of the arguments
originally passed to build_trainer() for this policy.
"""
return build_trainer(**dict(original_kwargs, **overrides))
Keyword Args:
overrides (dict): use this to override any of the arguments
originally passed to build_trainer() for this policy.
"""
return build_trainer(**dict(original_kwargs, **overrides))
trainer_cls.with_updates = staticmethod(with_updates)
trainer_cls.__name__ = name
trainer_cls.__qualname__ = name
return trainer_cls
rllib/evaluation/sampler.py
+4 -1
View File
@@ -1093,7 +1093,10 @@ def _process_observations_w_trajectory_view_api(
# Invoke the step callback after the step is logged to the episode
callbacks.on_episode_step(
worker=worker, base_env=base_env, episode=episode)
worker=worker,
base_env=base_env,
episode=episode,
env_index=env_id)
# Cut the batch if ...
# - all-agents-done and not packing multiple episodes into one
rllib/examples/unity3d_env_local.py
+4 -4
View File
@@ -117,18 +117,18 @@ if __name__ == "__main__":
config["exploration_config"] = {
"type": "Curiosity",
"eta": 0.1,
"lr": tune.grid_search([0.0003, 0.001]),
"lr": 0.001,
# No actual feature net: map directly from observations to feature
# vector (linearly).
"feature_net_config": {
"fcnet_hiddens": tune.grid_search([[], [256]]),
"fcnet_hiddens": [],
"fcnet_activation": "relu",
},
"sub_exploration": {
"type": "StochasticSampling",
},
"forward_net_activation": tune.grid_search(["relu", "swish"]),
"inverse_net_activation": tune.grid_search(["relu", "swish"]),
"forward_net_activation": "relu",
"inverse_net_activation": "relu",
}
stop = {
rllib/offline/json_reader.py
+1 -1
View File
@@ -93,7 +93,7 @@ class JsonReader(InputReader):
return SampleBatch.concat_samples(out)
else:
# TODO(ekl) this is trickier since the alignments between agent
# trajectories in the episode are not available any more.
# trajectories in the episode are not available any more.
raise NotImplementedError(
"Postprocessing of multi-agent data not implemented yet.")
rllib/tuned_examples/marwil/cartpole-bc.yaml
+20
View File
@@ -0,0 +1,20 @@
# To generate training data, first run:
# $ ./train.py --run=PPO --env=CartPole-v0 \
# --stop='{"timesteps_total": 50000}' \
# --config='{"output": "/tmp/out", "batch_mode": "complete_episodes"}'
cartpole-bc:
env: CartPole-v0
run: BC
stop:
timesteps_total: 500000
config:
# Works for both torch and tf.
framework: tf
# In order to evaluate on an actual environment, use these following
# settings:
evaluation_num_workers: 1
evaluation_interval: 1
evaluation_config:
input: sampler
# The historic (offline) data file from the PPO run (at the top).
input: /tmp/out
rllib/tuned_examples/marwil/cartpole-marwil.yaml
+1 -3
View File
@@ -16,8 +16,6 @@ cartpole-marwil:
evaluation_interval: 1
evaluation_config:
input: sampler
# Compare IL (beta=0) vs MARWIL.
beta:
grid_search: [0, 1]
beta: 1.0 # Compare to behavior cloning (beta=0.0).
# The historic (offline) data file from the PPO run (at the top).
input: /tmp/out