diff --git a/doc/source/rllib-algorithms.rst b/doc/source/rllib-algorithms.rst index cfe8ee90c..32bc0a7a9 100644 --- a/doc/source/rllib-algorithms.rst +++ b/doc/source/rllib-algorithms.rst @@ -24,16 +24,26 @@ Algorithm Frameworks Discrete Actions Continuous Actions Multi- `MARWIL`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_ `PG`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_, `+LSTM auto-wrapping`_, `+Transformer`_, `+autoreg`_ `PPO`_, `APPO`_ tf + torch **Yes** `+parametric`_ **Yes** **Yes** `+RNN`_, `+LSTM auto-wrapping`_, `+Transformer`_, `+autoreg`_ -`QMIX`_ torch **Yes** `+parametric`_ No **Yes** `+RNN`_ `SAC`_ tf + torch **Yes** **Yes** **Yes** -------------------- ---------- ----------------------- ------------------ ----------- ------------------------------------------------------------- -`AlphaZero`_ torch **Yes** `+parametric`_ No No `LinUCB`_, `LinTS`_ torch **Yes** `+parametric`_ No **Yes** -`MADDPG`_ tf **Yes** Partial **Yes** +`AlphaZero`_ torch **Yes** `+parametric`_ No No =================== ========== ======================= ================== =========== ============================================================= -.. _`+autoreg`: rllib-models.html#autoregressive-action-distributions -.. _`+LSTM auto-wrapping`: rllib-models.html#built-in-models +Multi-Agent only Methods + +============================= ========== ======================= ================== =========== ===================== +Algorithm Frameworks Discrete Actions Continuous Actions Multi-Agent Model Support +============================= ========== ======================= ================== =========== ===================== +`QMIX`_ torch **Yes** `+parametric`_ No **Yes** `+RNN`_ +`MADDPG`_ tf **Yes** Partial **Yes** +`Parameter Sharing`_ Depends on bootstrapped algorithm +----------------------------- --------------------------------------------------------------------------------------- +`Fully Independent Learning`_ Depends on bootstrapped algorithm +----------------------------- --------------------------------------------------------------------------------------- +`Centralized Critic Methods`_ Depends on bootstrapped algorithm +============================= ======================================================================================= + + .. _`+parametric`: rllib-models.html#variable-length-parametric-action-spaces .. _`+RNN`: rllib-models.html#recurrent-models .. _`+Transformer`: rllib-models.html#attention-networks @@ -42,6 +52,8 @@ Algorithm Frameworks Discrete Actions Continuous Actions Multi- .. _`APEX-DDPG`: rllib-algorithms.html#apex .. _`Rainbow`: rllib-algorithms.html#dqn .. _`TD3`: rllib-algorithms.html#ddpg +.. _`+autoreg`: rllib-models.html#autoregressive-action-distributions +.. _`+LSTM auto-wrapping`: rllib-models.html#built-in-models High-throughput architectures ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -439,7 +451,7 @@ Dreamer Dreamer is an image-only model-based RL method that learns by imagining trajectories in the future and is evaluated on the DeepMind Control Suite `environments `__. RLlib's Dreamer is adapted from the `official Google research repo `__. -To visualize learning, RLLib Dreamer's imagined trajectories are logged as gifs in Tensorboard. Examples of such can be seen `here `__. +To visualize learning, RLLib Dreamer's imagined trajectories are logged as gifs in Tensorboard. Examples of such can be seen `here `__. Tuned examples: `Deepmind Control Environments `__ @@ -503,38 +515,6 @@ Tuned examples: `Humanoid-v1 `__ `[implementation] `__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping `__ in the environment (see the `two-step game example `__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X. - -Tuned examples: `Two-step game `__ - -**QMIX-specific configs** (see also `common configs `__): - -.. literalinclude:: ../../rllib/agents/qmix/qmix.py - :language: python - :start-after: __sphinx_doc_begin__ - :end-before: __sphinx_doc_end__ - -.. _maddpg: - -Multi-Agent Deep Deterministic Policy Gradient (contrib/MADDPG) ---------------------------------------------------------------- -|tensorflow| -`[paper] `__ `[implementation] `__ MADDPG is a DDPG centralized critic algorithm. Code here is adapted from https://github.com/openai/maddpg to integrate with RLlib multi-agent APIs. Please check `justinkterry/maddpg-rllib `__ for examples and more information. Note that the implementation here is based on OpenAI's, and is intended for use with the discrete MPE environments. Please also note that people typically find this method difficult to get to work, even with all applicable optimizations for their environment applied. This method should be viewed as for research purposes, and for reproducing the results of the paper introducing it. - -**MADDPG-specific configs** (see also `common configs `__): - -Tuned examples: `Multi-Agent Particle Environment `__, `Two-step game `__ - -.. literalinclude:: ../../rllib/contrib/maddpg/maddpg.py - :language: python - :start-after: __sphinx_doc_begin__ - :end-before: __sphinx_doc_end__ - .. _marwil: Advantage Re-Weighted Imitation Learning (MARWIL) @@ -551,22 +531,6 @@ Tuned examples: `CartPole-v0 `__ `[implementation] `__ AlphaZero is an RL agent originally designed for two-player games. This version adapts it to handle single player games. The code can be sscaled to any number of workers. It also implements the ranked rewards `(R2) `__ strategy to enable self-play even in the one-player setting. The code is mainly purposed to be used for combinatorial optimization. - -Tuned examples: `CartPole-v0 `__ - -**AlphaZero-specific configs** (see also `common configs `__): - -.. literalinclude:: ../../rllib/contrib/alpha_zero/core/alpha_zero_trainer.py - :language: python - :start-after: __sphinx_doc_begin__ - :end-before: __sphinx_doc_end__ - Contextual Bandits (contrib/bandits) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -640,3 +604,74 @@ Tuned examples: `SimpleContextualBandit `__ `[implementation] `__ AlphaZero is an RL agent originally designed for two-player games. This version adapts it to handle single player games. The code can be sscaled to any number of workers. It also implements the ranked rewards `(R2) `__ strategy to enable self-play even in the one-player setting. The code is mainly purposed to be used for combinatorial optimization. + +Tuned examples: `CartPole-v0 `__ + +**AlphaZero-specific configs** (see also `common configs `__): + +.. literalinclude:: ../../rllib/contrib/alpha_zero/core/alpha_zero_trainer.py + :language: python + :start-after: __sphinx_doc_begin__ + + +Multi-Agent Methods +------------------- + +.. _qmix: + +QMIX Monotonic Value Factorisation (QMIX, VDN, IQN) +--------------------------------------------------- +|pytorch| +`[paper] `__ `[implementation] `__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping `__ in the environment (see the `two-step game example `__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X. + +Tuned examples: `Two-step game `__ + +**QMIX-specific configs** (see also `common configs `__): + +.. literalinclude:: ../../rllib/agents/qmix/qmix.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + +.. _maddpg: + +Multi-Agent Deep Deterministic Policy Gradient (contrib/MADDPG) +--------------------------------------------------------------- +|tensorflow| +`[paper] `__ `[implementation] `__ MADDPG is a DDPG centralized critic algorithm. Code here is adapted from https://github.com/openai/maddpg to integrate with RLlib multi-agent APIs. Please check `justinkterry/maddpg-rllib `__ for examples and more information. Note that the implementation here is based on OpenAI's, and is intended for use with the discrete MPE environments. Please also note that people typically find this method difficult to get to work, even with all applicable optimizations for their environment applied. This method should be viewed as for research purposes, and for reproducing the results of the paper introducing it. + +**MADDPG-specific configs** (see also `common configs `__): + +Tuned examples: `Multi-Agent Particle Environment `__, `Two-step game `__ + +.. literalinclude:: ../../rllib/contrib/maddpg/maddpg.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + +Parameter Sharing +----------------- + +`[paper] `__, `[paper] `__ and `[instructions] `__. Parameter sharing refers to a class of methods that take a base single agent method, and use it to learn a single policy for all agents. This simple approach has been shown to achieve state of the art performance in cooperative games, and is usually how you should start trying to learn a multi-agent problem. + +Tuned examples: `PettingZoo `__, `waterworld `__, `rock-paper-scissors `__, `multi-agent cartpole `__ + +Fully Independent Learning +-------------------------- +`[instructions] `__ Fully independent learning involves a collection of agents learning independently of each other via single agent methods. This typically works, but can be less effective than dedicated multi-agent RL methods, since they do not account for the non-stationarity of the multi-agent environment. + +Tuned examples: `waterworld `__, `multiagent-cartpole `__ + +Centralized Critic Methods +-------------------------- + +`[instructions] `__Centralized critic methods are when all agents use a single parameter shared critic network (in some cases with access to more of the observation space than agents can see). Note that many specialized multi-agent algorithms such as MADDPG are essentially centralized critic forms of their single-agent algorithm (DDPG in the case of MADDPG). +Tuned examples: `TwoStepGame `__ diff --git a/rllib/examples/multi_agent_independent_learning.py b/rllib/examples/multi_agent_independent_learning.py new file mode 100644 index 000000000..82248f17c --- /dev/null +++ b/rllib/examples/multi_agent_independent_learning.py @@ -0,0 +1,38 @@ +from ray import tune +from ray.tune.registry import register_env +from ray.rllib.env.pettingzoo_env import PettingZooEnv +from pettingzoo.sisl import waterworld_v0 + +# Based on code from github.com/parametersharingmadrl/parametersharingmadrl + +if __name__ == "__main__": + # RDQN - Rainbow DQN + # ADQN - Apex DQN + def env_creator(args): + return PettingZooEnv(waterworld_v0.env()) + + env = env_creator({}) + register_env("waterworld", env_creator) + + obs_space = env.observation_space + act_spc = env.action_space + + policies = {agent: (None, obs_space, act_spc, {}) for agent in env.agents} + + tune.run( + "APEX_DDPG", + stop={"episodes_total": 60000}, + checkpoint_freq=10, + config={ + # Enviroment specific + "env": "waterworld", + # General + "num_gpus": 1, + "num_workers": 2, + # Method specific + "multiagent": { + "policies": policies, + "policy_mapping_fn": (lambda agent_id: agent_id), + }, + }, + ) diff --git a/rllib/examples/multi_agent_parameter_sharing.py b/rllib/examples/multi_agent_parameter_sharing.py new file mode 100644 index 000000000..aabe88f68 --- /dev/null +++ b/rllib/examples/multi_agent_parameter_sharing.py @@ -0,0 +1,57 @@ +from ray import tune +from ray.tune.registry import register_env +from ray.rllib.env.pettingzoo_env import PettingZooEnv +from pettingzoo.sisl import waterworld_v0 + +# Based on code from github.com/parametersharingmadrl/parametersharingmadrl + +if __name__ == "__main__": + # RDQN - Rainbow DQN + # ADQN - Apex DQN + def env_creator(args): + return PettingZooEnv(waterworld_v0.env()) + + env = env_creator({}) + register_env("waterworld", env_creator) + + obs_space = env.observation_space + act_space = env.action_space + + policies = {"shared_policy": (None, obs_space, act_space, {})} + + # for all methods + policy_ids = list(policies.keys()) + + tune.run( + "APEX_DDPG", + stop={"episodes_total": 60000}, + checkpoint_freq=10, + config={ + + # Enviroment specific + "env": "waterworld", + + # General + "num_gpus": 1, + "num_workers": 2, + "num_envs_per_worker": 8, + "learning_starts": 1000, + "buffer_size": int(1e5), + "compress_observations": True, + "sample_batch_size": 20, + "train_batch_size": 512, + "gamma": .99, + "n_step": 3, + "lr": .0001, + "prioritized_replay_alpha": 0.5, + "final_prioritized_replay_beta": 1.0, + "target_network_update_freq": 50000, + "timesteps_per_iteration": 25000, + + # Method specific + "multiagent": { + "policies": policies, + "policy_mapping_fn": (lambda agent_id: "shared_policy"), + }, + }, + )