diff --git a/doc/source/rllib-algorithms.rst b/doc/source/rllib-algorithms.rst index b30d52259..127e8d474 100644 --- a/doc/source/rllib-algorithms.rst +++ b/doc/source/rllib-algorithms.rst @@ -476,9 +476,9 @@ Tuned examples: `Humanoid-v1 `__ `[implementation] `__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping `__ in the environment (see the `two-step game example `__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X. +`[paper] `__ `[implementation] `__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping `__ in the environment (see the `two-step game example `__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X. -Tuned examples: `Two-step game `__ +Tuned examples: `Two-step game `__ **QMIX-specific configs** (see also `common configs `__): @@ -496,7 +496,7 @@ Multi-Agent Deep Deterministic Policy Gradient (contrib/MADDPG) **MADDPG-specific configs** (see also `common configs `__): -Tuned examples: `Multi-Agent Particle Environment `__, `Two-step game `__ +Tuned examples: `Multi-Agent Particle Environment `__, `Two-step game `__ .. literalinclude:: ../../rllib/contrib/maddpg/maddpg.py :language: python diff --git a/doc/source/rllib-examples.rst b/doc/source/rllib-examples.rst index 487bc8be4..db07c3542 100644 --- a/doc/source/rllib-examples.rst +++ b/doc/source/rllib-examples.rst @@ -80,7 +80,7 @@ Multi-Agent and Hierarchical - `Rock-paper-scissors `__: Example of different heuristic and learned policies competing against each other in rock-paper-scissors. -- `Two-step game `__: +- `Two-step game `__: Example of the two-step game from the `QMIX paper `__. - `PPO with centralized critic on two-step game `__: Example of customizing PPO to leverage a centralized value function. diff --git a/rllib/BUILD b/rllib/BUILD index cfc050b4d..43f7cbb57 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -1943,39 +1943,39 @@ py_test( ) py_test( - name = "examples/twostep_game_maddpg", - main = "examples/twostep_game.py", + name = "examples/two_step_game_maddpg", + main = "examples/two_step_game.py", tags = ["examples", "examples_T"], size = "large", - srcs = ["examples/twostep_game.py"], + srcs = ["examples/two_step_game.py"], args = ["--stop-timesteps=2000", "--run=contrib/MADDPG"] ) py_test( - name = "examples/twostep_game_pg_tf", - main = "examples/twostep_game.py", + name = "examples/two_step_game_pg_tf", + main = "examples/two_step_game.py", tags = ["examples", "examples_T"], size = "medium", - srcs = ["examples/twostep_game.py"], + srcs = ["examples/two_step_game.py"], args = ["--as-test", "--stop-reward=7", "--run=PG"] ) py_test( - name = "examples/twostep_game_pg_torch", - main = "examples/twostep_game.py", + name = "examples/two_step_game_pg_torch", + main = "examples/two_step_game.py", tags = ["examples", "examples_T"], size = "medium", - srcs = ["examples/twostep_game.py"], + srcs = ["examples/two_step_game.py"], args = ["--as-test", "--torch", "--stop-reward=7", "--run=PG"] ) py_test( - name = "examples/twostep_game_qmix", - main = "examples/twostep_game.py", + name = "examples/two_step_game_qmix", + main = "examples/two_step_game.py", tags = ["examples", "examples_T"], size = "medium", - srcs = ["examples/twostep_game.py"], - args = ["--stop-timesteps=2000", "--run=QMIX"] + srcs = ["examples/two_step_game.py"], + args = ["--as-test", "--torch", "--stop-reward=7", "--run=QMIX"] ) py_test( diff --git a/rllib/agents/dqn/dqn.py b/rllib/agents/dqn/dqn.py index e94b4e2b6..476500bda 100644 --- a/rllib/agents/dqn/dqn.py +++ b/rllib/agents/dqn/dqn.py @@ -40,7 +40,7 @@ DEFAULT_CONFIG = with_common_config({ # N-step Q learning "n_step": 1, - # === Exploration Settings (Experimental) === + # === Exploration Settings === "exploration_config": { # The Exploration class to use. "type": "EpsilonGreedy", diff --git a/rllib/agents/dqn/dqn_torch_policy.py b/rllib/agents/dqn/dqn_torch_policy.py index f9d00f82d..678192023 100644 --- a/rllib/agents/dqn/dqn_torch_policy.py +++ b/rllib/agents/dqn/dqn_torch_policy.py @@ -169,7 +169,9 @@ def build_q_losses(policy, model, _, train_batch): # q scores for actions which we know were selected in the given state. one_hot_selection = F.one_hot(train_batch[SampleBatch.ACTIONS], policy.action_space.n) - q_t_selected = torch.sum(q_t * one_hot_selection, 1) + q_t_selected = torch.sum( + torch.where(q_t > -float("inf"), q_t, torch.tensor(0.0)) * + one_hot_selection, 1) # compute estimate of best possible value starting from state at t + 1 if config["double_q"]: @@ -182,11 +184,13 @@ def build_q_losses(policy, model, _, train_batch): q_tp1_best_using_online_net = torch.argmax(q_tp1_using_online_net, 1) q_tp1_best_one_hot_selection = F.one_hot(q_tp1_best_using_online_net, policy.action_space.n) - q_tp1_best = torch.sum(q_tp1 * q_tp1_best_one_hot_selection, 1) else: q_tp1_best_one_hot_selection = F.one_hot( torch.argmax(q_tp1, 1), policy.action_space.n) - q_tp1_best = torch.sum(q_tp1 * q_tp1_best_one_hot_selection, 1) + + q_tp1_best = torch.sum( + torch.where(q_tp1 > -float("inf"), q_tp1, torch.tensor(0.0)) * + q_tp1_best_one_hot_selection, 1) policy.q_loss = QLoss(q_t_selected, q_tp1_best, train_batch[PRIO_WEIGHTS], train_batch[SampleBatch.REWARDS], diff --git a/rllib/agents/qmix/qmix.py b/rllib/agents/qmix/qmix.py index 70332fd24..eb8b1f74a 100644 --- a/rllib/agents/qmix/qmix.py +++ b/rllib/agents/qmix/qmix.py @@ -7,6 +7,7 @@ from ray.rllib.execution.rollout_ops import ParallelRollouts, ConcatBatches from ray.rllib.execution.train_ops import TrainOneStep, UpdateTargetNetwork from ray.rllib.execution.metric_ops import StandardMetricsReporting from ray.rllib.execution.concurrency_ops import Concurrently +from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE # yapf: disable # __sphinx_doc_begin__ @@ -21,6 +22,22 @@ DEFAULT_CONFIG = with_common_config({ # Optimize over complete episodes by default. "batch_mode": "complete_episodes", + # === Exploration Settings === + "exploration_config": { + # The Exploration class to use. + "type": "EpsilonGreedy", + # Config for the Exploration class' constructor: + "initial_epsilon": 1.0, + "final_epsilon": 0.02, + "epsilon_timesteps": 10000, # Timesteps over which to anneal epsilon. + + # For soft_q, use: + # "exploration_config" = { + # "type": "SoftQ" + # "temperature": [float, e.g. 1.0] + # } + }, + # === Evaluation === # Evaluate with epsilon=0 every `evaluation_interval` training iterations. # The evaluation stats will be reported under the "evaluation" metric key. @@ -29,21 +46,13 @@ DEFAULT_CONFIG = with_common_config({ "evaluation_interval": None, # Number of episodes to run per evaluation period. "evaluation_num_episodes": 10, + # Switch to greedy actions in evaluation workers. + "evaluation_config": { + "explore": False, + }, - # === Exploration === - # Max num timesteps for annealing schedules. Exploration is annealed from - # 1.0 to exploration_fraction over this number of timesteps scaled by - # exploration_fraction - "schedule_max_timesteps": 100000, # Number of env steps to optimize for before returning "timesteps_per_iteration": 1000, - # Fraction of entire training period over which the exploration rate is - # annealed - "exploration_fraction": 0.1, - # Initial value of random action probability. - "exploration_initial_eps": 1.0, - # Final value of random action probability. - "exploration_final_eps": 0.02, # Update the target network every `target_network_update_freq` steps. "target_network_update_freq": 500, @@ -87,11 +96,46 @@ DEFAULT_CONFIG = with_common_config({ "lstm_cell_size": 64, "max_seq_len": 999999, }, + + # DEPRECATED VALUES (set to -1 to indicate they have not been overwritten + # by user's config). If we don't set them here, we will get an error + # from the config-key checker. + "schedule_max_timesteps": DEPRECATED_VALUE, + "exploration_fraction": DEPRECATED_VALUE, + "exploration_initial_eps": DEPRECATED_VALUE, + "exploration_final_eps": DEPRECATED_VALUE, + }) # __sphinx_doc_end__ # yapf: enable +def validate_config(config): + schedule_max_timesteps = None + if config.get("schedule_max_timesteps", DEPRECATED_VALUE) != \ + DEPRECATED_VALUE: + deprecation_warning( + "schedule_max_timesteps", + "exploration_config.epsilon_timesteps AND " + "prioritized_replay_beta_annealing_timesteps") + schedule_max_timesteps = config["schedule_max_timesteps"] + if config.get("exploration_final_eps", DEPRECATED_VALUE) != \ + DEPRECATED_VALUE: + deprecation_warning("exploration_final_eps", + "exploration_config.final_epsilon") + if isinstance(config["exploration_config"], dict): + config["exploration_config"]["final_epsilon"] = \ + config.pop("exploration_final_eps") + if config.get("exploration_fraction", DEPRECATED_VALUE) != \ + DEPRECATED_VALUE: + assert schedule_max_timesteps is not None + deprecation_warning("exploration_fraction", + "exploration_config.epsilon_timesteps") + if isinstance(config["exploration_config"], dict): + config["exploration_config"]["epsilon_timesteps"] = config.pop( + "exploration_fraction") * schedule_max_timesteps + + def execution_plan(workers, config): rollouts = ParallelRollouts(workers, mode="bulk_sync") replay_buffer = SimpleReplayBuffer(config["buffer_size"]) @@ -117,4 +161,5 @@ QMixTrainer = GenericOffPolicyTrainer.with_updates( default_config=DEFAULT_CONFIG, default_policy=QMixTorchPolicy, get_policy_class=None, + validate_config=validate_config, execution_plan=execution_plan) diff --git a/rllib/agents/qmix/qmix_policy.py b/rllib/agents/qmix/qmix_policy.py index ee06147b5..5633c2077 100644 --- a/rllib/agents/qmix/qmix_policy.py +++ b/rllib/agents/qmix/qmix_policy.py @@ -8,6 +8,7 @@ from ray.rllib.agents.qmix.mixers import VDNMixer, QMixer from ray.rllib.agents.qmix.model import RNNModel, _get_size from ray.rllib.env.multi_agent_env import ENV_STATE from ray.rllib.evaluation.metrics import LEARNER_STATS_KEY +from ray.rllib.models.torch.torch_action_dist import TorchCategorical from ray.rllib.policy.policy import Policy from ray.rllib.policy.rnn_sequencing import chop_into_sequences from ray.rllib.policy.sample_batch import SampleBatch @@ -257,6 +258,7 @@ class QMixTorchPolicy(Policy): info_batch=None, episodes=None, explore=None, + timestep=None, **kwargs): explore = explore if explore is not None else self.config["explore"] obs_batch, action_mask, _ = self._unpack_observation(obs_batch) @@ -277,15 +279,15 @@ class QMixTorchPolicy(Policy): action_mask, dtype=torch.float, device=self.device) masked_q_values = q_values.clone() masked_q_values[avail == 0.0] = -float("inf") - # epsilon-greedy action selector - random_numbers = torch.rand_like(q_values[:, :, 0]) - pick_random = (random_numbers < (self.cur_epsilon - if explore else 0.0)).long() - from torch.distributions import Categorical - random_actions = Categorical(avail).sample().long() - actions = (pick_random * random_actions + - (1 - pick_random) * masked_q_values.argmax(dim=2)) - actions = actions.cpu().numpy() + masked_q_values_folded = torch.reshape( + masked_q_values, + [-1] + list(masked_q_values.shape)[2:]) + actions, _ = self.exploration.get_exploration_action( + action_distribution=TorchCategorical(masked_q_values_folded), + timestep=timestep, + explore=explore) + actions = torch.reshape( + actions, list(masked_q_values.shape)[:-1]).cpu().numpy() hiddens = [s.cpu().numpy() for s in hiddens] return tuple(actions.transpose([1, 0])), hiddens, {} diff --git a/rllib/agents/qmix/tests/test_qmix.py b/rllib/agents/qmix/tests/test_qmix.py index 5576deebf..5c214a5d8 100644 --- a/rllib/agents/qmix/tests/test_qmix.py +++ b/rllib/agents/qmix/tests/test_qmix.py @@ -53,7 +53,7 @@ class AvailActionsTestEnv(MultiAgentEnv): return obs, rewards, dones, {} -class TestAvailActionsQMix(unittest.TestCase): +class TestQMix(unittest.TestCase): def test_avail_actions_qmix(self): grouping = { "group_1": ["agent_1"], # trivial grouping for testing diff --git a/rllib/contrib/maddpg/maddpg.py b/rllib/contrib/maddpg/maddpg.py index 217499a70..949d092d7 100644 --- a/rllib/contrib/maddpg/maddpg.py +++ b/rllib/contrib/maddpg/maddpg.py @@ -5,8 +5,8 @@ The implementation has a couple assumptions: - Each agent is bound to a policy of the same name. - Discrete actions are sent as logits (pre-softmax). -For a minimal example, see twostep_game.py, and the README for how to run -with the multi-agent particle envs. +For a minimal example, see rllib/examples/two_step_game.py, +and the README for how to run with the multi-agent particle envs. """ import logging diff --git a/rllib/examples/centralized_critic.py b/rllib/examples/centralized_critic.py index 55ae1eed6..951144e54 100644 --- a/rllib/examples/centralized_critic.py +++ b/rllib/examples/centralized_critic.py @@ -3,10 +3,11 @@ Here the model and policy are hard-coded to implement a centralized critic for TwoStepGame, but you can adapt this for your own use cases. -Compared to simply running `twostep_game.py --run=PPO`, this centralized -critic version reaches vf_explained_variance=1.0 more stably since it takes -into account the opponent actions as well as the policy's. Note that this is -also using two independent policies instead of weight-sharing with one. +Compared to simply running `rllib/examples/two_step_game.py --run=PPO`, +this centralized critic version reaches vf_explained_variance=1.0 more stably +since it takes into account the opponent actions as well as the policy's. +Note that this is also using two independent policies instead of weight-sharing +with one. See also: centralized_critic_2.py for a simpler approach that instead modifies the environment. diff --git a/rllib/examples/models/parametric_actions_model.py b/rllib/examples/models/parametric_actions_model.py index 225399286..1dc2945f9 100644 --- a/rllib/examples/models/parametric_actions_model.py +++ b/rllib/examples/models/parametric_actions_model.py @@ -7,7 +7,6 @@ from ray.rllib.agents.dqn.dqn_torch_model import \ from ray.rllib.models.tf.fcnet import FullyConnectedNetwork from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC from ray.rllib.utils.framework import try_import_tf, try_import_torch -from ray.rllib.utils.numpy import LARGE_INTEGER tf1, tf, tfv = try_import_tf() torch, nn = try_import_torch() @@ -99,11 +98,11 @@ class TorchParametricActionsModel(DQNTorchModel): # Batch dot product => shape of logits is [BATCH, MAX_ACTIONS]. action_logits = torch.sum(avail_actions * intent_vector, dim=2) - # Mask out invalid actions (use -LARGE_INTEGER to tag invalid). + # Mask out invalid actions (use -inf to tag invalid). # These are then recognized by the EpsilonGreedy exploration component # as invalid actions that are not to be chosen. inf_mask = torch.clamp( - torch.log(action_mask), -float(LARGE_INTEGER), float("inf")) + torch.log(action_mask), -float("inf"), float("inf")) return action_logits + inf_mask, state def value_function(self): diff --git a/rllib/examples/twostep_game.py b/rllib/examples/two_step_game.py similarity index 94% rename from rllib/examples/twostep_game.py rename to rllib/examples/two_step_game.py index ca30deeb5..579f02537 100644 --- a/rllib/examples/twostep_game.py +++ b/rllib/examples/two_step_game.py @@ -83,8 +83,10 @@ if __name__ == "__main__": config = { "rollout_fragment_length": 4, "train_batch_size": 32, - "exploration_fraction": .4, - "exploration_final_eps": 0.0, + "exploration_config": { + "epsilon_timesteps": 5000, + "final_epsilon": 0.05, + }, "num_workers": 0, "mixer": grid_search([None, "qmix", "vdn"]), "env_config": { @@ -109,7 +111,7 @@ if __name__ == "__main__": "env": "grouped_twostep" if group else TwoStepGame, }) - results = tune.run(args.run, stop=stop, config=config) + results = tune.run(args.run, stop=stop, config=config, verbose=1) if args.as_test: check_learning_achieved(results, args.stop_reward) diff --git a/rllib/utils/exploration/epsilon_greedy.py b/rllib/utils/exploration/epsilon_greedy.py index 8323f2c08..edd347068 100644 --- a/rllib/utils/exploration/epsilon_greedy.py +++ b/rllib/utils/exploration/epsilon_greedy.py @@ -6,7 +6,6 @@ from ray.rllib.utils.exploration.exploration import Exploration, TensorType from ray.rllib.utils.framework import try_import_tf, try_import_torch, \ get_variable from ray.rllib.utils.from_config import from_config -from ray.rllib.utils.numpy import LARGE_INTEGER from ray.rllib.utils.schedules import Schedule, PiecewiseSchedule tf1, tf, tfv = try_import_tf() @@ -140,7 +139,7 @@ class EpsilonGreedy(Exploration): # Mask out actions, whose Q-values are -inf, so that we don't # even consider them for exploration. random_valid_action_logits = torch.where( - q_values == -float(LARGE_INTEGER), + q_values == -float("inf"), torch.ones_like(q_values) * 0.0, torch.ones_like(q_values)) # A random action. random_actions = torch.squeeze(