wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-02 13:01:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[RLlib] Issue 8384: QMIX doesn't learn anything. (#9527)

Browse Source
This commit is contained in:
Sven Mika
2020-07-17 12:14:34 +02:00
committed by GitHub
parent 37942ea1e7
commit 78dfed2683
13 changed files with 109 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/rllib-algorithms.rst
+3 -3
View File
@@ -476,9 +476,9 @@ Tuned examples: `Humanoid-v1 <https://github.com/ray-project/ray/blob/master/rll
QMIX Monotonic Value Factorisation (QMIX, VDN, IQN)
---------------------------------------------------
|pytorch|
`[paper] <https://arxiv.org/abs/1803.11485>`__ `[implementation] <https://github.com/ray-project/ray/blob/master/rllib/agents/qmix/qmix.py>`__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping <rllib-env.html#grouping-agents>`__ in the environment (see the `two-step game example <https://github.com/ray-project/ray/blob/master/rllib/examples/twostep_game.py>`__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X.
`[paper] <https://arxiv.org/abs/1803.11485>`__ `[implementation] <https://github.com/ray-project/ray/blob/master/rllib/agents/qmix/qmix.py>`__ Q-Mix is a specialized multi-agent algorithm. Code here is adapted from https://github.com/oxwhirl/pymarl_alpha to integrate with RLlib multi-agent APIs. To use Q-Mix, you must specify an agent `grouping <rllib-env.html#grouping-agents>`__ in the environment (see the `two-step game example <https://github.com/ray-project/ray/blob/master/rllib/examples/two_step_game.py>`__). Currently, all agents in the group must be homogeneous. The algorithm can be scaled by increasing the number of workers or using Ape-X.
Tuned examples: `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/twostep_game.py>`__
Tuned examples: `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/two_step_game.py>`__
**QMIX-specific configs** (see also `common configs <rllib-training.html#common-parameters>`__):
@@ -496,7 +496,7 @@ Multi-Agent Deep Deterministic Policy Gradient (contrib/MADDPG)
**MADDPG-specific configs** (see also `common configs <rllib-training.html#common-parameters>`__):
Tuned examples: `Multi-Agent Particle Environment <https://github.com/wsjeon/maddpg-rllib/tree/master/plots>`__, `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/twostep_game.py>`__
Tuned examples: `Multi-Agent Particle Environment <https://github.com/wsjeon/maddpg-rllib/tree/master/plots>`__, `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/two_step_game.py>`__
.. literalinclude:: ../../rllib/contrib/maddpg/maddpg.py
:language: python
doc/source/rllib-examples.rst
+1 -1
View File
@@ -80,7 +80,7 @@ Multi-Agent and Hierarchical
- `Rock-paper-scissors <https://github.com/ray-project/ray/blob/master/rllib/examples/rock_paper_scissors_multiagent.py>`__:
Example of different heuristic and learned policies competing against each other in rock-paper-scissors.
- `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/twostep_game.py>`__:
- `Two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/two_step_game.py>`__:
Example of the two-step game from the `QMIX paper <https://arxiv.org/pdf/1803.11485.pdf>`__.
- `PPO with centralized critic on two-step game <https://github.com/ray-project/ray/blob/master/rllib/examples/centralized_critic.py>`__:
Example of customizing PPO to leverage a centralized value function.
rllib/BUILD
+13 -13
View File
@@ -1943,39 +1943,39 @@ py_test(
)
py_test(
name = "examples/twostep_game_maddpg",
main = "examples/twostep_game.py",
name = "examples/two_step_game_maddpg",
main = "examples/two_step_game.py",
tags = ["examples", "examples_T"],
size = "large",
srcs = ["examples/twostep_game.py"],
srcs = ["examples/two_step_game.py"],
args = ["--stop-timesteps=2000", "--run=contrib/MADDPG"]
)
py_test(
name = "examples/twostep_game_pg_tf",
main = "examples/twostep_game.py",
name = "examples/two_step_game_pg_tf",
main = "examples/two_step_game.py",
tags = ["examples", "examples_T"],
size = "medium",
srcs = ["examples/twostep_game.py"],
srcs = ["examples/two_step_game.py"],
args = ["--as-test", "--stop-reward=7", "--run=PG"]
)
py_test(
name = "examples/twostep_game_pg_torch",
main = "examples/twostep_game.py",
name = "examples/two_step_game_pg_torch",
main = "examples/two_step_game.py",
tags = ["examples", "examples_T"],
size = "medium",
srcs = ["examples/twostep_game.py"],
srcs = ["examples/two_step_game.py"],
args = ["--as-test", "--torch", "--stop-reward=7", "--run=PG"]
)
py_test(
name = "examples/twostep_game_qmix",
main = "examples/twostep_game.py",
name = "examples/two_step_game_qmix",
main = "examples/two_step_game.py",
tags = ["examples", "examples_T"],
size = "medium",
srcs = ["examples/twostep_game.py"],
args = ["--stop-timesteps=2000", "--run=QMIX"]
srcs = ["examples/two_step_game.py"],
args = ["--as-test", "--torch", "--stop-reward=7", "--run=QMIX"]
)
py_test(
rllib/agents/dqn/dqn.py
+1 -1
View File
@@ -40,7 +40,7 @@ DEFAULT_CONFIG = with_common_config({
# N-step Q learning
"n_step": 1,
# === Exploration Settings (Experimental) ===
# === Exploration Settings ===
"exploration_config": {
# The Exploration class to use.
"type": "EpsilonGreedy",
rllib/agents/dqn/dqn_torch_policy.py
+7 -3
View File
@@ -169,7 +169,9 @@ def build_q_losses(policy, model, _, train_batch):
# q scores for actions which we know were selected in the given state.
one_hot_selection = F.one_hot(train_batch[SampleBatch.ACTIONS],
policy.action_space.n)
q_t_selected = torch.sum(q_t * one_hot_selection, 1)
q_t_selected = torch.sum(
torch.where(q_t > -float("inf"), q_t, torch.tensor(0.0)) *
one_hot_selection, 1)
# compute estimate of best possible value starting from state at t + 1
if config["double_q"]:
@@ -182,11 +184,13 @@ def build_q_losses(policy, model, _, train_batch):
q_tp1_best_using_online_net = torch.argmax(q_tp1_using_online_net, 1)
q_tp1_best_one_hot_selection = F.one_hot(q_tp1_best_using_online_net,
policy.action_space.n)
q_tp1_best = torch.sum(q_tp1 * q_tp1_best_one_hot_selection, 1)
else:
q_tp1_best_one_hot_selection = F.one_hot(
torch.argmax(q_tp1, 1), policy.action_space.n)
q_tp1_best = torch.sum(q_tp1 * q_tp1_best_one_hot_selection, 1)
q_tp1_best = torch.sum(
torch.where(q_tp1 > -float("inf"), q_tp1, torch.tensor(0.0)) *
q_tp1_best_one_hot_selection, 1)
policy.q_loss = QLoss(q_t_selected, q_tp1_best, train_batch[PRIO_WEIGHTS],
train_batch[SampleBatch.REWARDS],
rllib/agents/qmix/qmix.py
+57 -12
View File
@@ -7,6 +7,7 @@ from ray.rllib.execution.rollout_ops import ParallelRollouts, ConcatBatches
from ray.rllib.execution.train_ops import TrainOneStep, UpdateTargetNetwork
from ray.rllib.execution.metric_ops import StandardMetricsReporting
from ray.rllib.execution.concurrency_ops import Concurrently
from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE
# yapf: disable
# __sphinx_doc_begin__
@@ -21,6 +22,22 @@ DEFAULT_CONFIG = with_common_config({
# Optimize over complete episodes by default.
"batch_mode": "complete_episodes",
# === Exploration Settings ===
"exploration_config": {
# The Exploration class to use.
"type": "EpsilonGreedy",
# Config for the Exploration class' constructor:
"initial_epsilon": 1.0,
"final_epsilon": 0.02,
"epsilon_timesteps": 10000, # Timesteps over which to anneal epsilon.
# For soft_q, use:
# "exploration_config" = {
# "type": "SoftQ"
# "temperature": [float, e.g. 1.0]
# }
},
# === Evaluation ===
# Evaluate with epsilon=0 every `evaluation_interval` training iterations.
# The evaluation stats will be reported under the "evaluation" metric key.
@@ -29,21 +46,13 @@ DEFAULT_CONFIG = with_common_config({
"evaluation_interval": None,
# Number of episodes to run per evaluation period.
"evaluation_num_episodes": 10,
# Switch to greedy actions in evaluation workers.
"evaluation_config": {
"explore": False,
},
# === Exploration ===
# Max num timesteps for annealing schedules. Exploration is annealed from
# 1.0 to exploration_fraction over this number of timesteps scaled by
# exploration_fraction
"schedule_max_timesteps": 100000,
# Number of env steps to optimize for before returning
"timesteps_per_iteration": 1000,
# Fraction of entire training period over which the exploration rate is
# annealed
"exploration_fraction": 0.1,
# Initial value of random action probability.
"exploration_initial_eps": 1.0,
# Final value of random action probability.
"exploration_final_eps": 0.02,
# Update the target network every `target_network_update_freq` steps.
"target_network_update_freq": 500,
@@ -87,11 +96,46 @@ DEFAULT_CONFIG = with_common_config({
"lstm_cell_size": 64,
"max_seq_len": 999999,
},
# DEPRECATED VALUES (set to -1 to indicate they have not been overwritten
# by user's config). If we don't set them here, we will get an error
# from the config-key checker.
"schedule_max_timesteps": DEPRECATED_VALUE,
"exploration_fraction": DEPRECATED_VALUE,
"exploration_initial_eps": DEPRECATED_VALUE,
"exploration_final_eps": DEPRECATED_VALUE,
})
# __sphinx_doc_end__
# yapf: enable
def validate_config(config):
schedule_max_timesteps = None
if config.get("schedule_max_timesteps", DEPRECATED_VALUE) != \
DEPRECATED_VALUE:
deprecation_warning(
"schedule_max_timesteps",
"exploration_config.epsilon_timesteps AND "
"prioritized_replay_beta_annealing_timesteps")
schedule_max_timesteps = config["schedule_max_timesteps"]
if config.get("exploration_final_eps", DEPRECATED_VALUE) != \
DEPRECATED_VALUE:
deprecation_warning("exploration_final_eps",
"exploration_config.final_epsilon")
if isinstance(config["exploration_config"], dict):
config["exploration_config"]["final_epsilon"] = \
config.pop("exploration_final_eps")
if config.get("exploration_fraction", DEPRECATED_VALUE) != \
DEPRECATED_VALUE:
assert schedule_max_timesteps is not None
deprecation_warning("exploration_fraction",
"exploration_config.epsilon_timesteps")
if isinstance(config["exploration_config"], dict):
config["exploration_config"]["epsilon_timesteps"] = config.pop(
"exploration_fraction") * schedule_max_timesteps
def execution_plan(workers, config):
rollouts = ParallelRollouts(workers, mode="bulk_sync")
replay_buffer = SimpleReplayBuffer(config["buffer_size"])
@@ -117,4 +161,5 @@ QMixTrainer = GenericOffPolicyTrainer.with_updates(
default_config=DEFAULT_CONFIG,
default_policy=QMixTorchPolicy,
get_policy_class=None,
validate_config=validate_config,
execution_plan=execution_plan)
rllib/agents/qmix/qmix_policy.py
+11 -9
View File
@@ -8,6 +8,7 @@ from ray.rllib.agents.qmix.mixers import VDNMixer, QMixer
from ray.rllib.agents.qmix.model import RNNModel, _get_size
from ray.rllib.env.multi_agent_env import ENV_STATE
from ray.rllib.evaluation.metrics import LEARNER_STATS_KEY
from ray.rllib.models.torch.torch_action_dist import TorchCategorical
from ray.rllib.policy.policy import Policy
from ray.rllib.policy.rnn_sequencing import chop_into_sequences
from ray.rllib.policy.sample_batch import SampleBatch
@@ -257,6 +258,7 @@ class QMixTorchPolicy(Policy):
info_batch=None,
episodes=None,
explore=None,
timestep=None,
**kwargs):
explore = explore if explore is not None else self.config["explore"]
obs_batch, action_mask, _ = self._unpack_observation(obs_batch)
@@ -277,15 +279,15 @@ class QMixTorchPolicy(Policy):
action_mask, dtype=torch.float, device=self.device)
masked_q_values = q_values.clone()
masked_q_values[avail == 0.0] = -float("inf")
# epsilon-greedy action selector
random_numbers = torch.rand_like(q_values[:, :, 0])
pick_random = (random_numbers < (self.cur_epsilon
if explore else 0.0)).long()
from torch.distributions import Categorical
random_actions = Categorical(avail).sample().long()
actions = (pick_random * random_actions +
(1 - pick_random) * masked_q_values.argmax(dim=2))
actions = actions.cpu().numpy()
masked_q_values_folded = torch.reshape(
masked_q_values,
[-1] + list(masked_q_values.shape)[2:])
actions, _ = self.exploration.get_exploration_action(
action_distribution=TorchCategorical(masked_q_values_folded),
timestep=timestep,
explore=explore)
actions = torch.reshape(
actions, list(masked_q_values.shape)[:-1]).cpu().numpy()
hiddens = [s.cpu().numpy() for s in hiddens]
return tuple(actions.transpose([1, 0])), hiddens, {}
rllib/agents/qmix/tests/test_qmix.py
+1 -1
View File
@@ -53,7 +53,7 @@ class AvailActionsTestEnv(MultiAgentEnv):
return obs, rewards, dones, {}
class TestAvailActionsQMix(unittest.TestCase):
class TestQMix(unittest.TestCase):
def test_avail_actions_qmix(self):
grouping = {
"group_1": ["agent_1"], # trivial grouping for testing
rllib/contrib/maddpg/maddpg.py
+2 -2
View File
@@ -5,8 +5,8 @@ The implementation has a couple assumptions:
- Each agent is bound to a policy of the same name.
- Discrete actions are sent as logits (pre-softmax).
For a minimal example, see twostep_game.py, and the README for how to run
with the multi-agent particle envs.
For a minimal example, see rllib/examples/two_step_game.py,
and the README for how to run with the multi-agent particle envs.
"""
import logging
rllib/examples/centralized_critic.py
+5 -4
View File
@@ -3,10 +3,11 @@
Here the model and policy are hard-coded to implement a centralized critic
for TwoStepGame, but you can adapt this for your own use cases.
Compared to simply running `twostep_game.py --run=PPO`, this centralized
critic version reaches vf_explained_variance=1.0 more stably since it takes
into account the opponent actions as well as the policy's. Note that this is
also using two independent policies instead of weight-sharing with one.
Compared to simply running `rllib/examples/two_step_game.py --run=PPO`,
this centralized critic version reaches vf_explained_variance=1.0 more stably
since it takes into account the opponent actions as well as the policy's.
Note that this is also using two independent policies instead of weight-sharing
with one.
See also: centralized_critic_2.py for a simpler approach that instead
modifies the environment.
rllib/examples/models/parametric_actions_model.py
+2 -3
View File
@@ -7,7 +7,6 @@ from ray.rllib.agents.dqn.dqn_torch_model import \
from ray.rllib.models.tf.fcnet import FullyConnectedNetwork
from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC
from ray.rllib.utils.framework import try_import_tf, try_import_torch
from ray.rllib.utils.numpy import LARGE_INTEGER
tf1, tf, tfv = try_import_tf()
torch, nn = try_import_torch()
@@ -99,11 +98,11 @@ class TorchParametricActionsModel(DQNTorchModel):
# Batch dot product => shape of logits is [BATCH, MAX_ACTIONS].
action_logits = torch.sum(avail_actions * intent_vector, dim=2)
# Mask out invalid actions (use -LARGE_INTEGER to tag invalid).
# Mask out invalid actions (use -inf to tag invalid).
# These are then recognized by the EpsilonGreedy exploration component
# as invalid actions that are not to be chosen.
inf_mask = torch.clamp(
torch.log(action_mask), -float(LARGE_INTEGER), float("inf"))
torch.log(action_mask), -float("inf"), float("inf"))
return action_logits + inf_mask, state
def value_function(self):
rllib/examples/twostep_game.py → rllib/examples/two_step_game.py
+5 -3
View File
@@ -83,8 +83,10 @@ if __name__ == "__main__":
config = {
"rollout_fragment_length": 4,
"train_batch_size": 32,
"exploration_fraction": .4,
"exploration_final_eps": 0.0,
"exploration_config": {
"epsilon_timesteps": 5000,
"final_epsilon": 0.05,
},
"num_workers": 0,
"mixer": grid_search([None, "qmix", "vdn"]),
"env_config": {
@@ -109,7 +111,7 @@ if __name__ == "__main__":
"env": "grouped_twostep" if group else TwoStepGame,
})
results = tune.run(args.run, stop=stop, config=config)
results = tune.run(args.run, stop=stop, config=config, verbose=1)
if args.as_test:
check_learning_achieved(results, args.stop_reward)
rllib/utils/exploration/epsilon_greedy.py
+1 -2
View File
@@ -6,7 +6,6 @@ from ray.rllib.utils.exploration.exploration import Exploration, TensorType
from ray.rllib.utils.framework import try_import_tf, try_import_torch, \
get_variable
from ray.rllib.utils.from_config import from_config
from ray.rllib.utils.numpy import LARGE_INTEGER
from ray.rllib.utils.schedules import Schedule, PiecewiseSchedule
tf1, tf, tfv = try_import_tf()
@@ -140,7 +139,7 @@ class EpsilonGreedy(Exploration):
# Mask out actions, whose Q-values are -inf, so that we don't
# even consider them for exploration.
random_valid_action_logits = torch.where(
q_values == -float(LARGE_INTEGER),
q_values == -float("inf"),
torch.ones_like(q_values) * 0.0, torch.ones_like(q_values))
# A random action.
random_actions = torch.squeeze(