wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

MADDPG implementation in RLlib (#5348)

Browse Source
This commit is contained in:
Wonseok Jeon
2019-08-06 16:22:06 -07:00
committed by Eric Liang
parent 094ec7adbc
commit 281829e712
13 changed files with 736 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rllib/examples/twostep_game.py
+54 -8
View File
@@ -6,6 +6,7 @@ from __future__ import print_function
import argparse
from gym.spaces import Tuple, Discrete
import numpy as np
import ray
from ray import tune
@@ -26,14 +27,24 @@ class TwoStepGame(MultiAgentEnv):
def __init__(self, env_config):
self.state = None
self.agent_1 = 0
self.agent_2 = 1
# MADDPG emits action logits instead of actual discrete actions
self.actions_are_logits = env_config.get("actions_are_logits", False)
def reset(self):
self.state = 0
return {"agent_1": self.state, "agent_2": self.state + 3}
return {self.agent_1: self.state, self.agent_2: self.state + 3}
def step(self, action_dict):
if self.actions_are_logits:
action_dict = {
k: np.random.choice([0, 1], p=v)
for k, v in action_dict.items()
}
if self.state == 0:
action = action_dict["agent_1"]
action = action_dict[self.agent_1]
assert action in [0, 1], action
if action == 0:
self.state = 1
@@ -45,16 +56,21 @@ class TwoStepGame(MultiAgentEnv):
global_rew = 7
done = True
else:
if action_dict["agent_1"] == 0 and action_dict["agent_2"] == 0:
if action_dict[self.agent_1] == 0 and action_dict[self.
agent_2] == 0:
global_rew = 0
elif action_dict["agent_1"] == 1 and action_dict["agent_2"] == 1:
elif action_dict[self.agent_1] == 1 and action_dict[self.
agent_2] == 1:
global_rew = 8
else:
global_rew = 1
done = True
rewards = {"agent_1": global_rew / 2.0, "agent_2": global_rew / 2.0}
obs = {"agent_1": self.state, "agent_2": self.state + 3}
rewards = {
self.agent_1: global_rew / 2.0,
self.agent_2: global_rew / 2.0
}
obs = {self.agent_1: self.state, self.agent_2: self.state + 3}
dones = {"__all__": done}
infos = {}
return obs, rewards, dones, infos
@@ -64,7 +80,7 @@ if __name__ == "__main__":
args = parser.parse_args()
grouping = {
"group_1": ["agent_1", "agent_2"],
"group_1": [0, 1],
}
obs_space = Tuple([
TwoStepGame.observation_space,
@@ -79,7 +95,37 @@ if __name__ == "__main__":
lambda config: TwoStepGame(config).with_agent_groups(
grouping, obs_space=obs_space, act_space=act_space))
if args.run == "QMIX":
if args.run == "contrib/MADDPG":
obs_space_dict = {
"agent_1": TwoStepGame.observation_space,
"agent_2": TwoStepGame.observation_space,
}
act_space_dict = {
"agent_1": TwoStepGame.action_space,
"agent_2": TwoStepGame.action_space,
}
config = {
"learning_starts": 100,
"env_config": {
"actions_are_logits": True,
},
"multiagent": {
"policies": {
"pol1": (None, TwoStepGame.observation_space,
TwoStepGame.action_space, {
"agent_id": 0,
}),
"pol2": (None, TwoStepGame.observation_space,
TwoStepGame.action_space, {
"agent_id": 1,
}),
},
"policy_mapping_fn": tune.function(
lambda x: "pol1" if x == 0 else "pol2"),
},
}
group = False
elif args.run == "QMIX":
config = {
"sample_batch_size": 4,
"train_batch_size": 32,