wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-13 13:10:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

[rllib] format with yapf (#2427)

Browse Source 
* initial yapf

* manual fix yapf bugs
This commit is contained in:
Eric Liang
2018-07-19 15:30:36 -07:00
committed by GitHub
parent 24eb140e07
commit d01dc9e22d
86 changed files with 1276 additions and 978 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/rllib/agents/a3c/a3c.py
+5 -5
View File
@@ -92,15 +92,15 @@ class A3CAgent(Agent):
self.remote_evaluators = self.make_remote_evaluators(
self.env_creator, policy_cls, self.config["num_workers"],
{"num_gpus": 1 if self.config["use_gpu_for_workers"] else 0})
self.optimizer = AsyncGradientsOptimizer(
self.local_evaluator, self.remote_evaluators,
self.config["optimizer"])
self.optimizer = AsyncGradientsOptimizer(self.local_evaluator,
self.remote_evaluators,
self.config["optimizer"])
def _train(self):
prev_steps = self.optimizer.num_steps_sampled
self.optimizer.step()
FilterManager.synchronize(
self.local_evaluator.filters, self.remote_evaluators)
FilterManager.synchronize(self.local_evaluator.filters,
self.remote_evaluators)
result = self.optimizer.collect_metrics()
result = result._replace(
timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps)
python/ray/rllib/agents/a3c/a3c_tf_policy_graph.py
+27 -17
View File
@@ -14,19 +14,23 @@ from ray.rllib.models.catalog import ModelCatalog
class A3CLoss(object):
def __init__(
self, action_dist, actions, advantages, v_target, vf,
vf_loss_coeff=0.5, entropy_coeff=-0.01):
def __init__(self,
action_dist,
actions,
advantages,
v_target,
vf,
vf_loss_coeff=0.5,
entropy_coeff=-0.01):
log_prob = action_dist.logp(actions)
# The "policy gradients" loss
self.pi_loss = - tf.reduce_sum(log_prob * advantages)
self.pi_loss = -tf.reduce_sum(log_prob * advantages)
delta = vf - v_target
self.vf_loss = 0.5 * tf.reduce_sum(tf.square(delta))
self.entropy = tf.reduce_sum(action_dist.entropy())
self.total_loss = (self.pi_loss +
self.vf_loss * vf_loss_coeff +
self.total_loss = (self.pi_loss + self.vf_loss * vf_loss_coeff +
self.entropy * entropy_coeff)
@@ -41,8 +45,8 @@ class A3CPolicyGraph(TFPolicyGraph):
tf.float32, [None] + list(observation_space.shape))
dist_class, logit_dim = ModelCatalog.get_action_dist(
action_space, self.config["model"])
self.model = ModelCatalog.get_model(
self.observations, logit_dim, self.config["model"])
self.model = ModelCatalog.get_model(self.observations, logit_dim,
self.config["model"])
action_dist = dist_class(self.model.outputs)
self.vf = tf.reshape(
linear(self.model.last_layer, 1, "value", normc_initializer(1.0)),
@@ -62,9 +66,9 @@ class A3CPolicyGraph(TFPolicyGraph):
action_space))
advantages = tf.placeholder(tf.float32, [None], name="advantages")
v_target = tf.placeholder(tf.float32, [None], name="v_target")
self.loss = A3CLoss(
action_dist, actions, advantages, v_target, self.vf,
self.config["vf_loss_coeff"], self.config["entropy_coeff"])
self.loss = A3CLoss(action_dist, actions, advantages, v_target,
self.vf, self.config["vf_loss_coeff"],
self.config["entropy_coeff"])
# Initialize TFPolicyGraph
loss_in = [
@@ -76,10 +80,16 @@ class A3CPolicyGraph(TFPolicyGraph):
self.state_in = self.model.state_in
self.state_out = self.model.state_out
TFPolicyGraph.__init__(
self, observation_space, action_space, self.sess,
obs_input=self.observations, action_sampler=action_dist.sample(),
loss=self.loss.total_loss, loss_inputs=loss_in,
state_inputs=self.state_in, state_outputs=self.state_out,
self,
observation_space,
action_space,
self.sess,
obs_input=self.observations,
action_sampler=action_dist.sample(),
loss=self.loss.total_loss,
loss_inputs=loss_in,
state_inputs=self.state_in,
state_outputs=self.state_out,
seq_lens=self.model.seq_lens,
max_seq_len=self.config["model"]["max_seq_len"])
@@ -132,5 +142,5 @@ class A3CPolicyGraph(TFPolicyGraph):
for i in range(len(self.state_in)):
next_state.append([sample_batch["state_out_{}".format(i)][-1]])
last_r = self.value(sample_batch["new_obs"][-1], *next_state)
return compute_advantages(
sample_batch, last_r, self.config["gamma"], self.config["lambda"])
return compute_advantages(sample_batch, last_r, self.config["gamma"],
self.config["lambda"])
python/ray/rllib/agents/a3c/a3c_torch_policy_graph.py
+11 -10
View File
@@ -46,20 +46,21 @@ class A3CTorchPolicyGraph(TorchPolicyGraph):
action_space, self.config["model"])
self.model = ModelCatalog.get_torch_model(
obs_space.shape, self.logit_dim, self.config["model"])
loss = A3CLoss(
self.model, self.config["vf_loss_coeff"],
self.config["entropy_coeff"])
loss = A3CLoss(self.model, self.config["vf_loss_coeff"],
self.config["entropy_coeff"])
TorchPolicyGraph.__init__(
self, obs_space, action_space, self.model, loss,
loss_inputs=[
"obs", "actions", "advantages", "value_targets"])
self,
obs_space,
action_space,
self.model,
loss,
loss_inputs=["obs", "actions", "advantages", "value_targets"])
def extra_action_out(self, model_out):
return {"vf_preds": var_to_np(model_out[1])}
def optimizer(self):
return torch.optim.Adam(
self.model.parameters(), lr=self.config["lr"])
return torch.optim.Adam(self.model.parameters(), lr=self.config["lr"])
def postprocess_trajectory(self, sample_batch, other_agent_batches=None):
completed = sample_batch["dones"][-1]
@@ -67,8 +68,8 @@ class A3CTorchPolicyGraph(TorchPolicyGraph):
last_r = 0.0
else:
last_r = self._value(sample_batch["new_obs"][-1])
return compute_advantages(
sample_batch, last_r, self.config["gamma"], self.config["lambda"])
return compute_advantages(sample_batch, last_r, self.config["gamma"],
self.config["lambda"])
def _value(self, obs):
with self.lock:
python/ray/rllib/agents/agent.py
+33 -28
View File
@@ -47,7 +47,9 @@ COMMON_CONFIG = {
"allow_growth": True,
},
"log_device_placement": False,
"device_count": {"CPU": 1},
"device_count": {
"CPU": 1
},
"allow_soft_placement": True, # required by PPO multi-gpu
},
# Whether to LZ4 compress observations
@@ -86,8 +88,7 @@ def _deep_update(original, new_dict, new_keys_allowed, whitelist):
for k, value in new_dict.items():
if k not in original and k != "env":
if not new_keys_allowed:
raise Exception(
"Unknown config parameter `{}` ".format(k))
raise Exception("Unknown config parameter `{}` ".format(k))
if type(original.get(k)) is dict:
if k in whitelist:
_deep_update(original[k], value, True, [])
@@ -112,22 +113,24 @@ class Agent(Trainable):
_allow_unknown_configs = False
_allow_unknown_subkeys = [
"tf_session_args", "env_config", "model", "optimizer", "multiagent"]
"tf_session_args", "env_config", "model", "optimizer", "multiagent"
]
def make_local_evaluator(self, env_creator, policy_graph):
"""Convenience method to return configured local evaluator."""
return self._make_evaluator(
PolicyEvaluator, env_creator, policy_graph, 0)
return self._make_evaluator(PolicyEvaluator, env_creator, policy_graph,
0)
def make_remote_evaluators(
self, env_creator, policy_graph, count, remote_args):
def make_remote_evaluators(self, env_creator, policy_graph, count,
remote_args):
"""Convenience method to return a number of remote evaluators."""
cls = PolicyEvaluator.as_remote(**remote_args).remote
return [
self._make_evaluator(cls, env_creator, policy_graph, i+1)
for i in range(count)]
self._make_evaluator(cls, env_creator, policy_graph, i + 1)
for i in range(count)
]
def _make_evaluator(self, cls, env_creator, policy_graph, worker_index):
config = self.config
@@ -140,8 +143,8 @@ class Agent(Trainable):
env_creator,
self.config["multiagent"]["policy_graphs"] or policy_graph,
policy_mapping_fn=self.config["multiagent"]["policy_mapping_fn"],
tf_session_creator=(
session_creator if config["tf_session_args"] else None),
tf_session_creator=(session_creator
if config["tf_session_args"] else None),
batch_steps=config["sample_batch_size"],
batch_mode=config["batch_mode"],
episode_horizon=config["horizon"],
@@ -157,14 +160,12 @@ class Agent(Trainable):
@classmethod
def resource_help(cls, config):
return (
"\n\nYou can adjust the resource requests of RLlib agents by "
"setting `num_workers` and other configs. See the "
"DEFAULT_CONFIG defined by each agent for more info.\n\n"
"The config of this agent is: " + json.dumps(config))
return ("\n\nYou can adjust the resource requests of RLlib agents by "
"setting `num_workers` and other configs. See the "
"DEFAULT_CONFIG defined by each agent for more info.\n\n"
"The config of this agent is: " + json.dumps(config))
def __init__(
self, config=None, env=None, logger_creator=None):
def __init__(self, config=None, env=None, logger_creator=None):
"""Initialize an RLLib agent.
Args:
@@ -235,8 +236,8 @@ class Agent(Trainable):
obs = self.local_evaluator.filters["default"](
observation, update=False)
return self.local_evaluator.for_policy(
lambda p: p.compute_single_action(
obs, state, is_training=False)[0])
lambda p: p.compute_single_action(obs, state, is_training=False)[0]
)
class _MockAgent(Agent):
@@ -257,8 +258,10 @@ class _MockAgent(Agent):
and (self.config["persistent_error"] or not self.restored):
raise Exception("mock error")
return TrainingResult(
episode_reward_mean=10, episode_len_mean=10,
timesteps_this_iter=10, info={})
episode_reward_mean=10,
episode_len_mean=10,
timesteps_this_iter=10,
info={})
def _save(self, checkpoint_dir):
path = os.path.join(checkpoint_dir, "mock_agent.pkl")
@@ -299,9 +302,11 @@ class _SigmoidFakeData(_MockAgent):
v = np.tanh(float(i) / self.config["width"])
v *= self.config["height"]
return TrainingResult(
episode_reward_mean=v, episode_len_mean=v,
episode_reward_mean=v,
episode_len_mean=v,
timesteps_this_iter=self.config["iter_timesteps"],
time_this_iter_s=self.config["iter_time"], info={})
time_this_iter_s=self.config["iter_time"],
info={})
class _ParameterTuningAgent(_MockAgent):
@@ -320,7 +325,8 @@ class _ParameterTuningAgent(_MockAgent):
episode_reward_mean=self.config["reward_amt"] * self.iteration,
episode_len_mean=self.config["reward_amt"],
timesteps_this_iter=self.config["iter_timesteps"],
time_this_iter_s=self.config["iter_time"], info={})
time_this_iter_s=self.config["iter_time"],
info={})
def get_agent_class(alg):
@@ -363,5 +369,4 @@ def get_agent_class(alg):
elif alg == "__parameter_tuning":
return _ParameterTuningAgent
else:
raise Exception(
("Unknown algorithm {}.").format(alg))
raise Exception(("Unknown algorithm {}.").format(alg))
python/ray/rllib/agents/bc/bc.py
+12 -9
View File
@@ -57,28 +57,31 @@ class BCAgent(Agent):
else:
num_gpus_per_worker = 0
return Resources(
cpu=1, gpu=cf["gpu"] and 1 or 0,
cpu=1,
gpu=cf["gpu"] and 1 or 0,
extra_cpu=cf["num_workers"],
extra_gpu=num_gpus_per_worker * cf["num_workers"])
def _init(self):
self.local_evaluator = BCEvaluator(
self.env_creator, self.config, self.logdir)
self.local_evaluator = BCEvaluator(self.env_creator, self.config,
self.logdir)
if self.config["use_gpu_for_workers"]:
remote_cls = GPURemoteBCEvaluator
else:
remote_cls = RemoteBCEvaluator
self.remote_evaluators = [
remote_cls.remote(self.env_creator, self.config, self.logdir)
for _ in range(self.config["num_workers"])]
self.optimizer = AsyncGradientsOptimizer(
self.local_evaluator, self.remote_evaluators,
self.config["optimizer"])
for _ in range(self.config["num_workers"])
]
self.optimizer = AsyncGradientsOptimizer(self.local_evaluator,
self.remote_evaluators,
self.config["optimizer"])
def _train(self):
self.optimizer.step()
metric_lists = [re.get_metrics.remote() for re in
self.remote_evaluators]
metric_lists = [
re.get_metrics.remote() for re in self.remote_evaluators
]
total_samples = 0
total_loss = 0
for metrics in metric_lists:
python/ray/rllib/agents/bc/bc_evaluator.py
+7 -6
View File
@@ -14,8 +14,8 @@ from ray.rllib.models import ModelCatalog
class BCEvaluator(EvaluatorInterface):
def __init__(self, env_creator, config, logdir):
env = ModelCatalog.get_preprocessor_as_wrapper(env_creator(
config["env_config"]), config["model"])
env = ModelCatalog.get_preprocessor_as_wrapper(
env_creator(config["env_config"]), config["model"])
self.dataset = ExperienceDataset(config["dataset_path"])
self.policy = BCPolicy(env.observation_space, env.action_space, config)
self.config = config
@@ -27,8 +27,10 @@ class BCEvaluator(EvaluatorInterface):
def compute_gradients(self, samples):
gradient, info = self.policy.compute_gradients(samples)
self.metrics_queue.put(
{"num_samples": info["num_samples"], "loss": info["loss"]})
self.metrics_queue.put({
"num_samples": info["num_samples"],
"loss": info["loss"]
})
return gradient, {}
def apply_gradients(self, grads):
@@ -42,8 +44,7 @@ class BCEvaluator(EvaluatorInterface):
def save(self):
weights = self.get_weights()
return pickle.dumps({
"weights": weights})
return pickle.dumps({"weights": weights})
def restore(self, objs):
objs = pickle.loads(objs)
python/ray/rllib/agents/bc/experience_dataset.py
+3 -2
View File
@@ -21,8 +21,9 @@ class ExperienceDataset(object):
elements.
The file must be available on each machine used by a BCEvaluator.
"""
self._dataset = list(itertools.chain.from_iterable(
pickle.load(open(dataset_path, "rb"))))
self._dataset = list(
itertools.chain.from_iterable(
pickle.load(open(dataset_path, "rb"))))
def sample(self, batch_size):
indexes = np.random.choice(len(self._dataset), batch_size)
python/ray/rllib/agents/bc/policy.py
+19 -18
View File
@@ -23,8 +23,8 @@ class BCPolicy(object):
self.x = tf.placeholder(tf.float32, [None] + list(obs_space.shape))
dist_class, self.logit_dim = ModelCatalog.get_action_dist(
ac_space, self.config["model"])
self._model = ModelCatalog.get_model(
self.x, self.logit_dim, self.config["model"])
self._model = ModelCatalog.get_model(self.x, self.logit_dim,
self.config["model"])
self.logits = self._model.outputs
self.curr_dist = dist_class(self.logits)
self.sample = self.curr_dist.sample()
@@ -33,17 +33,16 @@ class BCPolicy(object):
def setup_loss(self, action_space):
if isinstance(action_space, gym.spaces.Box):
self.ac = tf.placeholder(tf.float32,
[None] + list(action_space.shape),
name="ac")
self.ac = tf.placeholder(
tf.float32, [None] + list(action_space.shape), name="ac")
elif isinstance(action_space, gym.spaces.Discrete):
self.ac = tf.placeholder(tf.int64, [None], name="ac")
else:
raise NotImplementedError(
"action space" + str(type(action_space)) +
"currently not supported")
raise NotImplementedError("action space" +
str(type(action_space)) +
"currently not supported")
log_prob = self.curr_dist.logp(self.ac)
self.pi_loss = - tf.reduce_sum(log_prob)
self.pi_loss = -tf.reduce_sum(log_prob)
self.loss = self.pi_loss
def setup_gradients(self):
@@ -62,11 +61,14 @@ class BCPolicy(object):
self.summary_op = tf.summary.merge_all()
# TODO(rliaw): Can consider exposing these parameters
self.sess = tf.Session(graph=self.g, config=tf.ConfigProto(
intra_op_parallelism_threads=1, inter_op_parallelism_threads=2,
gpu_options=tf.GPUOptions(allow_growth=True)))
self.variables = ray.experimental.TensorFlowVariables(self.loss,
self.sess)
self.sess = tf.Session(
graph=self.g,
config=tf.ConfigProto(
intra_op_parallelism_threads=1,
inter_op_parallelism_threads=2,
gpu_options=tf.GPUOptions(allow_growth=True)))
self.variables = ray.experimental.TensorFlowVariables(
self.loss, self.sess)
self.sess.run(tf.global_variables_initializer())
def compute_gradients(self, samples):
@@ -82,15 +84,14 @@ class BCPolicy(object):
[self.loss, self.grads, self.summary_op], feed_dict=feed_dict)
info["summary"] = summ
else:
loss, grad = self.sess.run([self.loss, self.grads],
feed_dict=feed_dict)
loss, grad = self.sess.run(
[self.loss, self.grads], feed_dict=feed_dict)
info["num_samples"] = len(samples)
info["loss"] = loss
return grad, info
def apply_gradients(self, grads):
feed_dict = {self.grads[i]: grads[i]
for i in range(len(grads))}
feed_dict = {self.grads[i]: grads[i] for i in range(len(grads))}
self.sess.run(self._apply_gradients, feed_dict=feed_dict)
def get_weights(self):
python/ray/rllib/agents/ddpg/apex.py
+6 -7
View File
@@ -9,13 +9,12 @@ APEX_DDPG_DEFAULT_CONFIG = merge_dicts(
DDPG_CONFIG,
{
"optimizer_class": "AsyncSamplesOptimizer",
"optimizer":
merge_dicts(
DDPG_CONFIG["optimizer"], {
"max_weight_sync_delay": 400,
"num_replay_buffer_shards": 4,
"debug": False
}),
"optimizer": merge_dicts(
DDPG_CONFIG["optimizer"], {
"max_weight_sync_delay": 400,
"num_replay_buffer_shards": 4,
"debug": False
}),
"n_step": 3,
"num_workers": 32,
"buffer_size": 2000000,
python/ray/rllib/agents/ddpg/ddpg.py
+3 -3
View File
@@ -118,9 +118,9 @@ class DDPGAgent(DQNAgent):
if self.config["per_worker_exploration"]:
assert self.config["num_workers"] > 1, \
"This requires multiple workers"
return ConstantSchedule(
self.config["noise_scale"] * 0.4 **
(1 + worker_index / float(self.config["num_workers"] - 1) * 7))
exponent = (
1 + worker_index / float(self.config["num_workers"] - 1) * 7)
return ConstantSchedule(self.config["noise_scale"] * 0.4**exponent)
else:
return LinearSchedule(
schedule_timesteps=int(self.config["exploration_fraction"] *
python/ray/rllib/agents/ddpg/ddpg_policy_graph.py
+44 -35
View File
@@ -14,7 +14,6 @@ from ray.rllib.models import ModelCatalog
from ray.rllib.utils.error import UnsupportedSpaceException
from ray.rllib.evaluation.tf_policy_graph import TFPolicyGraph
A_SCOPE = "a_func"
P_SCOPE = "p_func"
P_TARGET_SCOPE = "target_p_func"
@@ -26,8 +25,8 @@ class PNetwork(object):
"""Maps an observations (i.e., state) to an action where each entry takes
value from (0, 1) due to the sigmoid function."""
def __init__(
self, model, dim_actions, hiddens=[64, 64], activation="relu"):
def __init__(self, model, dim_actions, hiddens=[64, 64],
activation="relu"):
action_out = model.last_layer
activation = tf.nn.__dict__[activation]
for hidden in hiddens:
@@ -44,9 +43,14 @@ class ActionNetwork(object):
for training, thus ignoring the batch_size issue when constructing a
stochastic action."""
def __init__(
self, p_values, low_action, high_action, stochastic, eps,
theta=0.15, sigma=0.2):
def __init__(self,
p_values,
low_action,
high_action,
stochastic,
eps,
theta=0.15,
sigma=0.2):
# shape is [None, dim_action]
deterministic_actions = (
@@ -65,15 +69,16 @@ class ActionNetwork(object):
stochastic_actions = deterministic_actions + eps * (
high_action - low_action) * exploration_value
self.actions = tf.cond(
stochastic, lambda: stochastic_actions,
lambda: deterministic_actions)
self.actions = tf.cond(stochastic, lambda: stochastic_actions,
lambda: deterministic_actions)
class QNetwork(object):
def __init__(
self, model, action_inputs,
hiddens=[64, 64], activation="relu"):
def __init__(self,
model,
action_inputs,
hiddens=[64, 64],
activation="relu"):
q_out = tf.concat([model.last_layer, action_inputs], axis=1)
activation = tf.nn.__dict__[activation]
for hidden in hiddens:
@@ -84,14 +89,21 @@ class QNetwork(object):
class ActorCriticLoss(object):
def __init__(
self, q_t, q_tp1, q_tp0, importance_weights, rewards, done_mask,
gamma=0.99, n_step=1, use_huber=False, huber_threshold=1.0):
def __init__(self,
q_t,
q_tp1,
q_tp0,
importance_weights,
rewards,
done_mask,
gamma=0.99,
n_step=1,
use_huber=False,
huber_threshold=1.0):
q_t_selected = tf.squeeze(q_t, axis=len(q_t.shape) - 1)
q_tp1_best = tf.squeeze(
input=q_tp1, axis=len(q_tp1.shape) - 1)
q_tp1_best = tf.squeeze(input=q_tp1, axis=len(q_tp1.shape) - 1)
q_tp1_best_masked = (1.0 - done_mask) * q_tp1_best
# compute RHS of bellman equation
@@ -131,27 +143,20 @@ class DDPGPolicyGraph(TFPolicyGraph):
def _build_q_network(obs, actions):
return QNetwork(
ModelCatalog.get_model(obs, 1, config["model"]),
actions,
ModelCatalog.get_model(obs, 1, config["model"]), actions,
config["critic_hiddens"],
config["critic_hidden_activation"]).value
def _build_p_network(obs):
return PNetwork(
ModelCatalog.get_model(obs, 1, config["model"]),
dim_actions,
ModelCatalog.get_model(obs, 1, config["model"]), dim_actions,
config["actor_hiddens"],
config["actor_hidden_activation"]).action_scores
def _build_action_network(p_values, stochastic, eps):
return ActionNetwork(
p_values,
low_action,
high_action,
stochastic,
eps,
config["exploration_theta"],
config["exploration_sigma"]).actions
return ActionNetwork(p_values, low_action, high_action, stochastic,
eps, config["exploration_theta"],
config["exploration_sigma"]).actions
# Action inputs
self.stochastic = tf.placeholder(tf.bool, (), name="stochastic")
@@ -263,9 +268,13 @@ class DDPGPolicyGraph(TFPolicyGraph):
("weights", self.importance_weights),
]
TFPolicyGraph.__init__(
self, observation_space, action_space, self.sess,
self,
observation_space,
action_space,
self.sess,
obs_input=self.cur_observations,
action_sampler=self.output_actions, loss=self.loss.total_loss,
action_sampler=self.output_actions,
loss=self.loss.total_loss,
loss_inputs=self.loss_inputs)
self.sess.run(tf.global_variables_initializer())
@@ -294,10 +303,10 @@ class DDPGPolicyGraph(TFPolicyGraph):
self.loss.actor_loss, var_list=self.p_func_vars)
critic_grads_and_vars = self.critic_optimizer.compute_gradients(
self.loss.critic_loss, var_list=self.q_func_vars)
actor_grads_and_vars = [
(g, v) for (g, v) in actor_grads_and_vars if g is not None]
critic_grads_and_vars = [
(g, v) for (g, v) in critic_grads_and_vars if g is not None]
actor_grads_and_vars = [(g, v) for (g, v) in actor_grads_and_vars
if g is not None]
critic_grads_and_vars = [(g, v) for (g, v) in critic_grads_and_vars
if g is not None]
grads_and_vars = actor_grads_and_vars + critic_grads_and_vars
return grads_and_vars
python/ray/rllib/agents/dqn/apex.py
+6 -7
View File
@@ -10,13 +10,12 @@ APEX_DEFAULT_CONFIG = merge_dicts(
DQN_CONFIG,
{
"optimizer_class": "AsyncSamplesOptimizer",
"optimizer":
merge_dicts(
DQN_CONFIG["optimizer"], {
"max_weight_sync_delay": 400,
"num_replay_buffer_shards": 4,
"debug": False
}),
"optimizer": merge_dicts(
DQN_CONFIG["optimizer"], {
"max_weight_sync_delay": 400,
"num_replay_buffer_shards": 4,
"debug": False
}),
"n_step": 3,
"gpu": True,
"num_workers": 32,
python/ray/rllib/agents/dqn/dqn.py
+25 -23
View File
@@ -13,11 +13,11 @@ from ray.rllib.evaluation.metrics import collect_metrics
from ray.rllib.utils.schedules import ConstantSchedule, LinearSchedule
from ray.tune.trial import Resources
OPTIMIZER_SHARED_CONFIGS = [
"buffer_size", "prioritized_replay", "prioritized_replay_alpha",
"prioritized_replay_beta", "prioritized_replay_eps", "sample_batch_size",
"train_batch_size", "learning_starts", "clip_rewards"]
"train_batch_size", "learning_starts", "clip_rewards"
]
DEFAULT_CONFIG = with_common_config({
# === Model ===
@@ -110,7 +110,8 @@ class DQNAgent(Agent):
def default_resource_request(cls, config):
cf = dict(cls._default_config, **config)
return Resources(
cpu=1, gpu=cf["gpu"] and 1 or 0,
cpu=1,
gpu=cf["gpu"] and 1 or 0,
extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"],
extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"])
@@ -123,7 +124,8 @@ class DQNAgent(Agent):
self.exploration0 = self._make_exploration_schedule(0)
self.explorations = [
self._make_exploration_schedule(i)
for i in range(self.config["num_workers"])]
for i in range(self.config["num_workers"])
]
for k in OPTIMIZER_SHARED_CONFIGS:
if k not in self.config["optimizer"]:
@@ -132,9 +134,10 @@ class DQNAgent(Agent):
self.local_evaluator = self.make_local_evaluator(
self.env_creator, self._policy_graph)
self.remote_evaluators = self.make_remote_evaluators(
self.env_creator, self._policy_graph, self.config["num_workers"],
{"num_cpus": self.config["num_cpus_per_worker"],
"num_gpus": self.config["num_gpus_per_worker"]})
self.env_creator, self._policy_graph, self.config["num_workers"], {
"num_cpus": self.config["num_cpus_per_worker"],
"num_gpus": self.config["num_gpus_per_worker"]
})
self.optimizer = getattr(optimizers, self.config["optimizer_class"])(
self.local_evaluator, self.remote_evaluators,
self.config["optimizer"])
@@ -147,14 +150,12 @@ class DQNAgent(Agent):
if self.config["per_worker_exploration"]:
assert self.config["num_workers"] > 1, \
"This requires multiple workers"
return ConstantSchedule(
0.4 ** (
1 + worker_index / float(
self.config["num_workers"] - 1) * 7))
exponent = (
1 + worker_index / float(self.config["num_workers"] - 1) * 7)
return ConstantSchedule(0.4**exponent)
return LinearSchedule(
schedule_timesteps=int(
self.config["exploration_fraction"] *
self.config["schedule_max_timesteps"]),
schedule_timesteps=int(self.config["exploration_fraction"] *
self.config["schedule_max_timesteps"]),
initial_p=1.0,
final_p=self.config["exploration_final_eps"])
@@ -191,8 +192,8 @@ class DQNAgent(Agent):
self.local_evaluator,
self.remote_evaluators[-len(self.remote_evaluators) // 3:])
else:
result = collect_metrics(
self.local_evaluator, self.remote_evaluators)
result = collect_metrics(self.local_evaluator,
self.remote_evaluators)
return result._replace(
timesteps_this_iter=self.global_timestep - start_timestep,
@@ -208,14 +209,14 @@ class DQNAgent(Agent):
ev.__ray_terminate__.remote()
def _save(self, checkpoint_dir):
checkpoint_path = os.path.join(
checkpoint_dir, "checkpoint-{}".format(self.iteration))
checkpoint_path = os.path.join(checkpoint_dir,
"checkpoint-{}".format(self.iteration))
extra_data = [
self.local_evaluator.save(),
ray.get([e.save.remote() for e in self.remote_evaluators]),
self.optimizer.save(),
self.num_target_updates,
self.last_target_update_ts]
self.optimizer.save(), self.num_target_updates,
self.last_target_update_ts
]
pickle.dump(extra_data, open(checkpoint_path + ".extra_data", "wb"))
return checkpoint_path
@@ -223,8 +224,9 @@ class DQNAgent(Agent):
extra_data = pickle.load(open(checkpoint_path + ".extra_data", "rb"))
self.local_evaluator.restore(extra_data[0])
ray.get([
e.restore.remote(d) for (d, e)
in zip(extra_data[1], self.remote_evaluators)])
e.restore.remote(d)
for (d, e) in zip(extra_data[1], self.remote_evaluators)
])
self.optimizer.restore(extra_data[2])
self.num_target_updates = extra_data[3]
self.last_target_update_ts = extra_data[4]
python/ray/rllib/agents/dqn/dqn_policy_graph.py
+62 -50
View File
@@ -13,7 +13,6 @@ from ray.rllib.evaluation.sample_batch import SampleBatch
from ray.rllib.utils.error import UnsupportedSpaceException
from ray.rllib.evaluation.tf_policy_graph import TFPolicyGraph
Q_SCOPE = "q_func"
Q_TARGET_SCOPE = "target_q_func"
@@ -33,7 +32,8 @@ class QNetwork(object):
state_out = model.last_layer
for hidden in hiddens:
state_out = layers.fully_connected(
state_out, num_outputs=hidden,
state_out,
num_outputs=hidden,
activation_fn=tf.nn.relu)
state_score = layers.fully_connected(
state_out, num_outputs=1, activation_fn=None)
@@ -50,26 +50,32 @@ class QValuePolicy(object):
deterministic_actions = tf.argmax(q_values, axis=1)
batch_size = tf.shape(observations)[0]
random_actions = tf.random_uniform(
tf.stack([batch_size]), minval=0, maxval=num_actions,
tf.stack([batch_size]),
minval=0,
maxval=num_actions,
dtype=tf.int64)
chose_random = tf.random_uniform(
tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps
stochastic_actions = tf.where(
chose_random, random_actions, deterministic_actions)
self.action = tf.cond(
stochastic, lambda: stochastic_actions,
lambda: deterministic_actions)
stochastic_actions = tf.where(chose_random, random_actions,
deterministic_actions)
self.action = tf.cond(stochastic, lambda: stochastic_actions,
lambda: deterministic_actions)
class QLoss(object):
def __init__(
self, q_t_selected, q_tp1_best, importance_weights, rewards,
done_mask, gamma=0.99, n_step=1):
def __init__(self,
q_t_selected,
q_tp1_best,
importance_weights,
rewards,
done_mask,
gamma=0.99,
n_step=1):
q_tp1_best_masked = (1.0 - done_mask) * q_tp1_best
# compute RHS of bellman equation
q_t_selected_target = rewards + gamma ** n_step * q_tp1_best_masked
q_t_selected_target = rewards + gamma**n_step * q_tp1_best_masked
# compute the error (potentially clipped)
self.td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)
@@ -91,14 +97,14 @@ class DQNPolicyGraph(TFPolicyGraph):
def _build_q_network(obs):
return QNetwork(
ModelCatalog.get_model(obs, 1, config["model"]),
num_actions, config["dueling"], config["hiddens"]).value
ModelCatalog.get_model(obs, 1, config["model"]), num_actions,
config["dueling"], config["hiddens"]).value
# Action inputs
self.stochastic = tf.placeholder(tf.bool, (), name="stochastic")
self.eps = tf.placeholder(tf.float32, (), name="eps")
self.cur_observations = tf.placeholder(
tf.float32, shape=(None,) + observation_space.shape)
tf.float32, shape=(None, ) + observation_space.shape)
# Action Q network
with tf.variable_scope(Q_SCOPE) as scope:
@@ -106,20 +112,17 @@ class DQNPolicyGraph(TFPolicyGraph):
self.q_func_vars = _scope_vars(scope.name)
# Action outputs
self.output_actions = QValuePolicy(
q_values,
self.cur_observations,
num_actions,
self.stochastic,
self.eps).action
self.output_actions = QValuePolicy(q_values, self.cur_observations,
num_actions, self.stochastic,
self.eps).action
# Replay inputs
self.obs_t = tf.placeholder(
tf.float32, shape=(None,) + observation_space.shape)
tf.float32, shape=(None, ) + observation_space.shape)
self.act_t = tf.placeholder(tf.int32, [None], name="action")
self.rew_t = tf.placeholder(tf.float32, [None], name="reward")
self.obs_tp1 = tf.placeholder(
tf.float32, shape=(None,) + observation_space.shape)
tf.float32, shape=(None, ) + observation_space.shape)
self.done_mask = tf.placeholder(tf.float32, [None], name="done")
self.importance_weights = tf.placeholder(
tf.float32, [None], name="weight")
@@ -134,8 +137,8 @@ class DQNPolicyGraph(TFPolicyGraph):
self.target_q_func_vars = _scope_vars(scope.name)
# q scores for actions which we know were selected in the given state.
q_t_selected = tf.reduce_sum(
q_t * tf.one_hot(self.act_t, num_actions), 1)
q_t_selected = tf.reduce_sum(q_t * tf.one_hot(self.act_t, num_actions),
1)
# compute estimate of best possible value starting from state at t + 1
if config["double_q"]:
@@ -143,20 +146,20 @@ class DQNPolicyGraph(TFPolicyGraph):
q_tp1_using_online_net = _build_q_network(self.obs_tp1)
q_tp1_best_using_online_net = tf.argmax(q_tp1_using_online_net, 1)
q_tp1_best = tf.reduce_sum(
q_tp1 * tf.one_hot(
q_tp1_best_using_online_net, num_actions), 1)
q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions),
1)
else:
q_tp1_best = tf.reduce_max(q_tp1, 1)
self.loss = QLoss(
q_t_selected, q_tp1_best, self.importance_weights,
self.rew_t, self.done_mask, config["gamma"], config["n_step"])
self.loss = QLoss(q_t_selected, q_tp1_best, self.importance_weights,
self.rew_t, self.done_mask, config["gamma"],
config["n_step"])
# update_target_fn will be called periodically to copy Q network to
# target Q network
update_target_expr = []
for var, var_target in zip(
sorted(self.q_func_vars, key=lambda v: v.name),
sorted(self.q_func_vars, key=lambda v: v.name),
sorted(self.target_q_func_vars, key=lambda v: v.name)):
update_target_expr.append(var_target.assign(var))
self.update_target_expr = tf.group(*update_target_expr)
@@ -172,9 +175,13 @@ class DQNPolicyGraph(TFPolicyGraph):
("weights", self.importance_weights),
]
TFPolicyGraph.__init__(
self, observation_space, action_space, self.sess,
self,
observation_space,
action_space,
self.sess,
obs_input=self.cur_observations,
action_sampler=self.output_actions, loss=self.loss.loss,
action_sampler=self.output_actions,
loss=self.loss.loss,
loss_inputs=self.loss_inputs)
self.sess.run(tf.global_variables_initializer())
@@ -184,13 +191,14 @@ class DQNPolicyGraph(TFPolicyGraph):
def gradients(self, optimizer):
if self.config["grad_norm_clipping"] is not None:
grads_and_vars = _minimize_and_clip(
optimizer, self.loss.loss, var_list=self.q_func_vars,
optimizer,
self.loss.loss,
var_list=self.q_func_vars,
clip_val=self.config["grad_norm_clipping"])
else:
grads_and_vars = optimizer.compute_gradients(
self.loss.loss, var_list=self.q_func_vars)
grads_and_vars = [
(g, v) for (g, v) in grads_and_vars if g is not None]
grads_and_vars = [(g, v) for (g, v) in grads_and_vars if g is not None]
return grads_and_vars
def extra_compute_action_feed_dict(self):
@@ -207,8 +215,8 @@ class DQNPolicyGraph(TFPolicyGraph):
def postprocess_trajectory(self, sample_batch, other_agent_batches=None):
return _postprocess_dqn(self, sample_batch)
def compute_td_error(
self, obs_t, act_t, rew_t, obs_tp1, done_mask, importance_weights):
def compute_td_error(self, obs_t, act_t, rew_t, obs_tp1, done_mask,
importance_weights):
td_err = self.sess.run(
self.loss.td_error,
feed_dict={
@@ -254,7 +262,7 @@ def adjust_nstep(n_step, gamma, obs, actions, rewards, new_obs, dones):
continue # episode end
for j in range(1, n_step):
new_obs[i] = new_obs[i + j]
rewards[i] += gamma ** j * rewards[i + j]
rewards[i] += gamma**j * rewards[i + j]
if dones[i + j]:
break # episode end
# truncate ends of the trajectory
@@ -266,24 +274,29 @@ def adjust_nstep(n_step, gamma, obs, actions, rewards, new_obs, dones):
def _postprocess_dqn(policy_graph, sample_batch):
obs, actions, rewards, new_obs, dones = [
list(x) for x in sample_batch.columns(
["obs", "actions", "rewards", "new_obs", "dones"])]
["obs", "actions", "rewards", "new_obs", "dones"])
]
# N-step Q adjustments
if policy_graph.config["n_step"] > 1:
adjust_nstep(
policy_graph.config["n_step"], policy_graph.config["gamma"],
obs, actions, rewards, new_obs, dones)
adjust_nstep(policy_graph.config["n_step"],
policy_graph.config["gamma"], obs, actions, rewards,
new_obs, dones)
batch = SampleBatch({
"obs": obs, "actions": actions, "rewards": rewards,
"new_obs": new_obs, "dones": dones,
"weights": np.ones_like(rewards)})
"obs": obs,
"actions": actions,
"rewards": rewards,
"new_obs": new_obs,
"dones": dones,
"weights": np.ones_like(rewards)
})
# Prioritize on the worker side
if batch.count > 0 and policy_graph.config["worker_side_prioritization"]:
td_errors = policy_graph.compute_td_error(
batch["obs"], batch["actions"], batch["rewards"],
batch["new_obs"], batch["dones"], batch["weights"])
batch["obs"], batch["actions"], batch["rewards"], batch["new_obs"],
batch["dones"], batch["weights"])
new_priorities = (
np.abs(td_errors) + policy_graph.config["prioritized_replay_eps"])
batch.data["weights"] = new_priorities
@@ -295,8 +308,7 @@ def _huber_loss(x, delta=1.0):
"""Reference: https://en.wikipedia.org/wiki/Huber_loss"""
return tf.where(
tf.abs(x) < delta,
tf.square(x) * 0.5,
delta * (tf.abs(x) - 0.5 * delta))
tf.square(x) * 0.5, delta * (tf.abs(x) - 0.5 * delta))
def _minimize_and_clip(optimizer, objective, var_list, clip_val=10):
python/ray/rllib/agents/es/es.py
+32 -36
View File
@@ -20,13 +20,11 @@ from ray.rllib.agents.es import policies
from ray.rllib.agents.es import tabular_logger as tlogger
from ray.rllib.agents.es import utils
Result = namedtuple("Result", [
"noise_indices", "noisy_returns", "sign_noisy_returns", "noisy_lengths",
"eval_returns", "eval_lengths"
])
DEFAULT_CONFIG = {
'l2_coeff': 0.005,
'noise_stdev': 0.02,
@@ -64,7 +62,11 @@ class SharedNoiseTable(object):
@ray.remote
class Worker(object):
def __init__(self, config, policy_params, env_creator, noise,
def __init__(self,
config,
policy_params,
env_creator,
noise,
min_task_runtime=0.2):
self.min_task_runtime = min_task_runtime
self.config = config
@@ -82,7 +84,9 @@ class Worker(object):
def rollout(self, timestep_limit, add_noise=True):
rollout_rewards, rollout_length = policies.rollout(
self.policy, self.env, timestep_limit=timestep_limit,
self.policy,
self.env,
timestep_limit=timestep_limit,
add_noise=add_noise)
return rollout_rewards, rollout_length
@@ -95,8 +99,8 @@ class Worker(object):
# Perform some rollouts with noise.
task_tstart = time.time()
while (len(noise_indices) == 0 or
time.time() - task_tstart < self.min_task_runtime):
while (len(noise_indices) == 0
or time.time() - task_tstart < self.min_task_runtime):
if np.random.uniform() < self.config["eval_prob"]:
# Do an evaluation run with no perturbation.
@@ -122,7 +126,8 @@ class Worker(object):
noise_indices.append(noise_index)
returns.append([rewards_pos.sum(), rewards_neg.sum()])
sign_returns.append(
[np.sign(rewards_pos).sum(), np.sign(rewards_neg).sum()])
[np.sign(rewards_pos).sum(),
np.sign(rewards_neg).sum()])
lengths.append([lengths_pos, lengths_neg])
return Result(
@@ -146,9 +151,7 @@ class ESAgent(Agent):
return Resources(cpu=1, gpu=0, extra_cpu=cf["num_workers"])
def _init(self):
policy_params = {
"action_noise_std": 0.01
}
policy_params = {"action_noise_std": 0.01}
env = self.env_creator(self.config["env_config"])
from ray.rllib import models
@@ -168,9 +171,9 @@ class ESAgent(Agent):
# Create the actors.
print("Creating actors.")
self.workers = [
Worker.remote(
self.config, policy_params, self.env_creator, noise_id)
for _ in range(self.config["num_workers"])]
Worker.remote(self.config, policy_params, self.env_creator,
noise_id) for _ in range(self.config["num_workers"])
]
self.episodes_so_far = 0
self.timesteps_so_far = 0
@@ -180,21 +183,20 @@ class ESAgent(Agent):
num_episodes, num_timesteps = 0, 0
results = []
while num_episodes < min_episodes or num_timesteps < min_timesteps:
print(
"Collected {} episodes {} timesteps so far this iter".format(
num_episodes, num_timesteps))
rollout_ids = [worker.do_rollouts.remote(theta_id)
for worker in self.workers]
print("Collected {} episodes {} timesteps so far this iter".format(
num_episodes, num_timesteps))
rollout_ids = [
worker.do_rollouts.remote(theta_id) for worker in self.workers
]
# Get the results of the rollouts.
for result in ray.get(rollout_ids):
results.append(result)
# Update the number of episodes and the number of timesteps
# keeping in mind that result.noisy_lengths is a list of lists,
# where the inner lists have length 2.
num_episodes += sum(len(pair) for pair
in result.noisy_lengths)
num_timesteps += sum(sum(pair) for pair
in result.noisy_lengths)
num_episodes += sum(len(pair) for pair in result.noisy_lengths)
num_timesteps += sum(
sum(pair) for pair in result.noisy_lengths)
return results, num_episodes, num_timesteps
def _train(self):
@@ -209,8 +211,7 @@ class ESAgent(Agent):
# Use the actors to do rollouts, note that we pass in the ID of the
# policy weights.
results, num_episodes, num_timesteps = self._collect_results(
theta_id,
config["episodes_per_batch"],
theta_id, config["episodes_per_batch"],
config["timesteps_per_batch"])
all_noise_indices = []
@@ -255,13 +256,11 @@ class ESAgent(Agent):
for index in noise_indices),
batch_size=500)
g /= noisy_returns.size
assert (
g.shape == (self.policy.num_params,) and
g.dtype == np.float32 and
count == len(noise_indices))
assert (g.shape == (self.policy.num_params, ) and g.dtype == np.float32
and count == len(noise_indices))
# Compute the new weights theta.
theta, update_ratio = self.optimizer.update(
-g + config["l2_coeff"] * theta)
theta, update_ratio = self.optimizer.update(-g +
config["l2_coeff"] * theta)
# Set the new weights in the local copy of the policy.
self.policy.set_weights(theta)
@@ -313,13 +312,10 @@ class ESAgent(Agent):
w.__ray_terminate__.remote()
def _save(self, checkpoint_dir):
checkpoint_path = os.path.join(
checkpoint_dir, "checkpoint-{}".format(self.iteration))
checkpoint_path = os.path.join(checkpoint_dir,
"checkpoint-{}".format(self.iteration))
weights = self.policy.get_weights()
objects = [
weights,
self.episodes_so_far,
self.timesteps_so_far]
objects = [weights, self.episodes_so_far, self.timesteps_so_far]
pickle.dump(objects, open(checkpoint_path, "wb"))
return checkpoint_path
python/ray/rllib/agents/es/optimizers.py
+2 -2
View File
@@ -48,8 +48,8 @@ class Adam(Optimizer):
self.v = np.zeros(self.dim, dtype=np.float32)
def _compute_step(self, globalg):
a = self.stepsize * (np.sqrt(1 - self.beta2 ** self.t) /
(1 - self.beta1 ** self.t))
a = self.stepsize * (np.sqrt(1 - self.beta2**self.t) /
(1 - self.beta1**self.t))
self.m = self.beta1 * self.m + (1 - self.beta1) * globalg
self.v = self.beta2 * self.v + (1 - self.beta2) * (globalg * globalg)
step = -a * self.m / (np.sqrt(self.v) + self.epsilon)
python/ray/rllib/agents/es/policies.py
+13 -13
View File
@@ -21,8 +21,8 @@ def rollout(policy, env, timestep_limit=None, add_noise=False):
noise drawn from that stream. Otherwise, no action noise will be added.
"""
env_timestep_limit = env.spec.max_episode_steps
timestep_limit = (env_timestep_limit if timestep_limit is None
else min(timestep_limit, env_timestep_limit))
timestep_limit = (env_timestep_limit if timestep_limit is None else min(
timestep_limit, env_timestep_limit))
rews = []
t = 0
observation = env.reset()
@@ -38,16 +38,16 @@ def rollout(policy, env, timestep_limit=None, add_noise=False):
class GenericPolicy(object):
def __init__(self, sess, action_space, preprocessor,
observation_filter, action_noise_std):
def __init__(self, sess, action_space, preprocessor, observation_filter,
action_noise_std):
self.sess = sess
self.action_space = action_space
self.action_noise_std = action_noise_std
self.preprocessor = preprocessor
self.observation_filter = get_filter(
observation_filter, self.preprocessor.shape)
self.inputs = tf.placeholder(
tf.float32, [None] + list(self.preprocessor.shape))
self.observation_filter = get_filter(observation_filter,
self.preprocessor.shape)
self.inputs = tf.placeholder(tf.float32,
[None] + list(self.preprocessor.shape))
# Policy network.
dist_class, dist_dim = ModelCatalog.get_action_dist(
@@ -59,16 +59,16 @@ class GenericPolicy(object):
self.variables = ray.experimental.TensorFlowVariables(
model.outputs, self.sess)
self.num_params = sum(np.prod(variable.shape.as_list())
for _, variable
in self.variables.variables.items())
self.num_params = sum(
np.prod(variable.shape.as_list())
for _, variable in self.variables.variables.items())
self.sess.run(tf.global_variables_initializer())
def compute(self, observation, add_noise=False, update=True):
observation = self.preprocessor.transform(observation)
observation = self.observation_filter(observation[None], update=update)
action = self.sess.run(self.sampler,
feed_dict={self.inputs: observation})
action = self.sess.run(
self.sampler, feed_dict={self.inputs: observation})
if add_noise and isinstance(self.action_space, gym.spaces.Box):
action += np.random.randn(*action.shape) * self.action_noise_std
return action
python/ray/rllib/agents/es/tabular_logger.py
+10 -6
View File
@@ -25,6 +25,7 @@ DISABLED = 50
class TbWriter(object):
"""Based on SummaryWriter, but changed to allow for a different prefix."""
def __init__(self, dir, prefix):
self.dir = dir
# Start at 1, because EvWriter automatically generates an object with
@@ -34,9 +35,10 @@ class TbWriter(object):
compat.as_bytes(os.path.join(dir, prefix)))
def write_values(self, key2val):
summary = tf.Summary(value=[tf.Summary.Value(tag=k,
simple_value=float(v))
for (k, v) in key2val.items()])
summary = tf.Summary(value=[
tf.Summary.Value(tag=k, simple_value=float(v))
for (k, v) in key2val.items()
])
event = event_pb2.Event(wall_time=time.time(), summary=summary)
event.step = self.step
self.evwriter.WriteEvent(event)
@@ -46,6 +48,7 @@ class TbWriter(object):
def close(self):
self.evwriter.Close()
# API
@@ -126,6 +129,7 @@ def get_expt_dir():
sys.stderr.write("get_expt_dir() is Deprecated. Switch to get_dir()\n")
return get_dir()
# Backend
@@ -167,8 +171,8 @@ class _Logger(object):
# Write to all text outputs
self._write_text("-" * (keywidth + valwidth + 7), "\n")
for (key, val) in key2str.items():
self._write_text("| ", key, " " * (keywidth - len(key)),
" | ", val, " " * (valwidth - len(val)), " |\n")
self._write_text("| ", key, " " * (keywidth - len(key)), " | ",
val, " " * (valwidth - len(val)), " |\n")
self._write_text("-" * (keywidth + valwidth + 7), "\n")
for f in self.text_outputs:
try:
@@ -202,7 +206,7 @@ class _Logger(object):
# Misc
def _do_log(self, *args):
self._write_text(*args + ('\n',))
self._write_text(*args + ('\n', ))
for f in self.text_outputs:
try:
f.flush()
python/ray/rllib/agents/es/utils.py
+8 -6
View File
@@ -31,8 +31,9 @@ def compute_centered_ranks(x):
def make_session(single_threaded):
if not single_threaded:
return tf.Session()
return tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1))
return tf.Session(
config=tf.ConfigProto(
inter_op_parallelism_threads=1, intra_op_parallelism_threads=1))
def itergroups(items, group_size):
@@ -50,10 +51,11 @@ def itergroups(items, group_size):
def batched_weighted_sum(weights, vecs, batch_size):
total = 0
num_items_summed = 0
for batch_weights, batch_vecs in zip(itergroups(weights, batch_size),
itergroups(vecs, batch_size)):
for batch_weights, batch_vecs in zip(
itergroups(weights, batch_size), itergroups(vecs, batch_size)):
assert len(batch_weights) == len(batch_vecs) <= batch_size
total += np.dot(np.asarray(batch_weights, dtype=np.float32),
np.asarray(batch_vecs, dtype=np.float32))
total += np.dot(
np.asarray(batch_weights, dtype=np.float32),
np.asarray(batch_vecs, dtype=np.float32))
num_items_summed += len(batch_weights)
return total, num_items_summed
python/ray/rllib/agents/pg/pg.py
+3 -4
View File
@@ -7,7 +7,6 @@ from ray.rllib.agents.pg.pg_policy_graph import PGPolicyGraph
from ray.rllib.optimizers import SyncSamplesOptimizer
from ray.tune.trial import Resources
DEFAULT_CONFIG = with_common_config({
# No remote workers by default
"num_workers": 0,
@@ -43,9 +42,9 @@ class PGAgent(Agent):
self.env_creator, PGPolicyGraph)
self.remote_evaluators = self.make_remote_evaluators(
self.env_creator, PGPolicyGraph, self.config["num_workers"], {})
self.optimizer = SyncSamplesOptimizer(
self.local_evaluator, self.remote_evaluators,
self.config["optimizer"])
self.optimizer = SyncSamplesOptimizer(self.local_evaluator,
self.remote_evaluators,
self.config["optimizer"])
def _train(self):
prev_steps = self.optimizer.num_steps_sampled
python/ray/rllib/agents/pg/pg_policy_graph.py
+9 -3
View File
@@ -42,9 +42,15 @@ class PGPolicyGraph(TFPolicyGraph):
]
TFPolicyGraph.__init__(
self, obs_space, action_space, sess, obs_input=obs,
action_sampler=action_dist.sample(), loss=loss,
loss_inputs=loss_in, state_inputs=self.model.state_in,
self,
obs_space,
action_space,
sess,
obs_input=obs,
action_sampler=action_dist.sample(),
loss=loss,
loss_inputs=loss_in,
state_inputs=self.model.state_in,
state_outputs=self.model.state_out,
seq_lens=self.model.seq_lens,
max_seq_len=config["model"]["max_seq_len"])
python/ray/rllib/agents/ppo/ppo.py
+16 -15
View File
@@ -77,28 +77,30 @@ class PPOAgent(Agent):
self.local_evaluator = self.make_local_evaluator(
self.env_creator, PPOPolicyGraph)
self.remote_evaluators = self.make_remote_evaluators(
self.env_creator, PPOPolicyGraph, self.config["num_workers"],
{"num_cpus": self.config["num_cpus_per_worker"],
"num_gpus": self.config["num_gpus_per_worker"]})
self.env_creator, PPOPolicyGraph, self.config["num_workers"], {
"num_cpus": self.config["num_cpus_per_worker"],
"num_gpus": self.config["num_gpus_per_worker"]
})
if self.config["simple_optimizer"]:
self.optimizer = SyncSamplesOptimizer(
self.local_evaluator, self.remote_evaluators,
{"num_sgd_iter": self.config["num_sgd_iter"]})
else:
self.optimizer = LocalMultiGPUOptimizer(
self.local_evaluator, self.remote_evaluators,
{"sgd_batch_size": self.config["sgd_batchsize"],
"sgd_stepsize": self.config["sgd_stepsize"],
"num_sgd_iter": self.config["num_sgd_iter"],
"timesteps_per_batch": self.config["timesteps_per_batch"],
"standardize_fields": ["advantages"]})
self.local_evaluator, self.remote_evaluators, {
"sgd_batch_size": self.config["sgd_batchsize"],
"sgd_stepsize": self.config["sgd_stepsize"],
"num_sgd_iter": self.config["num_sgd_iter"],
"timesteps_per_batch": self.config["timesteps_per_batch"],
"standardize_fields": ["advantages"]
})
def _train(self):
prev_steps = self.optimizer.num_steps_sampled
fetches = self.optimizer.step()
self.local_evaluator.for_policy(lambda pi: pi.update_kl(fetches["kl"]))
FilterManager.synchronize(
self.local_evaluator.filters, self.remote_evaluators)
FilterManager.synchronize(self.local_evaluator.filters,
self.remote_evaluators)
res = self.optimizer.collect_metrics()
res = res._replace(
timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps,
@@ -115,9 +117,7 @@ class PPOAgent(Agent):
"checkpoint-{}".format(self.iteration))
agent_state = ray.get(
[a.save.remote() for a in self.remote_evaluators])
extra_data = [
self.local_evaluator.save(),
agent_state]
extra_data = [self.local_evaluator.save(), agent_state]
pickle.dump(extra_data, open(checkpoint_path + ".extra_data", "wb"))
return checkpoint_path
@@ -126,4 +126,5 @@ class PPOAgent(Agent):
self.local_evaluator.restore(extra_data[0])
ray.get([
a.restore.remote(o)
for (a, o) in zip(self.remote_evaluators, extra_data[1])])
for (a, o) in zip(self.remote_evaluators, extra_data[1])
])
python/ray/rllib/agents/ppo/ppo_policy_graph.py
+71 -34
View File
@@ -10,10 +10,20 @@ from ray.rllib.models.catalog import ModelCatalog
class PPOLoss(object):
def __init__(
self, action_space, value_targets, advantages, actions, logits,
vf_preds, curr_action_dist, value_fn, cur_kl_coeff,
entropy_coeff=0, clip_param=0.1, vf_loss_coeff=1.0, use_gae=True):
def __init__(self,
action_space,
value_targets,
advantages,
actions,
logits,
vf_preds,
curr_action_dist,
value_fn,
cur_kl_coeff,
entropy_coeff=0,
clip_param=0.1,
vf_loss_coeff=1.0,
use_gae=True):
"""Constructs the loss for Proximal Policy Objective.
Arguments:
@@ -51,31 +61,33 @@ class PPOLoss(object):
surrogate_loss = tf.minimum(
advantages * logp_ratio,
advantages * tf.clip_by_value(
logp_ratio, 1 - clip_param, 1 + clip_param))
advantages * tf.clip_by_value(logp_ratio, 1 - clip_param,
1 + clip_param))
self.mean_policy_loss = tf.reduce_mean(-surrogate_loss)
if use_gae:
vf_loss1 = tf.square(value_fn - value_targets)
vf_clipped = vf_preds + tf.clip_by_value(
value_fn - vf_preds, -clip_param, clip_param)
vf_clipped = vf_preds + tf.clip_by_value(value_fn - vf_preds,
-clip_param, clip_param)
vf_loss2 = tf.square(vf_clipped - value_targets)
vf_loss = tf.maximum(vf_loss1, vf_loss2)
self.mean_vf_loss = tf.reduce_mean(vf_loss)
loss = tf.reduce_mean(
-surrogate_loss + cur_kl_coeff*action_kl +
vf_loss_coeff*vf_loss - entropy_coeff*curr_entropy)
loss = tf.reduce_mean(-surrogate_loss + cur_kl_coeff * action_kl +
vf_loss_coeff * vf_loss -
entropy_coeff * curr_entropy)
else:
self.mean_vf_loss = tf.constant(0.0)
loss = tf.reduce_mean(
-surrogate_loss + cur_kl_coeff*action_kl -
entropy_coeff*curr_entropy)
loss = tf.reduce_mean(-surrogate_loss + cur_kl_coeff * action_kl -
entropy_coeff * curr_entropy)
self.loss = loss
class PPOPolicyGraph(TFPolicyGraph):
def __init__(self, observation_space, action_space,
config, existing_inputs=None):
def __init__(self,
observation_space,
action_space,
config,
existing_inputs=None):
"""
Arguments:
observation_space: Environment observation space specification.
@@ -98,16 +110,18 @@ class PPOPolicyGraph(TFPolicyGraph):
existing_seq_lens = existing_inputs[-1]
else:
obs_ph = tf.placeholder(
tf.float32, name="obs", shape=(None,)+observation_space.shape)
tf.float32,
name="obs",
shape=(None, ) + observation_space.shape)
adv_ph = tf.placeholder(
tf.float32, name="advantages", shape=(None,))
tf.float32, name="advantages", shape=(None, ))
act_ph = ModelCatalog.get_action_placeholder(action_space)
logits_ph = tf.placeholder(
tf.float32, name="logits", shape=(None, logit_dim))
vf_preds_ph = tf.placeholder(
tf.float32, name="vf_preds", shape=(None,))
tf.float32, name="vf_preds", shape=(None, ))
value_targets_ph = tf.placeholder(
tf.float32, name="value_targets", shape=(None,))
tf.float32, name="value_targets", shape=(None, ))
existing_state_in = None
existing_seq_lens = None
@@ -120,13 +134,19 @@ class PPOPolicyGraph(TFPolicyGraph):
("vf_preds", vf_preds_ph),
]
self.model = ModelCatalog.get_model(
obs_ph, logit_dim, self.config["model"],
state_in=existing_state_in, seq_lens=existing_seq_lens)
obs_ph,
logit_dim,
self.config["model"],
state_in=existing_state_in,
seq_lens=existing_seq_lens)
# KL Coefficient
self.kl_coeff = tf.get_variable(
initializer=tf.constant_initializer(self.kl_coeff_val),
name="kl_coeff", shape=(), trainable=False, dtype=tf.float32)
name="kl_coeff",
shape=(),
trainable=False,
dtype=tf.float32)
self.logits = self.model.outputs
curr_action_dist = dist_cls(self.logits)
@@ -146,20 +166,32 @@ class PPOPolicyGraph(TFPolicyGraph):
self.value_function = tf.zeros(shape=tf.shape(obs_ph)[:1])
self.loss_obj = PPOLoss(
action_space, value_targets_ph, adv_ph, act_ph,
logits_ph, vf_preds_ph,
curr_action_dist, self.value_function, self.kl_coeff,
action_space,
value_targets_ph,
adv_ph,
act_ph,
logits_ph,
vf_preds_ph,
curr_action_dist,
self.value_function,
self.kl_coeff,
entropy_coeff=self.config["entropy_coeff"],
clip_param=self.config["clip_param"],
vf_loss_coeff=self.config["kl_target"],
use_gae=self.config["use_gae"])
TFPolicyGraph.__init__(
self, observation_space, action_space,
self.sess, obs_input=obs_ph,
action_sampler=self.sampler, loss=self.loss_obj.loss,
loss_inputs=self.loss_in, state_inputs=self.model.state_in,
state_outputs=self.model.state_out, seq_lens=self.model.seq_lens,
self,
observation_space,
action_space,
self.sess,
obs_input=obs_ph,
action_sampler=self.sampler,
loss=self.loss_obj.loss,
loss_inputs=self.loss_in,
state_inputs=self.model.state_in,
state_outputs=self.model.state_out,
seq_lens=self.model.seq_lens,
max_seq_len=config["model"]["max_seq_len"])
self.sess.run(tf.global_variables_initializer())
@@ -167,7 +199,9 @@ class PPOPolicyGraph(TFPolicyGraph):
def copy(self, existing_inputs):
"""Creates a copy of self using existing input placeholders."""
return PPOPolicyGraph(
None, self.action_space, self.config,
None,
self.action_space,
self.config,
existing_inputs=existing_inputs)
def extra_compute_action_fetches(self):
@@ -193,8 +227,11 @@ class PPOPolicyGraph(TFPolicyGraph):
def postprocess_trajectory(self, sample_batch, other_agent_batches=None):
last_r = 0.0
batch = compute_advantages(
sample_batch, last_r, self.config["gamma"],
self.config["lambda"], use_gae=self.config["use_gae"])
sample_batch,
last_r,
self.config["gamma"],
self.config["lambda"],
use_gae=self.config["use_gae"])
return batch
def optimizer(self):
python/ray/rllib/agents/ppo/test/test.py
+4 -4
View File
@@ -13,7 +13,6 @@ from ray.rllib.agents.ppo.utils import flatten, concatenate
# TODO(ekl): move to rllib/models dir
class DistributionsTest(unittest.TestCase):
def testCategorical(self):
num_samples = 100000
logits = tf.placeholder(tf.float32, shape=(None, 10))
@@ -32,10 +31,11 @@ class DistributionsTest(unittest.TestCase):
class UtilsTest(unittest.TestCase):
def testFlatten(self):
d = {"s": np.array([[[1, -1], [2, -2]], [[3, -3], [4, -4]]]),
"a": np.array([[[5], [-5]], [[6], [-6]]])}
d = {
"s": np.array([[[1, -1], [2, -2]], [[3, -3], [4, -4]]]),
"a": np.array([[[5], [-5]], [[6], [-6]]])
}
flat = flatten(d.copy(), start=0, stop=2)
assert_allclose(d["s"][0][0][:], flat["s"][0][:])
assert_allclose(d["s"][0][1][:], flat["s"][1][:])
python/ray/rllib/agents/ppo/utils.py
+1 -1
View File
@@ -16,7 +16,7 @@ def flatten(weights, start=0, stop=2):
stop: The ending index.
"""
for key, val in weights.items():
new_shape = val.shape[0:start] + (-1,) + val.shape[stop:]
new_shape = val.shape[0:start] + (-1, ) + val.shape[stop:]
weights[key] = val.reshape(new_shape)
return weights