wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-13 12:30:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

[RLlib] Exploration API: ParamNoise Integration into DQN; working example/test cases. (#7814)

Browse Source
This commit is contained in:
Sven Mika
2020-04-03 10:44:25 -07:00
committed by GitHub
parent 1b477c6dd0
commit 5537fe13b0
22 changed files with 355 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rllib/agents/a3c/a3c_torch_policy.py
+3 -4
View File
@@ -45,10 +45,9 @@ def add_advantages(policy,
else:
last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1])
return compute_advantages(sample_batch, last_r, policy.config["gamma"],
policy.config["lambda"],
policy.config["use_gae"],
policy.config["use_critic"])
return compute_advantages(
sample_batch, last_r, policy.config["gamma"], policy.config["lambda"],
policy.config["use_gae"], policy.config["use_critic"])
def model_value_predictions(policy, input_dict, state_batches, model,
rllib/agents/ddpg/ddpg.py
+3
View File
@@ -80,9 +80,12 @@ DEFAULT_CONFIG = with_common_config({
},
# Number of env steps to optimize for before returning
"timesteps_per_iteration": 1000,
# TODO(sven): Move to Exploration API's (ParameterNoise class).
# If True parameter space noise will be used for exploration
# See https://blog.openai.com/better-exploration-with-parameter-noise/
"parameter_noise": False,
# Extra configuration that disables exploration.
"evaluation_config": {
"explore": False
rllib/agents/dqn/distributional_q_model.py
+29 -21
View File
@@ -19,20 +19,25 @@ class DistributionalQModel(TFModelV2):
Note that this class by itself is not a valid model unless you
implement forward() in a subclass."""
def __init__(self,
obs_space,
action_space,
num_outputs,
model_config,
name,
q_hiddens=(256, ),
dueling=False,
num_atoms=1,
use_noisy=False,
v_min=-10.0,
v_max=10.0,
sigma0=0.5,
parameter_noise=False):
def __init__(
self,
obs_space,
action_space,
num_outputs,
model_config,
name,
q_hiddens=(256, ),
dueling=False,
num_atoms=1,
use_noisy=False,
v_min=-10.0,
v_max=10.0,
sigma0=0.5,
# TODO(sven): Move `add_layer_norm` into ModelCatalog as
# generic option, then error if we use ParameterNoise as
# Exploration type and do not have any LayerNorm layers in
# the net.
add_layer_norm=False):
"""Initialize variables of this model.
Extra model kwargs:
@@ -45,7 +50,7 @@ class DistributionalQModel(TFModelV2):
v_min (float): min value support for distributional DQN
v_max (float): max value support for distributional DQN
sigma0 (float): initial value of noisy nets
parameter_noise (bool): enable layer norm for param noise
add_layer_norm (bool): Add a LayerNorm after each layer..
Note that the core layers for forward() are not defined here, this
only defines the layers for the Q head. Those layers for forward()
@@ -66,11 +71,12 @@ class DistributionalQModel(TFModelV2):
if use_noisy:
action_out = self._noisy_layer(
"hidden_%d" % i, action_out, q_hiddens[i], sigma0)
elif parameter_noise:
elif add_layer_norm:
action_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation_fn=tf.nn.relu,
normalizer_fn=tf.keras.layers.LayerNormalization)(
activation=tf.nn.relu)(action_out)
action_out = \
tf.keras.layers.LayerNormalization()(
action_out)
else:
action_out = tf.keras.layers.Dense(
@@ -125,12 +131,14 @@ class DistributionalQModel(TFModelV2):
state_out = self._noisy_layer("dueling_hidden_%d" % i,
state_out, q_hiddens[i],
sigma0)
elif add_layer_norm:
state_out = tf.keras.layers.Dense(
units=q_hiddens[i], activation=tf.nn.relu)(state_out)
state_out = \
tf.keras.layers.LayerNormalization()(state_out)
else:
state_out = tf.keras.layers.Dense(
units=q_hiddens[i], activation=tf.nn.relu)(state_out)
if parameter_noise:
state_out = tf.keras.layers.LayerNormalization()(
state_out)
if use_noisy:
state_score = self._noisy_layer(
"dueling_output",
rllib/agents/dqn/dqn.py
+16 -43
View File
@@ -6,7 +6,6 @@ from ray.rllib.agents.dqn.dqn_policy import DQNTFPolicy
from ray.rllib.agents.dqn.simple_q_policy import SimpleQPolicy
from ray.rllib.optimizers import SyncReplayOptimizer
from ray.rllib.optimizers.replay_buffer import ReplayBuffer
from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE
from ray.rllib.utils.exploration import PerWorkerEpsilonGreedy
from ray.rllib.utils.experimental_dsl import (
@@ -59,11 +58,6 @@ DEFAULT_CONFIG = with_common_config({
"explore": False,
},
# TODO(sven): Make Exploration class for parameter noise.
# If True parameter space noise will be used for exploration
# See https://blog.openai.com/better-exploration-with-parameter-noise/
"parameter_noise": False,
# Minimum env steps to optimize for per train call. This value does
# not affect learning, only the length of iterations.
"timesteps_per_iteration": 1000,
@@ -127,6 +121,7 @@ DEFAULT_CONFIG = with_common_config({
"per_worker_exploration": DEPRECATED_VALUE,
"softmax_temp": DEPRECATED_VALUE,
"soft_q": DEPRECATED_VALUE,
"parameter_noise": DEPRECATED_VALUE,
})
# __sphinx_doc_end__
# yapf: enable
@@ -221,49 +216,27 @@ def validate_config_and_setup_param_noise(config):
"type": "SoftQ",
"temperature": config.get("softmax_temp", 1.0)
}
if config.get("parameter_noise", DEPRECATED_VALUE) != DEPRECATED_VALUE:
deprecation_warning("parameter_noise", "exploration_config={"
"type=ParameterNoise"
"}")
if config["exploration_config"]["type"] == "ParameterNoise":
if config["batch_mode"] != "complete_episodes":
logger.warning(
"ParameterNoise Exploration requires `batch_mode` to be "
"'complete_episodes'. Setting batch_mode=complete_episodes.")
config["batch_mode"] = "complete_episodes"
if config.get("noisy", False):
raise ValueError(
"ParameterNoise Exploration and `noisy` network cannot be "
"used at the same time!")
# Update effective batch size to include n-step
adjusted_batch_size = max(config["rollout_fragment_length"],
config.get("n_step", 1))
config["rollout_fragment_length"] = adjusted_batch_size
# Setup parameter noise.
if config.get("parameter_noise", False):
if config["batch_mode"] != "complete_episodes":
raise ValueError("Exploration with parameter space noise requires "
"batch_mode to be complete_episodes.")
if config.get("noisy", False):
raise ValueError("Exploration with parameter space noise and "
"noisy network cannot be used at the same time.")
start_callback = config["callbacks"].get("on_episode_start")
def on_episode_start(info):
# as a callback function to sample and pose parameter space
# noise on the parameters of network
policies = info["policy"]
for pi in policies.values():
pi.add_parameter_noise()
if start_callback is not None:
start_callback(info)
config["callbacks"]["on_episode_start"] = on_episode_start
end_callback = config["callbacks"].get("on_episode_end")
def on_episode_end(info):
# as a callback function to monitor the distance
# between noisy policy and original policy
policies = info["policy"]
episode = info["episode"]
model = policies[DEFAULT_POLICY_ID].model
if hasattr(model, "pi_distance"):
episode.custom_metrics["policy_distance"] = model.pi_distance
if end_callback is not None:
end_callback(info)
config["callbacks"]["on_episode_end"] = on_episode_end
def get_initial_state(config):
return {
rllib/agents/dqn/dqn_policy.py
+12 -35
View File
@@ -1,17 +1,16 @@
from gym.spaces import Discrete
import numpy as np
from scipy.stats import entropy
import ray
from ray.rllib.agents.dqn.distributional_q_model import DistributionalQModel
from ray.rllib.agents.dqn.simple_q_policy import TargetNetworkMixin, \
ParameterNoiseMixin
from ray.rllib.agents.dqn.simple_q_policy import TargetNetworkMixin
from ray.rllib.policy.sample_batch import SampleBatch
from ray.rllib.policy.tf_policy import LearningRateSchedule
from ray.rllib.policy.tf_policy_template import build_tf_policy
from ray.rllib.models import ModelCatalog
from ray.rllib.models.tf.tf_action_dist import Categorical
from ray.rllib.utils.error import UnsupportedSpaceException
from ray.rllib.utils.exploration import ParameterNoise
from ray.rllib.utils.tf_ops import huber_loss, reduce_mean_ignore_inf, \
minimize_and_clip
from ray.rllib.utils import try_import_tf
@@ -124,34 +123,6 @@ class ComputeTDErrorMixin:
self.compute_td_error = compute_td_error
def postprocess_trajectory(policy,
sample_batch,
other_agent_batches=None,
episode=None):
if policy.config["parameter_noise"]:
# adjust the sigma of parameter space noise
states = [list(x) for x in sample_batch.columns(["obs"])][0]
noisy_action_distribution = policy.get_session().run(
policy.action_probs, feed_dict={policy.cur_observations: states})
policy.get_session().run(policy.remove_noise_op)
clean_action_distribution = policy.get_session().run(
policy.action_probs, feed_dict={policy.cur_observations: states})
distance_in_action_space = np.mean(
entropy(clean_action_distribution.T, noisy_action_distribution.T))
policy.pi_distance = distance_in_action_space
if (distance_in_action_space <
-np.log(1 - policy.cur_epsilon_value +
policy.cur_epsilon_value / policy.num_actions)):
policy.parameter_noise_sigma_val *= 1.01
else:
policy.parameter_noise_sigma_val /= 1.01
policy.parameter_noise_sigma.load(
policy.parameter_noise_sigma_val, session=policy.get_session())
return postprocess_nstep_and_prio(policy, sample_batch)
def build_q_model(policy, obs_space, action_space, config):
if not isinstance(action_space, Discrete):
@@ -180,7 +151,11 @@ def build_q_model(policy, obs_space, action_space, config):
v_min=config["v_min"],
v_max=config["v_max"],
sigma0=config["sigma0"],
parameter_noise=config["parameter_noise"])
# TODO(sven): Move option to add LayerNorm after each Dense
# generically into ModelCatalog.
add_layer_norm=isinstance(
getattr(policy, "exploration", None), ParameterNoise)
or config["exploration_config"]["type"] == "ParameterNoise")
policy.target_q_model = ModelCatalog.get_model_v2(
obs_space,
@@ -197,7 +172,11 @@ def build_q_model(policy, obs_space, action_space, config):
v_min=config["v_min"],
v_max=config["v_max"],
sigma0=config["sigma0"],
parameter_noise=config["parameter_noise"])
# TODO(sven): Move option to add LayerNorm after each Dense
# generically into ModelCatalog.
add_layer_norm=isinstance(
getattr(policy, "exploration", None), ParameterNoise)
or config["exploration_config"]["type"] == "ParameterNoise")
return policy.q_model
@@ -298,7 +277,6 @@ def build_q_stats(policy, batch):
def setup_early_mixins(policy, obs_space, action_space, config):
LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"])
ParameterNoiseMixin.__init__(policy, obs_space, action_space, config)
def setup_mid_mixins(policy, obs_space, action_space, config):
@@ -417,7 +395,6 @@ DQNTFPolicy = build_tf_policy(
after_init=setup_late_mixins,
obs_include_prev_action_reward=False,
mixins=[
ParameterNoiseMixin,
TargetNetworkMixin,
ComputeTDErrorMixin,
LearningRateSchedule,
rllib/agents/dqn/simple_q_policy.py
+1 -15
View File
@@ -22,15 +22,6 @@ Q_SCOPE = "q_func"
Q_TARGET_SCOPE = "target_q_func"
class ParameterNoiseMixin:
def __init__(self, obs_space, action_space, config):
pass
def add_parameter_noise(self):
if self.config["parameter_noise"]:
self.sess.run(self.add_noise_op)
class TargetNetworkMixin:
def __init__(self, obs_space, action_space, config):
@make_tf_callable(self.get_session())
@@ -154,10 +145,6 @@ def compute_q_values(policy, model, obs, explore):
return model.get_q_values(model_out)
def setup_early_mixins(policy, obs_space, action_space, config):
ParameterNoiseMixin.__init__(policy, obs_space, action_space, config)
def setup_late_mixins(policy, obs_space, action_space, config):
TargetNetworkMixin.__init__(policy, obs_space, action_space, config)
@@ -170,7 +157,6 @@ SimpleQPolicy = build_tf_policy(
loss_fn=build_q_losses,
extra_action_fetches_fn=lambda policy: {"q_values": policy.q_values},
extra_learn_fetches_fn=lambda policy: {"td_error": policy.td_error},
before_init=setup_early_mixins,
after_init=setup_late_mixins,
obs_include_prev_action_reward=False,
mixins=[ParameterNoiseMixin, TargetNetworkMixin])
mixins=[TargetNetworkMixin])
rllib/agents/dqn/tests/test_dqn.py
+181
View File
@@ -127,6 +127,187 @@ class TestDQN(unittest.TestCase):
actions.append(trainer.compute_action(obs))
check(np.std(actions), 0.0, false=True)
if eager_mode_ctx:
eager_mode_ctx.__exit__(None, None, None)
def test_dqn_parameter_noise_exploration(self):
"""Tests, whether a DQN Agent works with ParameterNoise."""
obs = np.array(0)
for fw in ["eager", "tf", "torch"]:
if fw == "torch":
continue
print("framework={}".format(fw))
core_config = dqn.DEFAULT_CONFIG.copy()
core_config["num_workers"] = 0 # Run locally.
core_config["env_config"] = {
"is_slippery": False,
"map_name": "4x4"
}
core_config["eager"] = fw == "eager"
core_config["use_pytorch"] = fw == "torch"
config = core_config.copy()
eager_mode_ctx = None
if fw == "tf":
assert not tf.executing_eagerly()
elif fw == "eager":
eager_mode_ctx = eager_mode()
eager_mode_ctx.__enter__()
assert tf.executing_eagerly()
# DQN with ParameterNoise exploration (config["explore"]=True).
# ----
config["exploration_config"] = {"type": "ParameterNoise"}
config["explore"] = True
trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0")
policy = trainer.get_policy()
self.assertFalse(policy.exploration.weights_are_currently_noisy)
noise_before = self._get_current_noise(policy, fw)
check(noise_before, 0.0)
initial_weights = self._get_current_weight(policy, fw)
# Pseudo-start an episode and compare the weights before and after.
policy.exploration.on_episode_start(policy, tf_sess=policy._sess)
self.assertFalse(policy.exploration.weights_are_currently_noisy)
noise_after_ep_start = self._get_current_noise(policy, fw)
weights_after_ep_start = self._get_current_weight(policy, fw)
# Should be the same, as we don't do anything at the beginning of
# the episode, only one step later.
check(noise_after_ep_start, noise_before)
check(initial_weights, weights_after_ep_start)
# Setting explore=False should always return the same action.
a_ = trainer.compute_action(obs, explore=False)
self.assertFalse(policy.exploration.weights_are_currently_noisy)
noise = self._get_current_noise(policy, fw)
# We sampled the first noise (not zero anymore).
check(noise, 0.0, false=True)
# But still not applied b/c explore=False.
check(self._get_current_weight(policy, fw), initial_weights)
for _ in range(10):
a = trainer.compute_action(obs, explore=False)
check(a, a_)
# Noise never gets applied.
check(self._get_current_weight(policy, fw), initial_weights)
self.assertFalse(
policy.exploration.weights_are_currently_noisy)
# Explore=None (default: True) should return different actions.
# However, this is only due to the underlying epsilon-greedy
# exploration.
actions = []
current_weight = None
for _ in range(10):
actions.append(trainer.compute_action(obs))
self.assertTrue(policy.exploration.weights_are_currently_noisy)
# Now, noise actually got applied (explore=True).
current_weight = self._get_current_weight(policy, fw)
check(current_weight, initial_weights, false=True)
check(current_weight, initial_weights + noise)
check(np.std(actions), 0.0, false=True)
# Pseudo-end the episode and compare weights again.
# Make sure they are the original ones.
policy.exploration.on_episode_end(policy, tf_sess=policy._sess)
weights_after_ep_end = self._get_current_weight(policy, fw)
check(current_weight - noise, weights_after_ep_end, decimals=5)
# DQN with ParameterNoise exploration (config["explore"]=False).
# ----
config = core_config.copy()
config["exploration_config"] = {"type": "ParameterNoise"}
config["explore"] = False
trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0")
policy = trainer.get_policy()
self.assertFalse(policy.exploration.weights_are_currently_noisy)
initial_weights = self._get_current_weight(policy, fw)
# Noise before anything (should be 0.0, no episode started yet).
noise = self._get_current_noise(policy, fw)
check(noise, 0.0)
# Pseudo-start an episode and compare the weights before and after
# (they should be the same).
policy.exploration.on_episode_start(policy, tf_sess=policy._sess)
self.assertFalse(policy.exploration.weights_are_currently_noisy)
# Should be the same, as we don't do anything at the beginning of
# the episode, only one step later.
noise = self._get_current_noise(policy, fw)
check(noise, 0.0)
noisy_weights = self._get_current_weight(policy, fw)
check(initial_weights, noisy_weights)
# Setting explore=False or None should always return the same
# action.
a_ = trainer.compute_action(obs, explore=False)
# Now we have re-sampled.
noise = self._get_current_noise(policy, fw)
check(noise, 0.0, false=True)
for _ in range(5):
a = trainer.compute_action(obs, explore=None)
check(a, a_)
a = trainer.compute_action(obs, explore=False)
check(a, a_)
# Pseudo-end the episode and compare weights again.
# Make sure they are the original ones (no noise permanently
# applied throughout the episode).
policy.exploration.on_episode_end(policy, tf_sess=policy._sess)
weights_after_episode_end = self._get_current_weight(policy, fw)
check(initial_weights, weights_after_episode_end)
# Noise should still be the same (re-sampling only happens at
# beginning of episode).
noise_after = self._get_current_noise(policy, fw)
check(noise, noise_after)
# Switch off EpsilonGreedy underlying exploration.
# ----
config = core_config.copy()
config["exploration_config"] = {
"type": "ParameterNoise",
"sub_exploration": {
"type": "EpsilonGreedy",
"action_space": trainer.get_policy().action_space,
"initial_epsilon": 0.0, # <- no randomness whatsoever
}
}
config["explore"] = True
trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0")
# Now, when we act - even with explore=True - we would expect
# the same action for the same input (parameter noise is
# deterministic).
policy = trainer.get_policy()
policy.exploration.on_episode_start(policy, tf_sess=policy._sess)
a_ = trainer.compute_action(obs)
for _ in range(10):
a = trainer.compute_action(obs, explore=True)
check(a, a_)
if eager_mode_ctx:
eager_mode_ctx.__exit__(None, None, None)
def _get_current_noise(self, policy, fw):
# If noise not even created yet, return 0.0.
if policy.exploration.noise is None:
return 0.0
noise = policy.exploration.noise[0][0][0]
if fw == "tf":
noise = policy.get_session().run(noise)
else:
noise = noise.numpy()
return noise
def _get_current_weight(self, policy, fw):
weights = policy.get_weights()
key = 0 if fw == "eager" else list(weights.keys())[0]
return weights[key][0][0]
if __name__ == "__main__":
import pytest
rllib/contrib/alpha_zero/core/alpha_zero_policy.py
+1 -1
View File
@@ -89,7 +89,7 @@ class AlphaZeroPolicy(TorchPolicy):
episode.user_data["mcts_policies"].append(mcts_policy)
return np.array(actions), [], self.extra_action_out(
input_dict, state_batches, self.model)
input_dict, state_batches, self.model, None)
@override(Policy)
def postprocess_trajectory(self,
rllib/evaluation/sampler.py
+6 -2
View File
@@ -579,12 +579,16 @@ def _do_policy_eval(tf_sess, to_eval, policies, active_episodes):
state_batches = _to_column_format(rnn_in)
# TODO(ekl): how can we make info batch available to TF code?
obs_batch = [t.obs for t in eval_data]
prev_action_batch = [t.prev_action for t in eval_data]
prev_reward_batch = [t.prev_reward for t in eval_data]
pending_fetches[policy_id] = policy._build_compute_actions(
builder,
obs_batch=obs_batch,
state_batches=state_batches,
prev_action_batch=[t.prev_action for t in eval_data],
prev_reward_batch=[t.prev_reward for t in eval_data],
prev_action_batch=prev_action_batch,
prev_reward_batch=prev_reward_batch,
timestep=policy.global_timestep)
else:
# TODO(sven): Does this work for LSTM torch?
rllib/policy/eager_tf_policy.py
+22 -15
View File
@@ -110,7 +110,7 @@ def traced_eager_policy(eager_policy_cls):
@convert_eager_outputs
def compute_actions(self,
obs_batch,
state_batches,
state_batches=None,
prev_action_batch=None,
prev_reward_batch=None,
info_batch=None,
@@ -302,7 +302,7 @@ def build_eager_tf_policy(name,
@convert_eager_outputs
def compute_actions(self,
obs_batch,
state_batches,
state_batches=None,
prev_action_batch=None,
prev_reward_batch=None,
info_batch=None,
@@ -320,22 +320,26 @@ def build_eager_tf_policy(name,
self._is_training = False
self._state_in = state_batches
if tf.executing_eagerly():
n = len(obs_batch)
else:
n = obs_batch.shape[0]
seq_lens = tf.ones(n, dtype=tf.int32)
input_dict = {
SampleBatch.CUR_OBS: tf.convert_to_tensor(obs_batch),
"is_training": tf.constant(False),
}
if obs_include_prev_action_reward:
input_dict.update({
SampleBatch.PREV_ACTIONS: tf.convert_to_tensor(
prev_action_batch),
SampleBatch.PREV_REWARDS: tf.convert_to_tensor(
prev_reward_batch),
})
input_dict[SampleBatch.PREV_ACTIONS] = \
tf.convert_to_tensor(prev_action_batch)
input_dict[SampleBatch.PREV_REWARDS] = \
tf.convert_to_tensor(prev_reward_batch)
# Use Exploration object.
with tf.variable_creator_scope(_disallow_var_creation):
if action_sampler_fn:
dist_class = dist_inputs = None
dist_inputs = None
state_out = []
actions, logp = self.action_sampler_fn(
self,
@@ -357,8 +361,7 @@ def build_eager_tf_policy(name,
else:
dist_class = self.dist_class
dist_inputs, state_out = self.model(
input_dict, state_batches,
tf.convert_to_tensor([1]))
input_dict, state_batches, seq_lens)
action_dist = dist_class(dist_inputs, self.model)
@@ -424,8 +427,10 @@ def build_eager_tf_policy(name,
else:
dist_inputs, _ = self.model(input_dict, state_batches,
seq_lens)
action_dist = self.dist_class(dist_inputs, self.model)
log_likelihoods = action_dist.logp(actions)
dist_class = self.dist_class
action_dist = dist_class(dist_inputs, self.model)
log_likelihoods = action_dist.logp(actions)
return log_likelihoods
@@ -609,9 +614,11 @@ def build_eager_tf_policy(name,
# Execute a forward pass to get self.action_dist etc initialized,
# and also obtain the extra action fetches
_, _, fetches = self.compute_actions(
dummy_batch[SampleBatch.CUR_OBS], self._state_in,
dummy_batch[SampleBatch.CUR_OBS],
self._state_in,
dummy_batch.get(SampleBatch.PREV_ACTIONS),
dummy_batch.get(SampleBatch.PREV_REWARDS))
dummy_batch.get(SampleBatch.PREV_REWARDS),
explore=False)
dummy_batch.update(fetches)
postprocessed_batch = self.postprocess_trajectory(
rllib/policy/tf_policy.py
+12 -8
View File
@@ -137,7 +137,7 @@ class TFPolicy(Policy):
if self._sampled_action_logp is not None
else None)
self._action_input = action_input # For logp calculations.
self._distr_inputs = dist_inputs
self._dist_inputs = dist_inputs
self.dist_class = dist_class
self._log_likelihood = log_likelihood
self._state_inputs = state_inputs or []
@@ -172,9 +172,9 @@ class TFPolicy(Policy):
# The log-likelihood calculator op.
self._log_likelihood = None
if self._distr_inputs is not None and self.dist_class is not None:
if self._dist_inputs is not None and self.dist_class is not None:
self._log_likelihood = self.dist_class(
self._distr_inputs, self.model).logp(self._action_input)
self._dist_inputs, self.model).logp(self._action_input)
def variables(self):
"""Return the list of all savable variables for this policy."""
@@ -424,8 +424,8 @@ class TFPolicy(Policy):
extra_fetches[SampleBatch.ACTION_PROB] = self._sampled_action_prob
extra_fetches[SampleBatch.ACTION_LOGP] = self._sampled_action_logp
# Action-dist inputs.
if self._distr_inputs is not None:
extra_fetches[SampleBatch.ACTION_DIST_INPUTS] = self._distr_inputs
if self._dist_inputs is not None:
extra_fetches[SampleBatch.ACTION_DIST_INPUTS] = self._dist_inputs
return extra_fetches
@DeveloperAPI
@@ -577,9 +577,13 @@ class TFPolicy(Policy):
if timestep is not None:
builder.add_feed_dict({self._timestep: timestep})
builder.add_feed_dict(dict(zip(self._state_inputs, state_batches)))
fetches = builder.add_fetches([self._sampled_action] +
self._state_outputs +
[self.extra_compute_action_fetches()])
# Determine, what exactly to fetch from the graph.
to_fetch = [self._sampled_action] + self._state_outputs + \
[self.extra_compute_action_fetches()]
# Perform the session call.
fetches = builder.add_fetches(to_fetch)
return fetches[0], fetches[1:-1], fetches[-1]
def _build_compute_gradients(self, builder, postprocessed_batch):
rllib/policy/torch_policy.py
+5 -9
View File
@@ -106,9 +106,9 @@ class TorchPolicy(Policy):
explore = explore if explore is not None else self.config["explore"]
timestep = timestep if timestep is not None else self.global_timestep
seq_lens = torch.ones(len(obs_batch), dtype=torch.int32)
with torch.no_grad():
seq_lens = torch.ones(len(obs_batch), dtype=torch.int32)
input_dict = self._lazy_tensor_dict({
SampleBatch.CUR_OBS: obs_batch,
})
@@ -301,19 +301,15 @@ class TorchPolicy(Policy):
return processing info."""
return {}
def extra_action_out(self,
input_dict,
state_batches,
model,
action_dist=None):
def extra_action_out(self, input_dict, state_batches, model, action_dist):
"""Returns dict of extra info to include in experience batch.
Arguments:
Args:
input_dict (dict): Dict of model input tensors.
state_batches (list): List of state tensors.
model (TorchModelV2): Reference to the model.
action_dist (Distribution): Torch Distribution object to get
log-probs (e.g. for already sampled actions).
action_dist (TorchActionDistribution): Torch action dist object
to get log-probs (e.g. for already sampled actions).
"""
return {}
rllib/policy/torch_policy_template.py
+10 -10
View File
@@ -121,15 +121,18 @@ def build_torch_policy(name,
sample_batch,
other_agent_batches=None,
episode=None):
if not postprocess_fn:
return sample_batch
# Do all post-processing always with no_grad().
# Not using this here will introduce a memory leak (issue #6962).
with torch.no_grad():
return postprocess_fn(
self, convert_to_non_torch_type(sample_batch),
# Call super's postprocess_trajectory first.
sample_batch = super().postprocess_trajectory(
convert_to_non_torch_type(sample_batch),
convert_to_non_torch_type(other_agent_batches), episode)
if postprocess_fn:
return postprocess_fn(self, sample_batch,
other_agent_batches, episode)
return sample_batch
@override(TorchPolicy)
def extra_grad_process(self):
@@ -139,11 +142,8 @@ def build_torch_policy(name,
return TorchPolicy.extra_grad_process(self)
@override(TorchPolicy)
def extra_action_out(self,
input_dict,
state_batches,
model,
action_dist=None):
def extra_action_out(self, input_dict, state_batches, model,
action_dist):
with torch.no_grad():
if extra_action_out_fn:
stats_dict = extra_action_out_fn(
rllib/tests/test_rollout.py
-6
View File
@@ -49,9 +49,6 @@ def rollout_test(algo, env="CartPole-v0"):
class TestRollout(unittest.TestCase):
def test_a2c(self):
rollout_test("A2C")
def test_a3c(self):
rollout_test("A3C")
@@ -79,9 +76,6 @@ class TestRollout(unittest.TestCase):
def test_sac(self):
rollout_test("SAC", env="Pendulum-v0")
def test_td3(self):
rollout_test("TD3", env="Pendulum-v0")
if __name__ == "__main__":
import pytest
rllib/tuned_examples/regression_tests/cartpole-dqn-param-noise.yaml
+17
View File
@@ -0,0 +1,17 @@
cartpole-dqn-w-param-noise:
env: CartPole-v0
run: DQN
stop:
episode_reward_mean: 150
timesteps_total: 100000
config:
exploration_config:
type: ParameterNoise
random_timesteps: 0
initial_stddev: 1.0
batch_mode: complete_episodes
lr: 0.001
num_workers: 0
model:
fcnet_hiddens: [16]
fcnet_activation: linear
rllib/tuned_examples/regression_tests/cartpole-sac.yaml
+1
View File
@@ -3,6 +3,7 @@ cartpole-sac:
run: SAC
stop:
episode_reward_mean: 150
timesteps_total: 100000
config:
gamma: 0.95
no_done_at_end: false
rllib/utils/exploration/exploration.py
+4 -4
View File
@@ -20,16 +20,16 @@ class Exploration:
"""
def __init__(self, action_space: Space, *, framework: str,
num_workers: int, worker_index: int, policy_config: dict,
model: ModelV2):
policy_config: dict, model: ModelV2, num_workers: int,
worker_index: int):
"""
Args:
action_space (Space): The action space in which to explore.
framework (str): One of "tf" or "torch".
num_workers (int): The overall number of workers used.
worker_index (int): The index of the worker using this class.
policy_config (dict): The Policy's config dict.
model (ModelV2): The Policy's model.
num_workers (int): The overall number of workers used.
worker_index (int): The index of the worker using this class.
"""
self.action_space = action_space
self.policy_config = policy_config
rllib/utils/exploration/parameter_noise.py
+11 -12
View File
@@ -117,10 +117,10 @@ class ParameterNoise(Exploration):
sub_exploration,
framework=self.framework,
action_space=self.action_space,
policy_config=self.policy_config,
model=self.model,
**kwargs)
# Store the default setting for `explore`.
self.default_explore = policy_config["explore"]
# Whether we need to call `self._delayed_on_episode_start` before
# the forward pass.
self.episode_started = False
@@ -131,13 +131,14 @@ class ParameterNoise(Exploration):
timestep=None,
explore=None,
tf_sess=None):
explore = explore if explore is not None else \
self.policy_config["explore"]
# Is this the first forward pass in the new episode? If yes, do the
# noise re-sampling and add to weights.
if self.episode_started:
self._delayed_on_episode_start(tf_sess)
self._delayed_on_episode_start(explore, tf_sess)
explore = explore if explore is not None else \
self.policy_config["explore"]
# Add noise if necessary.
if explore and not self.weights_are_currently_noisy:
self._add_stored_noise(tf_sess=tf_sess)
@@ -148,15 +149,13 @@ class ParameterNoise(Exploration):
@override(Exploration)
def get_exploration_action(self,
*,
distribution_inputs,
action_dist_class,
action_distribution,
timestep,
explore=True):
# Use our sub-exploration object to handle the final exploration
# action (depends on the algo-type/action-space/etc..).
return self.sub_exploration.get_exploration_action(
distribution_inputs=distribution_inputs,
action_dist_class=action_dist_class,
action_distribution=action_distribution,
timestep=timestep,
explore=explore)
@@ -173,9 +172,9 @@ class ParameterNoise(Exploration):
# We don't want to update into a noisy net.
self.episode_started = True
def _delayed_on_episode_start(self, tf_sess):
def _delayed_on_episode_start(self, explore, tf_sess):
# Sample fresh noise and add to weights.
if self.default_explore:
if explore:
self._sample_new_noise_and_add(tf_sess=tf_sess, override=True)
# Only sample, don't apply anything to the weights.
else:
@@ -227,7 +226,7 @@ class ParameterNoise(Exploration):
if policy.dist_class is Categorical:
action_dist = softmax(fetches[SampleBatch.ACTION_DIST_INPUTS])
if not self.weights_are_currently_noisy:
if noisy_action_dist is None:
noisy_action_dist = action_dist
else:
noise_free_action_dist = action_dist
rllib/utils/exploration/random.py
+3 -2
View File
@@ -29,8 +29,8 @@ class Random(Exploration):
"""
super().__init__(
action_space=action_space,
framework=framework,
model=model,
framework=framework,
**kwargs)
# Determine py_func types, depending on our action-space.
@@ -87,7 +87,8 @@ class Random(Exploration):
if explore:
# Unsqueeze will be unnecessary, once we support batch/time-aware
# Spaces.
action = tensor_fn(self.action_space.sample()).unsqueeze(0)
a = self.action_space.sample()
action = tensor_fn([a] if isinstance(a, int) else a)
else:
action = tensor_fn(action_dist.deterministic_sample())
logp = torch.zeros((action.size()[0], ), dtype=torch.float32)
rllib/utils/exploration/tests/test_explorations.py
+2 -2
View File
@@ -67,7 +67,7 @@ def do_test_explorations(run,
# Make sure all actions drawn are the same, given same
# observations.
actions = []
for _ in range(50):
for _ in range(25):
actions.append(
trainer.compute_action(
observation=dummy_obs,
@@ -79,7 +79,7 @@ def do_test_explorations(run,
# Make sure actions drawn are different
# (around some mean value), given constant observations.
actions = []
for _ in range(100):
for _ in range(50):
actions.append(
trainer.compute_action(
observation=dummy_obs,
rllib/utils/numpy.py
+3
View File
@@ -46,7 +46,10 @@ def softmax(x, axis=-1):
Returns:
np.ndarray: The softmax over x.
"""
# x_exp = np.maximum(np.exp(x), SMALL_NUMBER)
x_exp = np.exp(x)
# return x_exp /
# np.maximum(np.sum(x_exp, axis, keepdims=True), SMALL_NUMBER)
return np.maximum(x_exp / np.sum(x_exp, axis, keepdims=True), SMALL_NUMBER)
rllib/utils/test_utils.py
+13 -6
View File
@@ -120,14 +120,20 @@ def check(x, y, decimals=5, atol=None, rtol=None, false=False):
# Using decimals.
if atol is None and rtol is None:
# Assert equality of both values.
try:
np.testing.assert_almost_equal(x, y, decimal=decimals)
# Both values are not equal.
except AssertionError as e:
# Raise error in normal case.
if false is False:
raise e
# Both values are equal.
else:
# If false is set -> raise error (not expected to be equal).
if false is True:
assert False, \
"ERROR: x ({}) is the same as y ({})!".format(x, y)
except AssertionError as e:
if false is False:
raise e
# Using atol/rtol.
else:
@@ -138,9 +144,10 @@ def check(x, y, decimals=5, atol=None, rtol=None, false=False):
rtol = 1e-7
try:
np.testing.assert_allclose(x, y, atol=atol, rtol=rtol)
if false is True:
assert False, \
"ERROR: x ({}) is the same as y ({})!".format(x, y)
except AssertionError as e:
if false is False:
raise e
else:
if false is True:
assert False, \
"ERROR: x ({}) is the same as y ({})!".format(x, y)