diff --git a/rllib/agents/a3c/a3c_torch_policy.py b/rllib/agents/a3c/a3c_torch_policy.py index 0e4ede629..2168e7d11 100644 --- a/rllib/agents/a3c/a3c_torch_policy.py +++ b/rllib/agents/a3c/a3c_torch_policy.py @@ -45,10 +45,9 @@ def add_advantages(policy, else: last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1]) - return compute_advantages(sample_batch, last_r, policy.config["gamma"], - policy.config["lambda"], - policy.config["use_gae"], - policy.config["use_critic"]) + return compute_advantages( + sample_batch, last_r, policy.config["gamma"], policy.config["lambda"], + policy.config["use_gae"], policy.config["use_critic"]) def model_value_predictions(policy, input_dict, state_batches, model, diff --git a/rllib/agents/ddpg/ddpg.py b/rllib/agents/ddpg/ddpg.py index e0949505e..d22d5670c 100644 --- a/rllib/agents/ddpg/ddpg.py +++ b/rllib/agents/ddpg/ddpg.py @@ -80,9 +80,12 @@ DEFAULT_CONFIG = with_common_config({ }, # Number of env steps to optimize for before returning "timesteps_per_iteration": 1000, + + # TODO(sven): Move to Exploration API's (ParameterNoise class). # If True parameter space noise will be used for exploration # See https://blog.openai.com/better-exploration-with-parameter-noise/ "parameter_noise": False, + # Extra configuration that disables exploration. "evaluation_config": { "explore": False diff --git a/rllib/agents/dqn/distributional_q_model.py b/rllib/agents/dqn/distributional_q_model.py index dd981412b..5b4c329d3 100644 --- a/rllib/agents/dqn/distributional_q_model.py +++ b/rllib/agents/dqn/distributional_q_model.py @@ -19,20 +19,25 @@ class DistributionalQModel(TFModelV2): Note that this class by itself is not a valid model unless you implement forward() in a subclass.""" - def __init__(self, - obs_space, - action_space, - num_outputs, - model_config, - name, - q_hiddens=(256, ), - dueling=False, - num_atoms=1, - use_noisy=False, - v_min=-10.0, - v_max=10.0, - sigma0=0.5, - parameter_noise=False): + def __init__( + self, + obs_space, + action_space, + num_outputs, + model_config, + name, + q_hiddens=(256, ), + dueling=False, + num_atoms=1, + use_noisy=False, + v_min=-10.0, + v_max=10.0, + sigma0=0.5, + # TODO(sven): Move `add_layer_norm` into ModelCatalog as + # generic option, then error if we use ParameterNoise as + # Exploration type and do not have any LayerNorm layers in + # the net. + add_layer_norm=False): """Initialize variables of this model. Extra model kwargs: @@ -45,7 +50,7 @@ class DistributionalQModel(TFModelV2): v_min (float): min value support for distributional DQN v_max (float): max value support for distributional DQN sigma0 (float): initial value of noisy nets - parameter_noise (bool): enable layer norm for param noise + add_layer_norm (bool): Add a LayerNorm after each layer.. Note that the core layers for forward() are not defined here, this only defines the layers for the Q head. Those layers for forward() @@ -66,11 +71,12 @@ class DistributionalQModel(TFModelV2): if use_noisy: action_out = self._noisy_layer( "hidden_%d" % i, action_out, q_hiddens[i], sigma0) - elif parameter_noise: + elif add_layer_norm: action_out = tf.keras.layers.Dense( units=q_hiddens[i], - activation_fn=tf.nn.relu, - normalizer_fn=tf.keras.layers.LayerNormalization)( + activation=tf.nn.relu)(action_out) + action_out = \ + tf.keras.layers.LayerNormalization()( action_out) else: action_out = tf.keras.layers.Dense( @@ -125,12 +131,14 @@ class DistributionalQModel(TFModelV2): state_out = self._noisy_layer("dueling_hidden_%d" % i, state_out, q_hiddens[i], sigma0) + elif add_layer_norm: + state_out = tf.keras.layers.Dense( + units=q_hiddens[i], activation=tf.nn.relu)(state_out) + state_out = \ + tf.keras.layers.LayerNormalization()(state_out) else: state_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu)(state_out) - if parameter_noise: - state_out = tf.keras.layers.LayerNormalization()( - state_out) if use_noisy: state_score = self._noisy_layer( "dueling_output", diff --git a/rllib/agents/dqn/dqn.py b/rllib/agents/dqn/dqn.py index 4774330cc..aa9dec8dc 100644 --- a/rllib/agents/dqn/dqn.py +++ b/rllib/agents/dqn/dqn.py @@ -6,7 +6,6 @@ from ray.rllib.agents.dqn.dqn_policy import DQNTFPolicy from ray.rllib.agents.dqn.simple_q_policy import SimpleQPolicy from ray.rllib.optimizers import SyncReplayOptimizer from ray.rllib.optimizers.replay_buffer import ReplayBuffer -from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE from ray.rllib.utils.exploration import PerWorkerEpsilonGreedy from ray.rllib.utils.experimental_dsl import ( @@ -59,11 +58,6 @@ DEFAULT_CONFIG = with_common_config({ "explore": False, }, - # TODO(sven): Make Exploration class for parameter noise. - # If True parameter space noise will be used for exploration - # See https://blog.openai.com/better-exploration-with-parameter-noise/ - "parameter_noise": False, - # Minimum env steps to optimize for per train call. This value does # not affect learning, only the length of iterations. "timesteps_per_iteration": 1000, @@ -127,6 +121,7 @@ DEFAULT_CONFIG = with_common_config({ "per_worker_exploration": DEPRECATED_VALUE, "softmax_temp": DEPRECATED_VALUE, "soft_q": DEPRECATED_VALUE, + "parameter_noise": DEPRECATED_VALUE, }) # __sphinx_doc_end__ # yapf: enable @@ -221,49 +216,27 @@ def validate_config_and_setup_param_noise(config): "type": "SoftQ", "temperature": config.get("softmax_temp", 1.0) } + if config.get("parameter_noise", DEPRECATED_VALUE) != DEPRECATED_VALUE: + deprecation_warning("parameter_noise", "exploration_config={" + "type=ParameterNoise" + "}") + + if config["exploration_config"]["type"] == "ParameterNoise": + if config["batch_mode"] != "complete_episodes": + logger.warning( + "ParameterNoise Exploration requires `batch_mode` to be " + "'complete_episodes'. Setting batch_mode=complete_episodes.") + config["batch_mode"] = "complete_episodes" + if config.get("noisy", False): + raise ValueError( + "ParameterNoise Exploration and `noisy` network cannot be " + "used at the same time!") # Update effective batch size to include n-step adjusted_batch_size = max(config["rollout_fragment_length"], config.get("n_step", 1)) config["rollout_fragment_length"] = adjusted_batch_size - # Setup parameter noise. - if config.get("parameter_noise", False): - if config["batch_mode"] != "complete_episodes": - raise ValueError("Exploration with parameter space noise requires " - "batch_mode to be complete_episodes.") - if config.get("noisy", False): - raise ValueError("Exploration with parameter space noise and " - "noisy network cannot be used at the same time.") - - start_callback = config["callbacks"].get("on_episode_start") - - def on_episode_start(info): - # as a callback function to sample and pose parameter space - # noise on the parameters of network - policies = info["policy"] - for pi in policies.values(): - pi.add_parameter_noise() - if start_callback is not None: - start_callback(info) - - config["callbacks"]["on_episode_start"] = on_episode_start - - end_callback = config["callbacks"].get("on_episode_end") - - def on_episode_end(info): - # as a callback function to monitor the distance - # between noisy policy and original policy - policies = info["policy"] - episode = info["episode"] - model = policies[DEFAULT_POLICY_ID].model - if hasattr(model, "pi_distance"): - episode.custom_metrics["policy_distance"] = model.pi_distance - if end_callback is not None: - end_callback(info) - - config["callbacks"]["on_episode_end"] = on_episode_end - def get_initial_state(config): return { diff --git a/rllib/agents/dqn/dqn_policy.py b/rllib/agents/dqn/dqn_policy.py index a7c1dc46a..7f13639cd 100644 --- a/rllib/agents/dqn/dqn_policy.py +++ b/rllib/agents/dqn/dqn_policy.py @@ -1,17 +1,16 @@ from gym.spaces import Discrete import numpy as np -from scipy.stats import entropy import ray from ray.rllib.agents.dqn.distributional_q_model import DistributionalQModel -from ray.rllib.agents.dqn.simple_q_policy import TargetNetworkMixin, \ - ParameterNoiseMixin +from ray.rllib.agents.dqn.simple_q_policy import TargetNetworkMixin from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.policy.tf_policy import LearningRateSchedule from ray.rllib.policy.tf_policy_template import build_tf_policy from ray.rllib.models import ModelCatalog from ray.rllib.models.tf.tf_action_dist import Categorical from ray.rllib.utils.error import UnsupportedSpaceException +from ray.rllib.utils.exploration import ParameterNoise from ray.rllib.utils.tf_ops import huber_loss, reduce_mean_ignore_inf, \ minimize_and_clip from ray.rllib.utils import try_import_tf @@ -124,34 +123,6 @@ class ComputeTDErrorMixin: self.compute_td_error = compute_td_error -def postprocess_trajectory(policy, - sample_batch, - other_agent_batches=None, - episode=None): - if policy.config["parameter_noise"]: - # adjust the sigma of parameter space noise - states = [list(x) for x in sample_batch.columns(["obs"])][0] - - noisy_action_distribution = policy.get_session().run( - policy.action_probs, feed_dict={policy.cur_observations: states}) - policy.get_session().run(policy.remove_noise_op) - clean_action_distribution = policy.get_session().run( - policy.action_probs, feed_dict={policy.cur_observations: states}) - distance_in_action_space = np.mean( - entropy(clean_action_distribution.T, noisy_action_distribution.T)) - policy.pi_distance = distance_in_action_space - if (distance_in_action_space < - -np.log(1 - policy.cur_epsilon_value + - policy.cur_epsilon_value / policy.num_actions)): - policy.parameter_noise_sigma_val *= 1.01 - else: - policy.parameter_noise_sigma_val /= 1.01 - policy.parameter_noise_sigma.load( - policy.parameter_noise_sigma_val, session=policy.get_session()) - - return postprocess_nstep_and_prio(policy, sample_batch) - - def build_q_model(policy, obs_space, action_space, config): if not isinstance(action_space, Discrete): @@ -180,7 +151,11 @@ def build_q_model(policy, obs_space, action_space, config): v_min=config["v_min"], v_max=config["v_max"], sigma0=config["sigma0"], - parameter_noise=config["parameter_noise"]) + # TODO(sven): Move option to add LayerNorm after each Dense + # generically into ModelCatalog. + add_layer_norm=isinstance( + getattr(policy, "exploration", None), ParameterNoise) + or config["exploration_config"]["type"] == "ParameterNoise") policy.target_q_model = ModelCatalog.get_model_v2( obs_space, @@ -197,7 +172,11 @@ def build_q_model(policy, obs_space, action_space, config): v_min=config["v_min"], v_max=config["v_max"], sigma0=config["sigma0"], - parameter_noise=config["parameter_noise"]) + # TODO(sven): Move option to add LayerNorm after each Dense + # generically into ModelCatalog. + add_layer_norm=isinstance( + getattr(policy, "exploration", None), ParameterNoise) + or config["exploration_config"]["type"] == "ParameterNoise") return policy.q_model @@ -298,7 +277,6 @@ def build_q_stats(policy, batch): def setup_early_mixins(policy, obs_space, action_space, config): LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"]) - ParameterNoiseMixin.__init__(policy, obs_space, action_space, config) def setup_mid_mixins(policy, obs_space, action_space, config): @@ -417,7 +395,6 @@ DQNTFPolicy = build_tf_policy( after_init=setup_late_mixins, obs_include_prev_action_reward=False, mixins=[ - ParameterNoiseMixin, TargetNetworkMixin, ComputeTDErrorMixin, LearningRateSchedule, diff --git a/rllib/agents/dqn/simple_q_policy.py b/rllib/agents/dqn/simple_q_policy.py index 31c623026..d0473ceb6 100644 --- a/rllib/agents/dqn/simple_q_policy.py +++ b/rllib/agents/dqn/simple_q_policy.py @@ -22,15 +22,6 @@ Q_SCOPE = "q_func" Q_TARGET_SCOPE = "target_q_func" -class ParameterNoiseMixin: - def __init__(self, obs_space, action_space, config): - pass - - def add_parameter_noise(self): - if self.config["parameter_noise"]: - self.sess.run(self.add_noise_op) - - class TargetNetworkMixin: def __init__(self, obs_space, action_space, config): @make_tf_callable(self.get_session()) @@ -154,10 +145,6 @@ def compute_q_values(policy, model, obs, explore): return model.get_q_values(model_out) -def setup_early_mixins(policy, obs_space, action_space, config): - ParameterNoiseMixin.__init__(policy, obs_space, action_space, config) - - def setup_late_mixins(policy, obs_space, action_space, config): TargetNetworkMixin.__init__(policy, obs_space, action_space, config) @@ -170,7 +157,6 @@ SimpleQPolicy = build_tf_policy( loss_fn=build_q_losses, extra_action_fetches_fn=lambda policy: {"q_values": policy.q_values}, extra_learn_fetches_fn=lambda policy: {"td_error": policy.td_error}, - before_init=setup_early_mixins, after_init=setup_late_mixins, obs_include_prev_action_reward=False, - mixins=[ParameterNoiseMixin, TargetNetworkMixin]) + mixins=[TargetNetworkMixin]) diff --git a/rllib/agents/dqn/tests/test_dqn.py b/rllib/agents/dqn/tests/test_dqn.py index aab555c0a..32b3dc5a9 100644 --- a/rllib/agents/dqn/tests/test_dqn.py +++ b/rllib/agents/dqn/tests/test_dqn.py @@ -127,6 +127,187 @@ class TestDQN(unittest.TestCase): actions.append(trainer.compute_action(obs)) check(np.std(actions), 0.0, false=True) + if eager_mode_ctx: + eager_mode_ctx.__exit__(None, None, None) + + def test_dqn_parameter_noise_exploration(self): + """Tests, whether a DQN Agent works with ParameterNoise.""" + obs = np.array(0) + + for fw in ["eager", "tf", "torch"]: + if fw == "torch": + continue + print("framework={}".format(fw)) + + core_config = dqn.DEFAULT_CONFIG.copy() + core_config["num_workers"] = 0 # Run locally. + core_config["env_config"] = { + "is_slippery": False, + "map_name": "4x4" + } + core_config["eager"] = fw == "eager" + core_config["use_pytorch"] = fw == "torch" + + config = core_config.copy() + + eager_mode_ctx = None + if fw == "tf": + assert not tf.executing_eagerly() + elif fw == "eager": + eager_mode_ctx = eager_mode() + eager_mode_ctx.__enter__() + assert tf.executing_eagerly() + + # DQN with ParameterNoise exploration (config["explore"]=True). + # ---- + config["exploration_config"] = {"type": "ParameterNoise"} + config["explore"] = True + + trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0") + policy = trainer.get_policy() + self.assertFalse(policy.exploration.weights_are_currently_noisy) + noise_before = self._get_current_noise(policy, fw) + check(noise_before, 0.0) + initial_weights = self._get_current_weight(policy, fw) + + # Pseudo-start an episode and compare the weights before and after. + policy.exploration.on_episode_start(policy, tf_sess=policy._sess) + self.assertFalse(policy.exploration.weights_are_currently_noisy) + noise_after_ep_start = self._get_current_noise(policy, fw) + weights_after_ep_start = self._get_current_weight(policy, fw) + # Should be the same, as we don't do anything at the beginning of + # the episode, only one step later. + check(noise_after_ep_start, noise_before) + check(initial_weights, weights_after_ep_start) + + # Setting explore=False should always return the same action. + a_ = trainer.compute_action(obs, explore=False) + self.assertFalse(policy.exploration.weights_are_currently_noisy) + noise = self._get_current_noise(policy, fw) + # We sampled the first noise (not zero anymore). + check(noise, 0.0, false=True) + # But still not applied b/c explore=False. + check(self._get_current_weight(policy, fw), initial_weights) + for _ in range(10): + a = trainer.compute_action(obs, explore=False) + check(a, a_) + # Noise never gets applied. + check(self._get_current_weight(policy, fw), initial_weights) + self.assertFalse( + policy.exploration.weights_are_currently_noisy) + + # Explore=None (default: True) should return different actions. + # However, this is only due to the underlying epsilon-greedy + # exploration. + actions = [] + current_weight = None + for _ in range(10): + actions.append(trainer.compute_action(obs)) + self.assertTrue(policy.exploration.weights_are_currently_noisy) + # Now, noise actually got applied (explore=True). + current_weight = self._get_current_weight(policy, fw) + check(current_weight, initial_weights, false=True) + check(current_weight, initial_weights + noise) + check(np.std(actions), 0.0, false=True) + + # Pseudo-end the episode and compare weights again. + # Make sure they are the original ones. + policy.exploration.on_episode_end(policy, tf_sess=policy._sess) + weights_after_ep_end = self._get_current_weight(policy, fw) + check(current_weight - noise, weights_after_ep_end, decimals=5) + + # DQN with ParameterNoise exploration (config["explore"]=False). + # ---- + config = core_config.copy() + config["exploration_config"] = {"type": "ParameterNoise"} + config["explore"] = False + trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0") + policy = trainer.get_policy() + self.assertFalse(policy.exploration.weights_are_currently_noisy) + initial_weights = self._get_current_weight(policy, fw) + + # Noise before anything (should be 0.0, no episode started yet). + noise = self._get_current_noise(policy, fw) + check(noise, 0.0) + + # Pseudo-start an episode and compare the weights before and after + # (they should be the same). + policy.exploration.on_episode_start(policy, tf_sess=policy._sess) + self.assertFalse(policy.exploration.weights_are_currently_noisy) + + # Should be the same, as we don't do anything at the beginning of + # the episode, only one step later. + noise = self._get_current_noise(policy, fw) + check(noise, 0.0) + noisy_weights = self._get_current_weight(policy, fw) + check(initial_weights, noisy_weights) + + # Setting explore=False or None should always return the same + # action. + a_ = trainer.compute_action(obs, explore=False) + # Now we have re-sampled. + noise = self._get_current_noise(policy, fw) + check(noise, 0.0, false=True) + for _ in range(5): + a = trainer.compute_action(obs, explore=None) + check(a, a_) + a = trainer.compute_action(obs, explore=False) + check(a, a_) + + # Pseudo-end the episode and compare weights again. + # Make sure they are the original ones (no noise permanently + # applied throughout the episode). + policy.exploration.on_episode_end(policy, tf_sess=policy._sess) + weights_after_episode_end = self._get_current_weight(policy, fw) + check(initial_weights, weights_after_episode_end) + # Noise should still be the same (re-sampling only happens at + # beginning of episode). + noise_after = self._get_current_noise(policy, fw) + check(noise, noise_after) + + # Switch off EpsilonGreedy underlying exploration. + # ---- + config = core_config.copy() + config["exploration_config"] = { + "type": "ParameterNoise", + "sub_exploration": { + "type": "EpsilonGreedy", + "action_space": trainer.get_policy().action_space, + "initial_epsilon": 0.0, # <- no randomness whatsoever + } + } + config["explore"] = True + trainer = dqn.DQNTrainer(config=config, env="FrozenLake-v0") + # Now, when we act - even with explore=True - we would expect + # the same action for the same input (parameter noise is + # deterministic). + policy = trainer.get_policy() + policy.exploration.on_episode_start(policy, tf_sess=policy._sess) + a_ = trainer.compute_action(obs) + for _ in range(10): + a = trainer.compute_action(obs, explore=True) + check(a, a_) + + if eager_mode_ctx: + eager_mode_ctx.__exit__(None, None, None) + + def _get_current_noise(self, policy, fw): + # If noise not even created yet, return 0.0. + if policy.exploration.noise is None: + return 0.0 + + noise = policy.exploration.noise[0][0][0] + if fw == "tf": + noise = policy.get_session().run(noise) + else: + noise = noise.numpy() + return noise + + def _get_current_weight(self, policy, fw): + weights = policy.get_weights() + key = 0 if fw == "eager" else list(weights.keys())[0] + return weights[key][0][0] + if __name__ == "__main__": import pytest diff --git a/rllib/contrib/alpha_zero/core/alpha_zero_policy.py b/rllib/contrib/alpha_zero/core/alpha_zero_policy.py index 29916fb20..46f24c92e 100644 --- a/rllib/contrib/alpha_zero/core/alpha_zero_policy.py +++ b/rllib/contrib/alpha_zero/core/alpha_zero_policy.py @@ -89,7 +89,7 @@ class AlphaZeroPolicy(TorchPolicy): episode.user_data["mcts_policies"].append(mcts_policy) return np.array(actions), [], self.extra_action_out( - input_dict, state_batches, self.model) + input_dict, state_batches, self.model, None) @override(Policy) def postprocess_trajectory(self, diff --git a/rllib/evaluation/sampler.py b/rllib/evaluation/sampler.py index f77461592..3e01e8b71 100644 --- a/rllib/evaluation/sampler.py +++ b/rllib/evaluation/sampler.py @@ -579,12 +579,16 @@ def _do_policy_eval(tf_sess, to_eval, policies, active_episodes): state_batches = _to_column_format(rnn_in) # TODO(ekl): how can we make info batch available to TF code? + obs_batch = [t.obs for t in eval_data] + prev_action_batch = [t.prev_action for t in eval_data] + prev_reward_batch = [t.prev_reward for t in eval_data] + pending_fetches[policy_id] = policy._build_compute_actions( builder, obs_batch=obs_batch, state_batches=state_batches, - prev_action_batch=[t.prev_action for t in eval_data], - prev_reward_batch=[t.prev_reward for t in eval_data], + prev_action_batch=prev_action_batch, + prev_reward_batch=prev_reward_batch, timestep=policy.global_timestep) else: # TODO(sven): Does this work for LSTM torch? diff --git a/rllib/policy/eager_tf_policy.py b/rllib/policy/eager_tf_policy.py index 2aa034746..d4c557c4f 100644 --- a/rllib/policy/eager_tf_policy.py +++ b/rllib/policy/eager_tf_policy.py @@ -110,7 +110,7 @@ def traced_eager_policy(eager_policy_cls): @convert_eager_outputs def compute_actions(self, obs_batch, - state_batches, + state_batches=None, prev_action_batch=None, prev_reward_batch=None, info_batch=None, @@ -302,7 +302,7 @@ def build_eager_tf_policy(name, @convert_eager_outputs def compute_actions(self, obs_batch, - state_batches, + state_batches=None, prev_action_batch=None, prev_reward_batch=None, info_batch=None, @@ -320,22 +320,26 @@ def build_eager_tf_policy(name, self._is_training = False self._state_in = state_batches + if tf.executing_eagerly(): + n = len(obs_batch) + else: + n = obs_batch.shape[0] + seq_lens = tf.ones(n, dtype=tf.int32) + input_dict = { SampleBatch.CUR_OBS: tf.convert_to_tensor(obs_batch), "is_training": tf.constant(False), } if obs_include_prev_action_reward: - input_dict.update({ - SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( - prev_action_batch), - SampleBatch.PREV_REWARDS: tf.convert_to_tensor( - prev_reward_batch), - }) + input_dict[SampleBatch.PREV_ACTIONS] = \ + tf.convert_to_tensor(prev_action_batch) + input_dict[SampleBatch.PREV_REWARDS] = \ + tf.convert_to_tensor(prev_reward_batch) # Use Exploration object. with tf.variable_creator_scope(_disallow_var_creation): if action_sampler_fn: - dist_class = dist_inputs = None + dist_inputs = None state_out = [] actions, logp = self.action_sampler_fn( self, @@ -357,8 +361,7 @@ def build_eager_tf_policy(name, else: dist_class = self.dist_class dist_inputs, state_out = self.model( - input_dict, state_batches, - tf.convert_to_tensor([1])) + input_dict, state_batches, seq_lens) action_dist = dist_class(dist_inputs, self.model) @@ -424,8 +427,10 @@ def build_eager_tf_policy(name, else: dist_inputs, _ = self.model(input_dict, state_batches, seq_lens) - action_dist = self.dist_class(dist_inputs, self.model) - log_likelihoods = action_dist.logp(actions) + dist_class = self.dist_class + + action_dist = dist_class(dist_inputs, self.model) + log_likelihoods = action_dist.logp(actions) return log_likelihoods @@ -609,9 +614,11 @@ def build_eager_tf_policy(name, # Execute a forward pass to get self.action_dist etc initialized, # and also obtain the extra action fetches _, _, fetches = self.compute_actions( - dummy_batch[SampleBatch.CUR_OBS], self._state_in, + dummy_batch[SampleBatch.CUR_OBS], + self._state_in, dummy_batch.get(SampleBatch.PREV_ACTIONS), - dummy_batch.get(SampleBatch.PREV_REWARDS)) + dummy_batch.get(SampleBatch.PREV_REWARDS), + explore=False) dummy_batch.update(fetches) postprocessed_batch = self.postprocess_trajectory( diff --git a/rllib/policy/tf_policy.py b/rllib/policy/tf_policy.py index 368f8b990..62c27b6ef 100644 --- a/rllib/policy/tf_policy.py +++ b/rllib/policy/tf_policy.py @@ -137,7 +137,7 @@ class TFPolicy(Policy): if self._sampled_action_logp is not None else None) self._action_input = action_input # For logp calculations. - self._distr_inputs = dist_inputs + self._dist_inputs = dist_inputs self.dist_class = dist_class self._log_likelihood = log_likelihood self._state_inputs = state_inputs or [] @@ -172,9 +172,9 @@ class TFPolicy(Policy): # The log-likelihood calculator op. self._log_likelihood = None - if self._distr_inputs is not None and self.dist_class is not None: + if self._dist_inputs is not None and self.dist_class is not None: self._log_likelihood = self.dist_class( - self._distr_inputs, self.model).logp(self._action_input) + self._dist_inputs, self.model).logp(self._action_input) def variables(self): """Return the list of all savable variables for this policy.""" @@ -424,8 +424,8 @@ class TFPolicy(Policy): extra_fetches[SampleBatch.ACTION_PROB] = self._sampled_action_prob extra_fetches[SampleBatch.ACTION_LOGP] = self._sampled_action_logp # Action-dist inputs. - if self._distr_inputs is not None: - extra_fetches[SampleBatch.ACTION_DIST_INPUTS] = self._distr_inputs + if self._dist_inputs is not None: + extra_fetches[SampleBatch.ACTION_DIST_INPUTS] = self._dist_inputs return extra_fetches @DeveloperAPI @@ -577,9 +577,13 @@ class TFPolicy(Policy): if timestep is not None: builder.add_feed_dict({self._timestep: timestep}) builder.add_feed_dict(dict(zip(self._state_inputs, state_batches))) - fetches = builder.add_fetches([self._sampled_action] + - self._state_outputs + - [self.extra_compute_action_fetches()]) + + # Determine, what exactly to fetch from the graph. + to_fetch = [self._sampled_action] + self._state_outputs + \ + [self.extra_compute_action_fetches()] + + # Perform the session call. + fetches = builder.add_fetches(to_fetch) return fetches[0], fetches[1:-1], fetches[-1] def _build_compute_gradients(self, builder, postprocessed_batch): diff --git a/rllib/policy/torch_policy.py b/rllib/policy/torch_policy.py index 9df12a28a..be464eae5 100644 --- a/rllib/policy/torch_policy.py +++ b/rllib/policy/torch_policy.py @@ -106,9 +106,9 @@ class TorchPolicy(Policy): explore = explore if explore is not None else self.config["explore"] timestep = timestep if timestep is not None else self.global_timestep - seq_lens = torch.ones(len(obs_batch), dtype=torch.int32) with torch.no_grad(): + seq_lens = torch.ones(len(obs_batch), dtype=torch.int32) input_dict = self._lazy_tensor_dict({ SampleBatch.CUR_OBS: obs_batch, }) @@ -301,19 +301,15 @@ class TorchPolicy(Policy): return processing info.""" return {} - def extra_action_out(self, - input_dict, - state_batches, - model, - action_dist=None): + def extra_action_out(self, input_dict, state_batches, model, action_dist): """Returns dict of extra info to include in experience batch. - Arguments: + Args: input_dict (dict): Dict of model input tensors. state_batches (list): List of state tensors. model (TorchModelV2): Reference to the model. - action_dist (Distribution): Torch Distribution object to get - log-probs (e.g. for already sampled actions). + action_dist (TorchActionDistribution): Torch action dist object + to get log-probs (e.g. for already sampled actions). """ return {} diff --git a/rllib/policy/torch_policy_template.py b/rllib/policy/torch_policy_template.py index 27672118f..65320835d 100644 --- a/rllib/policy/torch_policy_template.py +++ b/rllib/policy/torch_policy_template.py @@ -121,15 +121,18 @@ def build_torch_policy(name, sample_batch, other_agent_batches=None, episode=None): - if not postprocess_fn: - return sample_batch - # Do all post-processing always with no_grad(). # Not using this here will introduce a memory leak (issue #6962). with torch.no_grad(): - return postprocess_fn( - self, convert_to_non_torch_type(sample_batch), + # Call super's postprocess_trajectory first. + sample_batch = super().postprocess_trajectory( + convert_to_non_torch_type(sample_batch), convert_to_non_torch_type(other_agent_batches), episode) + if postprocess_fn: + return postprocess_fn(self, sample_batch, + other_agent_batches, episode) + + return sample_batch @override(TorchPolicy) def extra_grad_process(self): @@ -139,11 +142,8 @@ def build_torch_policy(name, return TorchPolicy.extra_grad_process(self) @override(TorchPolicy) - def extra_action_out(self, - input_dict, - state_batches, - model, - action_dist=None): + def extra_action_out(self, input_dict, state_batches, model, + action_dist): with torch.no_grad(): if extra_action_out_fn: stats_dict = extra_action_out_fn( diff --git a/rllib/tests/test_rollout.py b/rllib/tests/test_rollout.py index 1364cbfb8..60f126769 100644 --- a/rllib/tests/test_rollout.py +++ b/rllib/tests/test_rollout.py @@ -49,9 +49,6 @@ def rollout_test(algo, env="CartPole-v0"): class TestRollout(unittest.TestCase): - def test_a2c(self): - rollout_test("A2C") - def test_a3c(self): rollout_test("A3C") @@ -79,9 +76,6 @@ class TestRollout(unittest.TestCase): def test_sac(self): rollout_test("SAC", env="Pendulum-v0") - def test_td3(self): - rollout_test("TD3", env="Pendulum-v0") - if __name__ == "__main__": import pytest diff --git a/rllib/tuned_examples/regression_tests/cartpole-dqn-param-noise.yaml b/rllib/tuned_examples/regression_tests/cartpole-dqn-param-noise.yaml new file mode 100644 index 000000000..7b8233a33 --- /dev/null +++ b/rllib/tuned_examples/regression_tests/cartpole-dqn-param-noise.yaml @@ -0,0 +1,17 @@ +cartpole-dqn-w-param-noise: + env: CartPole-v0 + run: DQN + stop: + episode_reward_mean: 150 + timesteps_total: 100000 + config: + exploration_config: + type: ParameterNoise + random_timesteps: 0 + initial_stddev: 1.0 + batch_mode: complete_episodes + lr: 0.001 + num_workers: 0 + model: + fcnet_hiddens: [16] + fcnet_activation: linear diff --git a/rllib/tuned_examples/regression_tests/cartpole-sac.yaml b/rllib/tuned_examples/regression_tests/cartpole-sac.yaml index 830a923fb..bfb35eb88 100644 --- a/rllib/tuned_examples/regression_tests/cartpole-sac.yaml +++ b/rllib/tuned_examples/regression_tests/cartpole-sac.yaml @@ -3,6 +3,7 @@ cartpole-sac: run: SAC stop: episode_reward_mean: 150 + timesteps_total: 100000 config: gamma: 0.95 no_done_at_end: false diff --git a/rllib/utils/exploration/exploration.py b/rllib/utils/exploration/exploration.py index 2f1442780..c31172e39 100644 --- a/rllib/utils/exploration/exploration.py +++ b/rllib/utils/exploration/exploration.py @@ -20,16 +20,16 @@ class Exploration: """ def __init__(self, action_space: Space, *, framework: str, - num_workers: int, worker_index: int, policy_config: dict, - model: ModelV2): + policy_config: dict, model: ModelV2, num_workers: int, + worker_index: int): """ Args: action_space (Space): The action space in which to explore. framework (str): One of "tf" or "torch". - num_workers (int): The overall number of workers used. - worker_index (int): The index of the worker using this class. policy_config (dict): The Policy's config dict. model (ModelV2): The Policy's model. + num_workers (int): The overall number of workers used. + worker_index (int): The index of the worker using this class. """ self.action_space = action_space self.policy_config = policy_config diff --git a/rllib/utils/exploration/parameter_noise.py b/rllib/utils/exploration/parameter_noise.py index fb41b0b41..ef0812f85 100644 --- a/rllib/utils/exploration/parameter_noise.py +++ b/rllib/utils/exploration/parameter_noise.py @@ -117,10 +117,10 @@ class ParameterNoise(Exploration): sub_exploration, framework=self.framework, action_space=self.action_space, + policy_config=self.policy_config, + model=self.model, **kwargs) - # Store the default setting for `explore`. - self.default_explore = policy_config["explore"] # Whether we need to call `self._delayed_on_episode_start` before # the forward pass. self.episode_started = False @@ -131,13 +131,14 @@ class ParameterNoise(Exploration): timestep=None, explore=None, tf_sess=None): + explore = explore if explore is not None else \ + self.policy_config["explore"] + # Is this the first forward pass in the new episode? If yes, do the # noise re-sampling and add to weights. if self.episode_started: - self._delayed_on_episode_start(tf_sess) + self._delayed_on_episode_start(explore, tf_sess) - explore = explore if explore is not None else \ - self.policy_config["explore"] # Add noise if necessary. if explore and not self.weights_are_currently_noisy: self._add_stored_noise(tf_sess=tf_sess) @@ -148,15 +149,13 @@ class ParameterNoise(Exploration): @override(Exploration) def get_exploration_action(self, *, - distribution_inputs, - action_dist_class, + action_distribution, timestep, explore=True): # Use our sub-exploration object to handle the final exploration # action (depends on the algo-type/action-space/etc..). return self.sub_exploration.get_exploration_action( - distribution_inputs=distribution_inputs, - action_dist_class=action_dist_class, + action_distribution=action_distribution, timestep=timestep, explore=explore) @@ -173,9 +172,9 @@ class ParameterNoise(Exploration): # We don't want to update into a noisy net. self.episode_started = True - def _delayed_on_episode_start(self, tf_sess): + def _delayed_on_episode_start(self, explore, tf_sess): # Sample fresh noise and add to weights. - if self.default_explore: + if explore: self._sample_new_noise_and_add(tf_sess=tf_sess, override=True) # Only sample, don't apply anything to the weights. else: @@ -227,7 +226,7 @@ class ParameterNoise(Exploration): if policy.dist_class is Categorical: action_dist = softmax(fetches[SampleBatch.ACTION_DIST_INPUTS]) - if not self.weights_are_currently_noisy: + if noisy_action_dist is None: noisy_action_dist = action_dist else: noise_free_action_dist = action_dist diff --git a/rllib/utils/exploration/random.py b/rllib/utils/exploration/random.py index b772b8843..c38a8f55a 100644 --- a/rllib/utils/exploration/random.py +++ b/rllib/utils/exploration/random.py @@ -29,8 +29,8 @@ class Random(Exploration): """ super().__init__( action_space=action_space, - framework=framework, model=model, + framework=framework, **kwargs) # Determine py_func types, depending on our action-space. @@ -87,7 +87,8 @@ class Random(Exploration): if explore: # Unsqueeze will be unnecessary, once we support batch/time-aware # Spaces. - action = tensor_fn(self.action_space.sample()).unsqueeze(0) + a = self.action_space.sample() + action = tensor_fn([a] if isinstance(a, int) else a) else: action = tensor_fn(action_dist.deterministic_sample()) logp = torch.zeros((action.size()[0], ), dtype=torch.float32) diff --git a/rllib/utils/exploration/tests/test_explorations.py b/rllib/utils/exploration/tests/test_explorations.py index c5329f112..7f8efe89a 100644 --- a/rllib/utils/exploration/tests/test_explorations.py +++ b/rllib/utils/exploration/tests/test_explorations.py @@ -67,7 +67,7 @@ def do_test_explorations(run, # Make sure all actions drawn are the same, given same # observations. actions = [] - for _ in range(50): + for _ in range(25): actions.append( trainer.compute_action( observation=dummy_obs, @@ -79,7 +79,7 @@ def do_test_explorations(run, # Make sure actions drawn are different # (around some mean value), given constant observations. actions = [] - for _ in range(100): + for _ in range(50): actions.append( trainer.compute_action( observation=dummy_obs, diff --git a/rllib/utils/numpy.py b/rllib/utils/numpy.py index c2a663c66..2751920ea 100644 --- a/rllib/utils/numpy.py +++ b/rllib/utils/numpy.py @@ -46,7 +46,10 @@ def softmax(x, axis=-1): Returns: np.ndarray: The softmax over x. """ + # x_exp = np.maximum(np.exp(x), SMALL_NUMBER) x_exp = np.exp(x) + # return x_exp / + # np.maximum(np.sum(x_exp, axis, keepdims=True), SMALL_NUMBER) return np.maximum(x_exp / np.sum(x_exp, axis, keepdims=True), SMALL_NUMBER) diff --git a/rllib/utils/test_utils.py b/rllib/utils/test_utils.py index 9c180b671..d4e8ea746 100644 --- a/rllib/utils/test_utils.py +++ b/rllib/utils/test_utils.py @@ -120,14 +120,20 @@ def check(x, y, decimals=5, atol=None, rtol=None, false=False): # Using decimals. if atol is None and rtol is None: + # Assert equality of both values. try: np.testing.assert_almost_equal(x, y, decimal=decimals) + # Both values are not equal. + except AssertionError as e: + # Raise error in normal case. + if false is False: + raise e + # Both values are equal. + else: + # If false is set -> raise error (not expected to be equal). if false is True: assert False, \ "ERROR: x ({}) is the same as y ({})!".format(x, y) - except AssertionError as e: - if false is False: - raise e # Using atol/rtol. else: @@ -138,9 +144,10 @@ def check(x, y, decimals=5, atol=None, rtol=None, false=False): rtol = 1e-7 try: np.testing.assert_allclose(x, y, atol=atol, rtol=rtol) - if false is True: - assert False, \ - "ERROR: x ({}) is the same as y ({})!".format(x, y) except AssertionError as e: if false is False: raise e + else: + if false is True: + assert False, \ + "ERROR: x ({}) is the same as y ({})!".format(x, y)