[RLlib] Deprecate old classes, methods, functions, config keys (in prep for RLlib 1.0). (#10544)

rllib/BUILD

@@ -436,14 +436,6 @@ py_test(
     srcs = ["agents/dqn/tests/test_simple_q.py"]
 )
 
-# DYNATrainer
-py_test(
-    name = "test_dyna",
-    tags = ["agents_dir"],
-    size = "medium",
-    srcs = ["agents/dyna/tests/test_dyna.py"]
-)
-
 # ES
 py_test(
     name = "test_es",

rllib/agents/__init__.py

@@ -1,4 +1,3 @@
 from ray.rllib.agents.trainer import Trainer, with_common_config
-from ray.rllib.agents.agent import Agent
 
-__all__ = ["Agent", "Trainer", "with_common_config"]
+__all__ = ["Trainer", "with_common_config"]

rllib/agents/agent.py

@@ -1,4 +0,0 @@
-from ray.rllib.agents.trainer import Trainer
-from ray.rllib.utils import renamed_agent
-
-Agent = renamed_agent(Trainer)

rllib/agents/ars/ars_tf_policy.py

@@ -3,19 +3,18 @@
 
 import gym
 import numpy as np
+import tree
 
 import ray
 import ray.experimental.tf_utils
 from ray.rllib.agents.es.es_tf_policy import make_session
 from ray.rllib.models import ModelCatalog
 from ray.rllib.policy.sample_batch import SampleBatch
-from ray.rllib.utils import try_import_tree
 from ray.rllib.utils.filter import get_filter
 from ray.rllib.utils.framework import try_import_tf
 from ray.rllib.utils.spaces.space_utils import unbatch
 
 tf1, tf, tfv = try_import_tf()
-tree = try_import_tree()
 
 
 class ARSTFPolicy:

rllib/agents/ddpg/ddpg.py

@@ -3,8 +3,6 @@ import logging
 from ray.rllib.agents.trainer import with_common_config
 from ray.rllib.agents.dqn.dqn import GenericOffPolicyTrainer
 from ray.rllib.agents.ddpg.ddpg_tf_policy import DDPGTFPolicy
-from ray.rllib.utils.deprecation import deprecation_warning, \
-    DEPRECATED_VALUE
 
 logger = logging.getLogger(__name__)
 
@@ -147,9 +145,6 @@ DEFAULT_CONFIG = with_common_config({
     "worker_side_prioritization": False,
     # Prevent iterations from going lower than this time span
     "min_iter_time_s": 1,
-
-    # Deprecated keys.
-    "grad_norm_clipping": DEPRECATED_VALUE,
 })
 # __sphinx_doc_end__
 # yapf: enable
@@ -162,10 +157,6 @@ def validate_config(config):
             "was specified.")
         config["use_state_preprocessor"] = True
 
-    if config.get("grad_norm_clipping", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning("grad_norm_clipping", "grad_clip")
-        config["grad_clip"] = config.pop("grad_norm_clipping")
-
     if config["grad_clip"] is not None and config["grad_clip"] <= 0.0:
         raise ValueError("`grad_clip` value must be > 0.0!")
 

rllib/agents/dqn/dqn.py

@@ -5,8 +5,6 @@ from ray.rllib.agents.trainer_template import build_trainer
 from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy
 from ray.rllib.agents.dqn.simple_q_tf_policy import SimpleQTFPolicy
 from ray.rllib.policy.policy import LEARNER_STATS_KEY
-from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE
-from ray.rllib.utils.exploration import PerWorkerEpsilonGreedy
 from ray.rllib.execution.replay_buffer import LocalReplayBuffer
 from ray.rllib.execution.rollout_ops import ParallelRollouts
 from ray.rllib.execution.concurrency_ops import Concurrently
@@ -119,19 +117,6 @@ DEFAULT_CONFIG = with_common_config({
     "worker_side_prioritization": False,
     # Prevent iterations from going lower than this time span
     "min_iter_time_s": 1,
-
-    # DEPRECATED VALUES (set to -1 to indicate they have not been overwritten
-    # by user's config). If we don't set them here, we will get an error
-    # from the config-key checker.
-    "schedule_max_timesteps": DEPRECATED_VALUE,
-    "exploration_final_eps": DEPRECATED_VALUE,
-    "exploration_fraction": DEPRECATED_VALUE,
-    "beta_annealing_fraction": DEPRECATED_VALUE,
-    "per_worker_exploration": DEPRECATED_VALUE,
-    "softmax_temp": DEPRECATED_VALUE,
-    "soft_q": DEPRECATED_VALUE,
-    "parameter_noise": DEPRECATED_VALUE,
-    "grad_norm_clipping": DEPRECATED_VALUE,
 })
 # __sphinx_doc_end__
 # yapf: enable
@@ -142,73 +127,6 @@ def validate_config(config):
 
     Rewrites rollout_fragment_length to take into account n_step truncation.
     """
-    # TODO(sven): Remove at some point.
-    #  Backward compatibility of epsilon-exploration config AND beta-annealing
-    # fraction settings (both based on schedule_max_timesteps, which is
-    # deprecated).
-    if config.get("grad_norm_clipping", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning("grad_norm_clipping", "grad_clip")
-        config["grad_clip"] = config.pop("grad_norm_clipping")
-
-    schedule_max_timesteps = None
-    if config.get("schedule_max_timesteps", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        deprecation_warning(
-            "schedule_max_timesteps",
-            "exploration_config.epsilon_timesteps AND "
-            "prioritized_replay_beta_annealing_timesteps")
-        schedule_max_timesteps = config["schedule_max_timesteps"]
-    if config.get("exploration_final_eps", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        deprecation_warning("exploration_final_eps",
-                            "exploration_config.final_epsilon")
-        if isinstance(config["exploration_config"], dict):
-            config["exploration_config"]["final_epsilon"] = \
-                config.pop("exploration_final_eps")
-    if config.get("exploration_fraction", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        assert schedule_max_timesteps is not None
-        deprecation_warning("exploration_fraction",
-                            "exploration_config.epsilon_timesteps")
-        if isinstance(config["exploration_config"], dict):
-            config["exploration_config"]["epsilon_timesteps"] = config.pop(
-                "exploration_fraction") * schedule_max_timesteps
-    if config.get("beta_annealing_fraction", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        assert schedule_max_timesteps is not None
-        deprecation_warning(
-            "beta_annealing_fraction (decimal)",
-            "prioritized_replay_beta_annealing_timesteps (int)")
-        config["prioritized_replay_beta_annealing_timesteps"] = config.pop(
-            "beta_annealing_fraction") * schedule_max_timesteps
-    if config.get("per_worker_exploration", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        deprecation_warning("per_worker_exploration",
-                            "exploration_config.type=PerWorkerEpsilonGreedy")
-        if isinstance(config["exploration_config"], dict):
-            config["exploration_config"]["type"] = PerWorkerEpsilonGreedy
-    if config.get("softmax_temp", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning(
-            "soft_q", "exploration_config={"
-            "type=StochasticSampling, temperature=[float]"
-            "}")
-        if config.get("softmax_temp", 1.0) < 0.00001:
-            logger.warning("softmax temp very low: Clipped it to 0.00001.")
-            config["softmax_temperature"] = 0.00001
-    if config.get("soft_q", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning(
-            "soft_q", "exploration_config={"
-            "type=SoftQ, temperature=[float]"
-            "}")
-        config["exploration_config"] = {
-            "type": "SoftQ",
-            "temperature": config.get("softmax_temp", 1.0)
-        }
-    if config.get("parameter_noise", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning("parameter_noise", "exploration_config={"
-                            "type=ParameterNoise"
-                            "}")
-
     if config["exploration_config"]["type"] == "ParameterNoise":
         if config["batch_mode"] != "complete_episodes":
             logger.warning(

rllib/agents/dyna/__init__.py

@@ -1,10 +0,0 @@
-from ray.rllib.agents.dyna.dyna import DYNATrainer, DEFAULT_CONFIG
-from ray.rllib.agents.dyna.dyna_torch_policy import dyna_torch_loss, \
-    DYNATorchPolicy
-
-__all__ = [
-    "dyna_torch_loss",
-    "DEFAULT_CONFIG",
-    "DYNATorchPolicy",
-    "DYNATrainer",
-]

rllib/agents/dyna/dyna.py

@@ -1,101 +0,0 @@
-import logging
-
-from ray.rllib.agents.trainer import with_common_config
-from ray.rllib.agents.trainer_template import build_trainer
-
-logger = logging.getLogger(__name__)
-
-# yapf: disable
-# __sphinx_doc_begin__
-DEFAULT_CONFIG = with_common_config({
-    # Default Trainer setting overrides.
-    "num_workers": 1,
-    "num_envs_per_worker": 1,
-
-    # The size of an entire epoch (for supervised learning the dynamics).
-    # The train-batch will be split into training and validation sets according
-    # to `training_set_ratio`, then n epochs (with minibatch
-    # size=`sgd_minibatch_size`) will be trained until the sliding average
-    # of the validation performance decreases.
-    "train_batch_size": 10000,
-    "sgd_minibatch_size": 500,
-    "rollout_fragment_length": 200,
-    # Learning rate for the dynamics optimizer.
-    "lr": 0.0003,
-
-    # Fraction of the entire data that should be used for training the dynamics
-    # model. The validation fraction is 1.0 - `training_set_ratio`. Training of
-    # a dynamics model over n some epochs (1 epoch = entire training set) stops
-    # when the validation set's performance starts to decrease.
-    "train_set_ratio": 0.8,
-
-    # The exploration strategy to apply on top of the (acting) policy.
-    # TODO: (sven) Use random for testing purposes for now.
-    "exploration_config": {"type": "Random"},
-
-    # Whether to predict the action that lead from obs(t) to obs(t+1), instead
-    # of predicting obs(t+1).
-    "predict_action": False,
-
-    # Whether the dynamics model should predict the reward, given obs(t)+a(t).
-    # NOTE: Only supported if `predict_action`=False.
-    "predict_reward": False,
-
-    # Whether to use the same network for predicting rewards than for
-    # predicting the next observation.
-    "reward_share_layers": True,
-
-    # TODO: (sven) figure out API to query the latent space vector given
-    #  some observation (not needed for MBMPO).
-    "learn_latent_space": False,
-
-    # Whether to predict `obs(t+1) - obs(t)` instead of `obs(t+1)` directly.
-    # NOTE: This only works for 1D Box observation spaces, e.g. Box(5,) and
-    # if `predict_action`=False.
-    "predict_obs_delta": True,
-    # TODO: loss function types: neg_log_llh, etc..?
-    "loss_function": "l2",
-
-    # Config for the dynamics learning model architecture.
-    "dynamics_model": {
-        "fcnet_hiddens": [512, 512],
-        "fcnet_activation": "relu",
-    },
-
-    # TODO: (sven) allow for having a default model config over many
-    #  sub-models: e.g. "model": {"ModelA": {[default_config]},
-    #  "ModelB": [default_config]}
-})
-# __sphinx_doc_end__
-# yapf: enable
-
-
-def validate_config(config):
-    if config["train_set_ratio"] <= 0.0 or \
-            config["train_set_ratio"] >= 1.0:
-        raise ValueError("`train_set_ratio` must be within (0.0, 1.0)!")
-    if config["predict_action"] or config["predict_reward"]:
-        raise ValueError(
-            "`predict_action`=True or `predict_reward`=True not supported "
-            "yet!")
-    if config["learn_latent_space"]:
-        raise ValueError("`learn_latent_space` not supported yet!")
-    if config["loss_function"] != "l2":
-        raise ValueError("`loss_function` other than 'l2' not supported yet!")
-
-
-def get_policy_class(config):
-    if config["framework"] == "torch":
-        from ray.rllib.agents.dyna.dyna_torch_policy import DYNATorchPolicy
-        return DYNATorchPolicy
-    else:
-        raise ValueError("tf not supported yet!")
-
-
-DYNATrainer = build_trainer(
-    name="DYNA",
-    default_policy=None,
-    get_policy_class=get_policy_class,
-    default_config=DEFAULT_CONFIG,
-    validate_config=validate_config,
-)

rllib/agents/dyna/dyna_torch_model.py

@@ -1,58 +0,0 @@
-import gym
-from gym.spaces import Discrete
-
-from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
-from ray.rllib.utils.framework import try_import_torch
-
-torch, nn = try_import_torch()
-
-
-class DYNATorchModel(TorchModelV2, nn.Module):
-    """Extension of standard TorchModelV2 for Env dynamics learning.
-
-    Data flow:
-        obs.cat(action) -> forward() -> next_obs|next_obs_delta
-        get_next_state(obs, action) -> next_obs|next_obs_delta
-
-    Note that this class by itself is not a valid model unless you
-    implement forward() in a subclass.
-    """
-
-    def __init__(self, obs_space, action_space, num_outputs, model_config,
-                 name):
-        """Initializes a DYNATorchModel object.
-        """
-
-        nn.Module.__init__(self)
-        # Construct the wrapped model handing it a concat'd observation and
-        # action space as "input_space" and our obs_space as "output_space".
-        # TODO: (sven) get rid of these restrictions on obs/action spaces.
-        assert isinstance(action_space, Discrete)
-        input_space = gym.spaces.Box(
-            obs_space.low[0],
-            obs_space.high[0],
-            shape=(obs_space.shape[0] + action_space.n, ))
-        super(DYNATorchModel, self).__init__(input_space, action_space,
-                                             num_outputs, model_config, name)
-
-    def get_next_observation(self, observations, actions):
-        """Returns a next obs prediction given current observation and action.
-
-        This implements p^(s'|s, a). With p being the environment dynamics.
-
-        Arguments:
-            observations (Tensor): The current observation Tensor.
-            actions (Tensor): The actions taken in `observations`.
-
-        Returns:
-            TensorType: The predicted next observations.
-        """
-
-        # One-hot the actions.
-        actions_flat = nn.functional.one_hot(
-            actions.long(), num_classes=self.action_space.n).float()
-        # Push through our underlying Model.
-        next_obs, _ = self.forward({
-            "obs_flat": torch.cat([observations, actions_flat], -1)
-        }, [], None)
-        return next_obs

rllib/agents/dyna/dyna_torch_policy.py

@@ -1,94 +0,0 @@
-import gym
-import logging
-
-import ray
-from ray.rllib.agents.dyna.dyna_torch_model import DYNATorchModel
-from ray.rllib.models.catalog import ModelCatalog
-from ray.rllib.policy.sample_batch import SampleBatch
-from ray.rllib.policy.torch_policy_template import build_torch_policy
-from ray.rllib.utils import try_import_torch
-
-torch, nn = try_import_torch()
-
-logger = logging.getLogger(__name__)
-
-
-def make_model_and_dist(policy, obs_space, action_space, config):
-    # Get the output distribution class for predicting rewards and next-obs.
-    policy.distr_cls_next_obs, num_outputs = ModelCatalog.get_action_dist(
-        obs_space, config, dist_type="deterministic", framework="torch")
-    if config["predict_reward"]:
-        # TODO: (sven) implement reward prediction.
-        _ = ModelCatalog.get_action_dist(
-            gym.spaces.Box(float("-inf"), float("inf"), ()),
-            config,
-            dist_type="")
-
-    # Build one dynamics model if we are a Worker.
-    # If we are the main MAML learner, build n (num_workers) dynamics Models
-    # for being able to create checkpoints for the current state of training.
-    policy.dynamics_model = ModelCatalog.get_model_v2(
-        obs_space,
-        action_space,
-        num_outputs=num_outputs,
-        model_config=config["dynamics_model"],
-        framework="torch",
-        name="dynamics_model",
-        model_interface=DYNATorchModel,
-    )
-
-    action_dist, num_outputs = ModelCatalog.get_action_dist(
-        action_space, config, dist_type="deterministic", framework="torch")
-    # Create the pi-model and register it with the Policy.
-    policy.pi = ModelCatalog.get_model_v2(
-        obs_space,
-        action_space,
-        num_outputs=num_outputs,
-        model_config=config["model"],
-        framework="torch",
-        name="policy_model",
-    )
-
-    return policy.pi, action_dist
-
-
-def dyna_torch_loss(policy, model, dist_class, train_batch):
-    # Split batch into train and validation sets according to
-    # `train_set_ratio`.
-    predicted_next_state_deltas = \
-        policy.dynamics_model.get_next_observation(
-            train_batch[SampleBatch.CUR_OBS], train_batch[SampleBatch.ACTIONS])
-    labels = train_batch[SampleBatch.NEXT_OBS] - train_batch[SampleBatch.
-                                                             CUR_OBS]
-    loss = torch.pow(
-        torch.sum(
-            torch.pow(labels - predicted_next_state_deltas, 2.0), dim=-1), 0.5)
-    batch_size = int(loss.shape[0])
-    train_set_size = int(batch_size * policy.config["train_set_ratio"])
-    train_loss, validation_loss = \
-        torch.split(loss, (train_set_size, batch_size - train_set_size), dim=0)
-    policy.dynamics_train_loss = torch.mean(train_loss)
-    policy.dynamics_validation_loss = torch.mean(validation_loss)
-    return policy.dynamics_train_loss
-
-
-def stats_fn(policy, train_batch):
-    return {
-        "dynamics_train_loss": policy.dynamics_train_loss,
-        "dynamics_validation_loss": policy.dynamics_validation_loss,
-    }
-
-
-def torch_optimizer(policy, config):
-    return torch.optim.Adam(
-        policy.dynamics_model.parameters(), lr=config["lr"])
-
-
-DYNATorchPolicy = build_torch_policy(
-    name="DYNATorchPolicy",
-    loss_fn=dyna_torch_loss,
-    get_default_config=lambda: ray.rllib.agents.dyna.dyna.DEFAULT_CONFIG,
-    stats_fn=stats_fn,
-    optimizer_fn=torch_optimizer,
-    make_model_and_action_dist=make_model_and_dist,
-)

rllib/agents/dyna/tests/test_dyna.py

@@ -1,72 +0,0 @@
-import copy
-import gym
-import numpy as np
-import unittest
-
-import ray
-import ray.rllib.agents.dyna as dyna
-from ray.rllib.utils.framework import try_import_torch
-from ray.rllib.utils.test_utils import check_compute_single_action, \
-    framework_iterator
-
-torch, _ = try_import_torch()
-
-
-class TestDYNA(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        ray.init(local_mode=True)
-
-    @classmethod
-    def tearDownClass(cls):
-        ray.shutdown()
-
-    def test_dyna_compilation(self):
-        """Test whether a DYNATrainer can be built with both frameworks."""
-        config = copy.deepcopy(dyna.DEFAULT_CONFIG)
-        config["num_workers"] = 1
-        config["train_batch_size"] = 1000
-        num_iterations = 30
-        env = "CartPole-v0"
-        test_env = gym.make(env)
-
-        for _ in framework_iterator(config, frameworks="torch"):
-            trainer = dyna.DYNATrainer(config=config, env=env)
-            policy = trainer.get_policy()
-            # Do n supervised epochs, each over `train_batch_size`.
-            # Ignore validation loss here as a stopping criteria.
-            for i in range(num_iterations):
-                info = trainer.train()["info"]["learner"]["default_policy"]
-                print("SL iteration: {}".format(i))
-                print("train loss {}".format(info["dynamics_train_loss"]))
-                print("validation loss {}".format(
-                    info["dynamics_validation_loss"]))
-            # Check, whether normal action stepping works with DYNA's policy.
-            # Note that DYNA does not train its Policy. It must be pushed
-            # down from the main model-based algo from time to time.
-            check_compute_single_action(trainer)
-
-            # Check, whether env dynamics were actually learnt - more or less.
-            obs = test_env.reset()
-            for _ in range(10):
-                action = trainer.compute_action(obs)
-                obs = torch.from_numpy(np.array([obs])).float()
-                # Make the prediction over the next state (deterministic delta
-                # like in MBMPO).
-                predicted_next_obs_delta = \
-                    policy.dynamics_model.get_next_observation(
-                        obs,
-                        torch.from_numpy(np.array([action])))
-                predicted_next_obs = obs + predicted_next_obs_delta
-                obs, _, done, _ = test_env.step(action)
-                self.assertLess(
-                    np.sum(obs - predicted_next_obs.detach().numpy()), 0.05)
-                # Reset if done.
-                if done:
-                    obs = test_env.reset()
-
-
-if __name__ == "__main__":
-    import pytest
-    import sys
-    sys.exit(pytest.main(["-v", __file__]))

rllib/agents/qmix/qmix.py

@@ -7,7 +7,6 @@ from ray.rllib.execution.rollout_ops import ParallelRollouts, ConcatBatches
 from ray.rllib.execution.train_ops import TrainOneStep, UpdateTargetNetwork
 from ray.rllib.execution.metric_ops import StandardMetricsReporting
 from ray.rllib.execution.concurrency_ops import Concurrently
-from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE
 
 # yapf: disable
 # __sphinx_doc_begin__
@@ -96,46 +95,11 @@ DEFAULT_CONFIG = with_common_config({
         "lstm_cell_size": 64,
         "max_seq_len": 999999,
     },
-
-    # DEPRECATED VALUES (set to -1 to indicate they have not been overwritten
-    # by user's config). If we don't set them here, we will get an error
-    # from the config-key checker.
-    "schedule_max_timesteps": DEPRECATED_VALUE,
-    "exploration_fraction": DEPRECATED_VALUE,
-    "exploration_initial_eps": DEPRECATED_VALUE,
-    "exploration_final_eps": DEPRECATED_VALUE,
-
 })
 # __sphinx_doc_end__
 # yapf: enable
 
 
-def validate_config(config):
-    schedule_max_timesteps = None
-    if config.get("schedule_max_timesteps", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        deprecation_warning(
-            "schedule_max_timesteps",
-            "exploration_config.epsilon_timesteps AND "
-            "prioritized_replay_beta_annealing_timesteps")
-        schedule_max_timesteps = config["schedule_max_timesteps"]
-    if config.get("exploration_final_eps", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        deprecation_warning("exploration_final_eps",
-                            "exploration_config.final_epsilon")
-        if isinstance(config["exploration_config"], dict):
-            config["exploration_config"]["final_epsilon"] = \
-                config.pop("exploration_final_eps")
-    if config.get("exploration_fraction", DEPRECATED_VALUE) != \
-            DEPRECATED_VALUE:
-        assert schedule_max_timesteps is not None
-        deprecation_warning("exploration_fraction",
-                            "exploration_config.epsilon_timesteps")
-        if isinstance(config["exploration_config"], dict):
-            config["exploration_config"]["epsilon_timesteps"] = config.pop(
-                "exploration_fraction") * schedule_max_timesteps
-
-
 def execution_plan(workers, config):
     rollouts = ParallelRollouts(workers, mode="bulk_sync")
     replay_buffer = SimpleReplayBuffer(config["buffer_size"])
@@ -161,5 +125,4 @@ QMixTrainer = GenericOffPolicyTrainer.with_updates(
     default_config=DEFAULT_CONFIG,
     default_policy=QMixTorchPolicy,
     get_policy_class=None,
-    validate_config=validate_config,
     execution_plan=execution_plan)

rllib/agents/sac/sac.py

@@ -3,7 +3,6 @@ import logging
 from ray.rllib.agents.trainer import with_common_config
 from ray.rllib.agents.dqn.dqn import GenericOffPolicyTrainer
 from ray.rllib.agents.sac.sac_tf_policy import SACTFPolicy
-from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE
 
 logger = logging.getLogger(__name__)
 
@@ -23,15 +22,11 @@ DEFAULT_CONFIG = with_common_config({
     "Q_model": {
         "fcnet_activation": "relu",
         "fcnet_hiddens": [256, 256],
-        "hidden_activation": DEPRECATED_VALUE,
-        "hidden_layer_sizes": DEPRECATED_VALUE,
     },
     # RLlib model options for the policy function.
     "policy_model": {
         "fcnet_activation": "relu",
         "fcnet_hiddens": [256, 256],
-        "hidden_activation": DEPRECATED_VALUE,
-        "hidden_layer_sizes": DEPRECATED_VALUE,
     },
     # Unsquash actions to the upper and lower bounds of env's action space.
     # Ignored for discrete action spaces.
@@ -117,11 +112,6 @@ DEFAULT_CONFIG = with_common_config({
     # Use a Beta-distribution instead of a SquashedGaussian for bounded,
     # continuous action spaces (not recommended, for debugging only).
     "_use_beta_distribution": False,
-
-    # DEPRECATED VALUES (set to -1 to indicate they have not been overwritten
-    # by user's config). If we don't set them here, we will get an error
-    # from the config-key checker.
-    "grad_norm_clipping": DEPRECATED_VALUE,
 })
 # __sphinx_doc_end__
 # yapf: enable
@@ -142,28 +132,9 @@ def validate_config(config):
             "was specified.")
         config["use_state_preprocessor"] = True
 
-    if config.get("grad_norm_clipping", DEPRECATED_VALUE) != DEPRECATED_VALUE:
-        deprecation_warning("grad_norm_clipping", "grad_clip")
-        config["grad_clip"] = config.pop("grad_norm_clipping")
-
     if config["grad_clip"] is not None and config["grad_clip"] <= 0.0:
         raise ValueError("`grad_clip` value must be > 0.0!")
 
-    # Use same keys as for standard Trainer "model" config.
-    for model in ["Q_model", "policy_model"]:
-        if config[model].get("hidden_activation", DEPRECATED_VALUE) != \
-                DEPRECATED_VALUE:
-            deprecation_warning(
-                "{}.hidden_activation".format(model),
-                "{}.fcnet_activation".format(model),
-                error=True)
-        if config[model].get("hidden_layer_sizes", DEPRECATED_VALUE) != \
-                DEPRECATED_VALUE:
-            deprecation_warning(
-                "{}.hidden_layer_sizes".format(model),
-                "{}.fcnet_hiddens".format(model),
-                error=True)
-
 
 SACTrainer = GenericOffPolicyTrainer.with_updates(
     name="SAC",

rllib/agents/trainer.py

@@ -23,7 +23,6 @@ from ray.rllib.utils import FilterManager, deep_update, merge_dicts
 from ray.rllib.utils.spaces import space_utils
 from ray.rllib.utils.framework import try_import_tf, TensorStructType
 from ray.rllib.utils.annotations import override, PublicAPI, DeveloperAPI
-from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning
 from ray.rllib.utils.from_config import from_config
 from ray.rllib.utils.typing import TrainerConfigDict, \
     PartialTrainerConfigDict, EnvInfoDict, ResultDict, EnvType, PolicyID
@@ -69,8 +68,6 @@ COMMON_CONFIG: TrainerConfigDict = {
     # The dataflow here can vary per algorithm. For example, PPO further
     # divides the train batch into minibatches for multi-epoch SGD.
     "rollout_fragment_length": 200,
-    # Deprecated; renamed to `rollout_fragment_length` in 0.8.4.
-    "sample_batch_size": DEPRECATED_VALUE,
     # Whether to rollout "complete_episodes" or "truncate_episodes" to
     # `rollout_fragment_length` length unrolls. Episode truncation guarantees
     # evenly sized batches, but increases variance as the reward-to-go will
@@ -379,10 +376,6 @@ COMMON_CONFIG: TrainerConfigDict = {
     # The number of contiguous environment steps to replay at once. This may
     # be set to greater than 1 to support recurrent models.
     "replay_sequence_length": 1,
-
-    # Deprecated keys:
-    "use_pytorch": DEPRECATED_VALUE,  # Replaced by `framework=torch`.
-    "eager": DEPRECATED_VALUE,  # Replaced by `framework=tfe`.
 }
 # __sphinx_doc_end__
 # yapf: enable
@@ -585,18 +578,6 @@ class Trainer(Trainable):
                                                     config)
 
         # Check and resolve DL framework settings.
-        if "use_pytorch" in self.config and \
-                self.config["use_pytorch"] != DEPRECATED_VALUE:
-            deprecation_warning("use_pytorch", "framework=torch", error=False)
-            if self.config["use_pytorch"]:
-                self.config["framework"] = "torch"
-            self.config.pop("use_pytorch")
-        if "eager" in self.config and self.config["eager"] != DEPRECATED_VALUE:
-            deprecation_warning("eager", "framework=tfe", error=False)
-            if self.config["eager"]:
-                self.config["framework"] = "tfe"
-            self.config.pop("eager")
-
         # Enable eager/tracing support.
         if tf1 and self.config["framework"] in ["tf2", "tfe"]:
             if self.config["framework"] == "tf2" and tfv < 2:
@@ -1050,17 +1031,6 @@ class Trainer(Trainable):
                               _allow_unknown_configs: Optional[bool] = None
                               ) -> TrainerConfigDict:
         config1 = copy.deepcopy(config1)
-        # Error if trainer default has deprecated value.
-        if config1["sample_batch_size"] != DEPRECATED_VALUE:
-            deprecation_warning(
-                "sample_batch_size", new="rollout_fragment_length", error=True)
-        # Warning if user override config has deprecated value.
-        if ("sample_batch_size" in config2
-                and config2["sample_batch_size"] != DEPRECATED_VALUE):
-            deprecation_warning(
-                "sample_batch_size", new="rollout_fragment_length")
-            config2["rollout_fragment_length"] = config2["sample_batch_size"]
-            del config2["sample_batch_size"]
         if "callbacks" in config2 and type(config2["callbacks"]) is dict:
             legacy_callbacks_dict = config2["callbacks"]
 
@@ -1088,19 +1058,6 @@ class Trainer(Trainable):
             raise ValueError("`model._time_major` only supported "
                              "iff `_use_trajectory_view_api` is True!")
 
-        if "policy_graphs" in config["multiagent"]:
-            deprecation_warning("policy_graphs", "policies")
-            # Backwards compatibility.
-            config["multiagent"]["policies"] = config["multiagent"].pop(
-                "policy_graphs")
-        if "gpu" in config:
-            deprecation_warning("gpu", "num_gpus=0|1", error=True)
-        if "gpu_fraction" in config:
-            deprecation_warning(
-                "gpu_fraction", "num_gpus=<fraction>", error=True)
-        if "use_gpu_for_workers" in config:
-            deprecation_warning(
-                "use_gpu_for_workers", "num_gpus_per_worker=1", error=True)
         if type(config["input_evaluation"]) != list:
             raise ValueError(
                 "`input_evaluation` must be a list of strings, got {}".format(

rllib/contrib/alpha_zero/core/alpha_zero_trainer.py

@@ -10,7 +10,7 @@ from ray.rllib.execution.concurrency_ops import Concurrently
 from ray.rllib.execution.train_ops import TrainOneStep
 from ray.rllib.execution.metric_ops import StandardMetricsReporting
 from ray.rllib.models.catalog import ModelCatalog
-from ray.rllib.models.model import restore_original_dimensions
+from ray.rllib.models.modelv2 import restore_original_dimensions
 from ray.rllib.models.torch.torch_action_dist import TorchCategorical
 from ray.rllib.utils.framework import try_import_torch
 from ray.tune.registry import ENV_CREATOR, _global_registry

rllib/contrib/alpha_zero/models/custom_torch_models.py

@@ -1,7 +1,7 @@
 from abc import ABC
 import numpy as np
 
-from ray.rllib.models.model import restore_original_dimensions
+from ray.rllib.models.modelv2 import restore_original_dimensions
 from ray.rllib.models.preprocessors import get_preprocessor
 from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
 from ray.rllib.utils.framework import try_import_torch

rllib/contrib/bandits/agents/policy.py

@@ -8,7 +8,7 @@ from ray.rllib.contrib.bandits.models.linear_regression import \
     DiscreteLinearModelUCB, DiscreteLinearModel, \
     ParametricLinearModelThompsonSampling, ParametricLinearModelUCB
 from ray.rllib.models.catalog import ModelCatalog
-from ray.rllib.models.model import restore_original_dimensions
+from ray.rllib.models.modelv2 import restore_original_dimensions
 from ray.rllib.policy.policy import LEARNER_STATS_KEY
 from ray.rllib.policy.sample_batch import SampleBatch
 from ray.rllib.policy.torch_policy import TorchPolicy

rllib/contrib/bandits/examples/simple_context_bandit.py

@@ -6,6 +6,7 @@ from gym.spaces import Discrete, Box
 import numpy as np
 import random
 
+import ray
 from ray import tune
 from ray.rllib.utils.test_utils import check_learning_achieved
 
@@ -40,6 +41,7 @@ class SimpleContextualBandit(gym.Env):
 
 
 if __name__ == "__main__":
+    ray.init(num_cpus=3)
     args = parser.parse_args()
 
     stop = {

rllib/dyna.yaml

@@ -1,17 +0,0 @@
-dynamics-dyna:
-    env: 
-        grid_search:
-            - HalfCheetah-v2
-            - Humanoid-v2
-            - Ant-v2
-            - Hopper-v2
-    run: DYNA
-    local_dir: ~/dyna_results
-    stop:
-      training_iteration: 4000
-    config:
-        # Works for both torch and tf.
-        framework: torch
-        rollout_fragment_length: 200
-        train_batch_size: 1000
-        num_workers: 1