[RLlib] Deprecate vf_share_layers in top-level PPO/MAML/MB-MPO configs. (#13397)

doc/source/rllib-models.rst

@@ -604,7 +604,7 @@ Custom models can be used to work with environments where (1) the set of valid a
             return action_logits + inf_mask, state
 
 
-Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_actions_cartpole.py>`__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `algorithm overview <rllib-algorithms.html#available-algorithms-overview>`__.
+Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py <https://github.com/ray-project/ray/blob/master/rllib/examples/parametric_actions_cartpole.py>`__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``model.vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `algorithm overview <rllib-algorithms.html#available-algorithms-overview>`__.
 
 
 Autoregressive Action Distributions

release/rllib_tests/regression_tests/compact-regression-tests-tf.yaml

@@ -129,7 +129,8 @@ ppo-tf-atari:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true
         num_gpus: 1
 
 # Expect roughly 1000 reward after 1h on 1GPU

release/rllib_tests/regression_tests/compact-regression-tests-torch.yaml

@@ -127,7 +127,8 @@ ppo-torch-atari:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true
         num_gpus: 1
 
 # Expect roughly 1000 reward after 1h on 1GPU

rllib/agents/maml/maml.py

@@ -1,6 +1,6 @@
 import logging
-
 import numpy as np
+
 from ray.rllib.utils.sgd import standardized
 from ray.rllib.agents import with_common_config
 from ray.rllib.agents.maml.maml_tf_policy import MAMLTFPolicy
@@ -11,8 +11,9 @@ from ray.rllib.execution.common import STEPS_SAMPLED_COUNTER, \
     STEPS_TRAINED_COUNTER, LEARNER_INFO, _get_shared_metrics
 from ray.rllib.policy.sample_batch import SampleBatch
 from ray.rllib.execution.metric_ops import CollectMetrics
-from ray.util.iter import from_actors
 from ray.rllib.evaluation.metrics import collect_metrics
+from ray.rllib.utils.deprecation import DEPRECATED_VALUE
+from ray.util.iter import from_actors
 
 logger = logging.getLogger(__name__)
 
@@ -32,8 +33,10 @@ DEFAULT_CONFIG = with_common_config({
     "create_env_on_driver": True,
     # Stepsize of SGD
     "lr": 1e-3,
-    # Share layers for value function
-    "vf_share_layers": False,
+    "model": {
+        # Share layers for value function.
+        "vf_share_layers": False,
+    },
     # Coefficient of the value function loss
     "vf_loss_coeff": 0.5,
     # Coefficient of the entropy regularizer
@@ -59,6 +62,12 @@ DEFAULT_CONFIG = with_common_config({
     "inner_lr": 0.1,
     # Use Meta Env Template
     "use_meta_env": True,
+
+    # Deprecated keys:
+    # Share layers for value function. If you set this to True, it's important
+    # to tune vf_loss_coeff.
+    # Use config.model.vf_share_layers instead.
+    "vf_share_layers": DEPRECATED_VALUE,
 })
 # __sphinx_doc_end__
 # yapf: enable

rllib/agents/mbmpo/mbmpo.py

@@ -29,6 +29,7 @@ from ray.rllib.execution.common import STEPS_SAMPLED_COUNTER, \
     STEPS_TRAINED_COUNTER, LEARNER_INFO, _get_shared_metrics
 from ray.rllib.execution.metric_ops import CollectMetrics
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
+from ray.rllib.utils.deprecation import DEPRECATED_VALUE
 from ray.rllib.utils.sgd import standardized
 from ray.rllib.utils.torch_ops import convert_to_torch_tensor
 from ray.rllib.utils.typing import EnvType, TrainerConfigDict
@@ -55,8 +56,6 @@ DEFAULT_CONFIG = with_common_config({
     "create_env_on_driver": True,
     # Step size of SGD.
     "lr": 1e-3,
-    # Share layers for value function.
-    "vf_share_layers": False,
     # Coefficient of the value function loss.
     "vf_loss_coeff": 0.5,
     # Coefficient of the entropy regularizer.
@@ -110,6 +109,12 @@ DEFAULT_CONFIG = with_common_config({
     "custom_vector_env": model_vector_env,
     # How many iterations through MAML per MBMPO iteration.
     "num_maml_steps": 10,
+
+    # Deprecated keys:
+    # Share layers for value function. If you set this to True, it's important
+    # to tune vf_loss_coeff.
+    # Use config.model.vf_share_layers instead.
+    "vf_share_layers": DEPRECATED_VALUE,
 })
 # __sphinx_doc_end__
 # yapf: enable

rllib/agents/ppo/ppo.py

@@ -22,6 +22,7 @@ from ray.rllib.execution.train_ops import TrainOneStep, TrainTFMultiGPU
 from ray.rllib.execution.metric_ops import StandardMetricsReporting
 from ray.rllib.policy.policy import Policy
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
+from ray.rllib.utils.deprecation import DEPRECATED_VALUE
 from ray.rllib.utils.typing import TrainerConfigDict
 from ray.util.iter import LocalIterator
 
@@ -60,12 +61,14 @@ DEFAULT_CONFIG = with_common_config({
     "lr": 5e-5,
     # Learning rate schedule.
     "lr_schedule": None,
-    # Share layers for value function. If you set this to True, it's important
-    # to tune vf_loss_coeff.
-    "vf_share_layers": False,
     # Coefficient of the value function loss. IMPORTANT: you must tune this if
-    # you set vf_share_layers: True.
+    # you set vf_share_layers=True inside your model's config.
     "vf_loss_coeff": 1.0,
+    "model": {
+        # Share layers for value function. If you set this to True, it's
+        # important to tune vf_loss_coeff.
+        "vf_share_layers": False,
+    },
     # Coefficient of the entropy regularizer.
     "entropy_coeff": 0.0,
     # Decay schedule for the entropy regularizer.
@@ -90,6 +93,12 @@ DEFAULT_CONFIG = with_common_config({
     # Whether to fake GPUs (using CPUs).
     # Set this to True for debugging on non-GPU machines (set `num_gpus` > 0).
     "_fake_gpus": False,
+
+    # Deprecated keys:
+    # Share layers for value function. If you set this to True, it's important
+    # to tune vf_loss_coeff.
+    # Use config.model.vf_share_layers instead.
+    "vf_share_layers": DEPRECATED_VALUE,
 })
 
 # __sphinx_doc_end__
@@ -199,7 +208,8 @@ def warn_about_bad_reward_scales(config, result):
         scaled_vf_loss = (config["vf_loss_coeff"] *
                           learner_stats[DEFAULT_POLICY_ID]["vf_loss"])
         policy_loss = learner_stats[DEFAULT_POLICY_ID]["policy_loss"]
-        if config["vf_share_layers"] and scaled_vf_loss > 100:
+        if config.get("model", {}).get("vf_share_layers") and \
+                scaled_vf_loss > 100:
             logger.warning(
                 "The magnitude of your value function loss is extremely large "
                 "({}) compared to the policy loss ({}). This can prevent the "

rllib/agents/ppo/ppo_tf_policy.py

@@ -17,6 +17,7 @@ from ray.rllib.policy.sample_batch import SampleBatch
 from ray.rllib.policy.tf_policy import LearningRateSchedule, \
     EntropyCoeffSchedule
 from ray.rllib.policy.tf_policy_template import build_tf_policy
+from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning
 from ray.rllib.utils.framework import try_import_tf, get_variable
 from ray.rllib.utils.tf_ops import explained_variance, make_tf_callable
 from ray.rllib.utils.typing import AgentID, LocalOptimizer, ModelGradients, \
@@ -354,8 +355,23 @@ def setup_config(policy: Policy, obs_space: gym.spaces.Space,
         action_space (gym.spaces.Space): The Policy's action space.
         config (TrainerConfigDict): The Policy's config.
     """
-    # Auto set the model option for VF layer sharing.
-    config["model"]["vf_share_layers"] = config["vf_share_layers"]
+    # Setting `vf_share_layers` in the top-level config is deprecated.
+    # It's confusing as some users might (correctly!) set it in their
+    # model config and then won't notice that it's silently overwritten
+    # here.
+    if config["vf_share_layers"] != DEPRECATED_VALUE:
+        deprecation_warning(
+            old="config[vf_share_layers]",
+            new="config[model][vf_share_layers]",
+            error=False,
+        )
+        config["model"]["vf_share_layers"] = config["vf_share_layers"]
+
+    # If vf_share_layers is True, inform about the need to tune vf_loss_coeff.
+    if config.get("model", {}).get("vf_share_layers") is True:
+        logger.info(
+            "`vf_share_layers=True` in your model. "
+            "Therefore, remember to tune the value of `vf_loss_coeff`!")
 
 
 def setup_mixins(policy: Policy, obs_space: gym.spaces.Space,

rllib/agents/ppo/tests/test_ppo.py

@@ -113,10 +113,10 @@ class TestPPO(unittest.TestCase):
         config["lr"] = 0.0003
         config["observation_filter"] = "MeanStdFilter"
         config["num_sgd_iter"] = 6
-        config["vf_share_layers"] = True
         config["vf_loss_coeff"] = 0.01
         config["model"]["fcnet_hiddens"] = [32]
         config["model"]["fcnet_activation"] = "linear"
+        config["model"]["vf_share_layers"] = True
 
         trainer = ppo.PPOTrainer(config=config, env="CartPole-v0")
         num_iterations = 200
@@ -181,7 +181,7 @@ class TestPPO(unittest.TestCase):
         config["model"]["fcnet_hiddens"] = [10]
         config["model"]["fcnet_activation"] = "linear"
         config["model"]["free_log_std"] = True
-        config["vf_share_layers"] = True
+        config["model"]["vf_share_layers"] = True
 
         for fw, sess in framework_iterator(config, session=True):
             trainer = ppo.PPOTrainer(config=config, env="CartPole-v0")
@@ -232,7 +232,7 @@ class TestPPO(unittest.TestCase):
         config["gamma"] = 0.99
         config["model"]["fcnet_hiddens"] = [10]
         config["model"]["fcnet_activation"] = "linear"
-        config["vf_share_layers"] = True
+        config["model"]["vf_share_layers"] = True
 
         for fw, sess in framework_iterator(config, session=True):
             trainer = ppo.PPOTrainer(config=config, env="CartPole-v0")

rllib/examples/cartpole_lstm.py

@@ -27,7 +27,9 @@ if __name__ == "__main__":
     configs = {
         "PPO": {
             "num_sgd_iter": 5,
-            "vf_share_layers": True,
+            "model": {
+                "vf_share_layers": True,
+            },
             "vf_loss_coeff": 0.0001,
         },
         "IMPALA": {

rllib/examples/custom_env.py

@@ -118,8 +118,8 @@ if __name__ == "__main__":
         "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
         "model": {
             "custom_model": "my_model",
+            "vf_share_layers": True,
         },
-        "vf_share_layers": True,
         "lr": grid_search([1e-2, 1e-4, 1e-6]),  # try different lrs
         "num_workers": 1,  # parallelism
         "framework": "torch" if args.torch else "tf",

rllib/examples/mobilenet_v2_with_lstm.py

@@ -51,8 +51,8 @@ if __name__ == "__main__":
                 "cnn_shape": cnn_shape_torch if args.torch else cnn_shape,
             },
             "max_seq_len": 20,
+            "vf_share_layers": True,
         },
-        "vf_share_layers": True,
         # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
         "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
         "num_workers": 0,  # no parallelism

rllib/tests/test_lstm.py

@@ -120,12 +120,12 @@ class TestRNNSequencing(unittest.TestCase):
                 "rollout_fragment_length": 10,
                 "train_batch_size": 10,
                 "sgd_minibatch_size": 10,
-                "vf_share_layers": True,
                 "simple_optimizer": True,
                 "num_sgd_iter": 1,
                 "model": {
                     "custom_model": "rnn",
                     "max_seq_len": 4,
+                    "vf_share_layers": True,
                 },
                 "framework": "tf",
             })
@@ -178,12 +178,12 @@ class TestRNNSequencing(unittest.TestCase):
                 "rollout_fragment_length": 20,
                 "train_batch_size": 20,
                 "sgd_minibatch_size": 10,
-                "vf_share_layers": True,
                 "simple_optimizer": False,
                 "num_sgd_iter": 1,
                 "model": {
                     "custom_model": "rnn",
                     "max_seq_len": 4,
+                    "vf_share_layers": True,
                 },
                 "framework": "tf",
             })

rllib/tests/test_model_imports.py

@@ -185,8 +185,9 @@ class TestModelImport(unittest.TestCase):
             "PPO",
             config={
                 "num_workers": 0,
-                "vf_share_layers": True,
-                "model": {}
+                "model": {
+                    "vf_share_layers": True,
+                },
             },
             env="CartPole-v0")
 

rllib/tuned_examples/compact-regression-test.yaml

@@ -43,7 +43,8 @@ atari-ppo-tf:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true
         num_gpus: 1
 atari-ppo-torch:
     env: BreakoutNoFrameskip-v4
@@ -67,7 +68,8 @@ atari-ppo-torch:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true
         num_gpus: 1
 apex:
     env: BreakoutNoFrameskip-v4

rllib/tuned_examples/ppo/atari-ddppo.yaml

@@ -29,4 +29,5 @@ atari-ddppo:
         entropy_coeff: 0.01
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true

rllib/tuned_examples/ppo/atari-ppo.yaml

@@ -25,5 +25,6 @@ atari-ppo:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
+        model:
+            vf_share_layers: true
         num_gpus: 1

rllib/tuned_examples/ppo/cartpole-ppo.yaml

@@ -12,8 +12,8 @@ cartpole-ppo:
         num_workers: 1
         observation_filter: MeanStdFilter
         num_sgd_iter: 6
-        vf_share_layers: true
         vf_loss_coeff: 0.01
         model:
-          fcnet_hiddens: [32]
-          fcnet_activation: linear
+            fcnet_hiddens: [32]
+            fcnet_activation: linear
+            vf_share_layers: true

rllib/tuned_examples/ppo/pong-ppo.yaml

@@ -22,7 +22,7 @@ pong-ppo:
         num_envs_per_worker: 5
         batch_mode: truncate_episodes
         observation_filter: NoFilter
-        vf_share_layers: true
         num_gpus: 1
         model:
             dim: 42
+            vf_share_layers: true

rllib/tuned_examples/ppo/repeatafterme-ppo-lstm.yaml

@@ -15,10 +15,10 @@ repeat-after-me-ppo-w-lstm:
         num_workers: 0
         num_envs_per_worker: 20
         num_sgd_iter: 5
-        vf_share_layers: true
         entropy_coeff: 0.00001
         model:
             use_lstm: true
             lstm_cell_size: 64
             max_seq_len: 20
             fcnet_hiddens: [64]
+            vf_share_layers: true