diff --git a/doc/source/rllib-models.rst b/doc/source/rllib-models.rst index 347f3cc34..59678af7e 100644 --- a/doc/source/rllib-models.rst +++ b/doc/source/rllib-models.rst @@ -604,7 +604,7 @@ Custom models can be used to work with environments where (1) the set of valid a return action_logits + inf_mask, state -Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `algorithm overview `__. +Depending on your use case it may make sense to use just the masking, just action embeddings, or both. For a runnable example of this in code, check out `parametric_actions_cartpole.py `__. Note that since masking introduces ``tf.float32.min`` values into the model output, this technique might not work with all algorithm options. For example, algorithms might crash if they incorrectly process the ``tf.float32.min`` values. The cartpole example has working configurations for DQN (must set ``hiddens=[]``), PPO (must disable running mean and set ``model.vf_share_layers=True``), and several other algorithms. Not all algorithms support parametric actions; see the `algorithm overview `__. Autoregressive Action Distributions diff --git a/release/rllib_tests/regression_tests/compact-regression-tests-tf.yaml b/release/rllib_tests/regression_tests/compact-regression-tests-tf.yaml index 4189d3886..e78a81f5f 100644 --- a/release/rllib_tests/regression_tests/compact-regression-tests-tf.yaml +++ b/release/rllib_tests/regression_tests/compact-regression-tests-tf.yaml @@ -129,7 +129,8 @@ ppo-tf-atari: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true num_gpus: 1 # Expect roughly 1000 reward after 1h on 1GPU diff --git a/release/rllib_tests/regression_tests/compact-regression-tests-torch.yaml b/release/rllib_tests/regression_tests/compact-regression-tests-torch.yaml index 2b259aeff..a283c78bc 100644 --- a/release/rllib_tests/regression_tests/compact-regression-tests-torch.yaml +++ b/release/rllib_tests/regression_tests/compact-regression-tests-torch.yaml @@ -127,7 +127,8 @@ ppo-torch-atari: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true num_gpus: 1 # Expect roughly 1000 reward after 1h on 1GPU diff --git a/rllib/agents/maml/maml.py b/rllib/agents/maml/maml.py index 8841d22b6..52477f43b 100644 --- a/rllib/agents/maml/maml.py +++ b/rllib/agents/maml/maml.py @@ -1,6 +1,6 @@ import logging - import numpy as np + from ray.rllib.utils.sgd import standardized from ray.rllib.agents import with_common_config from ray.rllib.agents.maml.maml_tf_policy import MAMLTFPolicy @@ -11,8 +11,9 @@ from ray.rllib.execution.common import STEPS_SAMPLED_COUNTER, \ STEPS_TRAINED_COUNTER, LEARNER_INFO, _get_shared_metrics from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.execution.metric_ops import CollectMetrics -from ray.util.iter import from_actors from ray.rllib.evaluation.metrics import collect_metrics +from ray.rllib.utils.deprecation import DEPRECATED_VALUE +from ray.util.iter import from_actors logger = logging.getLogger(__name__) @@ -32,8 +33,10 @@ DEFAULT_CONFIG = with_common_config({ "create_env_on_driver": True, # Stepsize of SGD "lr": 1e-3, - # Share layers for value function - "vf_share_layers": False, + "model": { + # Share layers for value function. + "vf_share_layers": False, + }, # Coefficient of the value function loss "vf_loss_coeff": 0.5, # Coefficient of the entropy regularizer @@ -59,6 +62,12 @@ DEFAULT_CONFIG = with_common_config({ "inner_lr": 0.1, # Use Meta Env Template "use_meta_env": True, + + # Deprecated keys: + # Share layers for value function. If you set this to True, it's important + # to tune vf_loss_coeff. + # Use config.model.vf_share_layers instead. + "vf_share_layers": DEPRECATED_VALUE, }) # __sphinx_doc_end__ # yapf: enable diff --git a/rllib/agents/mbmpo/mbmpo.py b/rllib/agents/mbmpo/mbmpo.py index c4dabf756..9cbf94c67 100644 --- a/rllib/agents/mbmpo/mbmpo.py +++ b/rllib/agents/mbmpo/mbmpo.py @@ -29,6 +29,7 @@ from ray.rllib.execution.common import STEPS_SAMPLED_COUNTER, \ STEPS_TRAINED_COUNTER, LEARNER_INFO, _get_shared_metrics from ray.rllib.execution.metric_ops import CollectMetrics from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch +from ray.rllib.utils.deprecation import DEPRECATED_VALUE from ray.rllib.utils.sgd import standardized from ray.rllib.utils.torch_ops import convert_to_torch_tensor from ray.rllib.utils.typing import EnvType, TrainerConfigDict @@ -55,8 +56,6 @@ DEFAULT_CONFIG = with_common_config({ "create_env_on_driver": True, # Step size of SGD. "lr": 1e-3, - # Share layers for value function. - "vf_share_layers": False, # Coefficient of the value function loss. "vf_loss_coeff": 0.5, # Coefficient of the entropy regularizer. @@ -110,6 +109,12 @@ DEFAULT_CONFIG = with_common_config({ "custom_vector_env": model_vector_env, # How many iterations through MAML per MBMPO iteration. "num_maml_steps": 10, + + # Deprecated keys: + # Share layers for value function. If you set this to True, it's important + # to tune vf_loss_coeff. + # Use config.model.vf_share_layers instead. + "vf_share_layers": DEPRECATED_VALUE, }) # __sphinx_doc_end__ # yapf: enable diff --git a/rllib/agents/ppo/ppo.py b/rllib/agents/ppo/ppo.py index 31e8e36e9..9270e6cf5 100644 --- a/rllib/agents/ppo/ppo.py +++ b/rllib/agents/ppo/ppo.py @@ -22,6 +22,7 @@ from ray.rllib.execution.train_ops import TrainOneStep, TrainTFMultiGPU from ray.rllib.execution.metric_ops import StandardMetricsReporting from ray.rllib.policy.policy import Policy from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID +from ray.rllib.utils.deprecation import DEPRECATED_VALUE from ray.rllib.utils.typing import TrainerConfigDict from ray.util.iter import LocalIterator @@ -60,12 +61,14 @@ DEFAULT_CONFIG = with_common_config({ "lr": 5e-5, # Learning rate schedule. "lr_schedule": None, - # Share layers for value function. If you set this to True, it's important - # to tune vf_loss_coeff. - "vf_share_layers": False, # Coefficient of the value function loss. IMPORTANT: you must tune this if - # you set vf_share_layers: True. + # you set vf_share_layers=True inside your model's config. "vf_loss_coeff": 1.0, + "model": { + # Share layers for value function. If you set this to True, it's + # important to tune vf_loss_coeff. + "vf_share_layers": False, + }, # Coefficient of the entropy regularizer. "entropy_coeff": 0.0, # Decay schedule for the entropy regularizer. @@ -90,6 +93,12 @@ DEFAULT_CONFIG = with_common_config({ # Whether to fake GPUs (using CPUs). # Set this to True for debugging on non-GPU machines (set `num_gpus` > 0). "_fake_gpus": False, + + # Deprecated keys: + # Share layers for value function. If you set this to True, it's important + # to tune vf_loss_coeff. + # Use config.model.vf_share_layers instead. + "vf_share_layers": DEPRECATED_VALUE, }) # __sphinx_doc_end__ @@ -199,7 +208,8 @@ def warn_about_bad_reward_scales(config, result): scaled_vf_loss = (config["vf_loss_coeff"] * learner_stats[DEFAULT_POLICY_ID]["vf_loss"]) policy_loss = learner_stats[DEFAULT_POLICY_ID]["policy_loss"] - if config["vf_share_layers"] and scaled_vf_loss > 100: + if config.get("model", {}).get("vf_share_layers") and \ + scaled_vf_loss > 100: logger.warning( "The magnitude of your value function loss is extremely large " "({}) compared to the policy loss ({}). This can prevent the " diff --git a/rllib/agents/ppo/ppo_tf_policy.py b/rllib/agents/ppo/ppo_tf_policy.py index 957d68ce3..1a8f0be71 100644 --- a/rllib/agents/ppo/ppo_tf_policy.py +++ b/rllib/agents/ppo/ppo_tf_policy.py @@ -17,6 +17,7 @@ from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.policy.tf_policy import LearningRateSchedule, \ EntropyCoeffSchedule from ray.rllib.policy.tf_policy_template import build_tf_policy +from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning from ray.rllib.utils.framework import try_import_tf, get_variable from ray.rllib.utils.tf_ops import explained_variance, make_tf_callable from ray.rllib.utils.typing import AgentID, LocalOptimizer, ModelGradients, \ @@ -354,8 +355,23 @@ def setup_config(policy: Policy, obs_space: gym.spaces.Space, action_space (gym.spaces.Space): The Policy's action space. config (TrainerConfigDict): The Policy's config. """ - # Auto set the model option for VF layer sharing. - config["model"]["vf_share_layers"] = config["vf_share_layers"] + # Setting `vf_share_layers` in the top-level config is deprecated. + # It's confusing as some users might (correctly!) set it in their + # model config and then won't notice that it's silently overwritten + # here. + if config["vf_share_layers"] != DEPRECATED_VALUE: + deprecation_warning( + old="config[vf_share_layers]", + new="config[model][vf_share_layers]", + error=False, + ) + config["model"]["vf_share_layers"] = config["vf_share_layers"] + + # If vf_share_layers is True, inform about the need to tune vf_loss_coeff. + if config.get("model", {}).get("vf_share_layers") is True: + logger.info( + "`vf_share_layers=True` in your model. " + "Therefore, remember to tune the value of `vf_loss_coeff`!") def setup_mixins(policy: Policy, obs_space: gym.spaces.Space, diff --git a/rllib/agents/ppo/tests/test_ppo.py b/rllib/agents/ppo/tests/test_ppo.py index b4259c144..cb304c57e 100644 --- a/rllib/agents/ppo/tests/test_ppo.py +++ b/rllib/agents/ppo/tests/test_ppo.py @@ -113,10 +113,10 @@ class TestPPO(unittest.TestCase): config["lr"] = 0.0003 config["observation_filter"] = "MeanStdFilter" config["num_sgd_iter"] = 6 - config["vf_share_layers"] = True config["vf_loss_coeff"] = 0.01 config["model"]["fcnet_hiddens"] = [32] config["model"]["fcnet_activation"] = "linear" + config["model"]["vf_share_layers"] = True trainer = ppo.PPOTrainer(config=config, env="CartPole-v0") num_iterations = 200 @@ -181,7 +181,7 @@ class TestPPO(unittest.TestCase): config["model"]["fcnet_hiddens"] = [10] config["model"]["fcnet_activation"] = "linear" config["model"]["free_log_std"] = True - config["vf_share_layers"] = True + config["model"]["vf_share_layers"] = True for fw, sess in framework_iterator(config, session=True): trainer = ppo.PPOTrainer(config=config, env="CartPole-v0") @@ -232,7 +232,7 @@ class TestPPO(unittest.TestCase): config["gamma"] = 0.99 config["model"]["fcnet_hiddens"] = [10] config["model"]["fcnet_activation"] = "linear" - config["vf_share_layers"] = True + config["model"]["vf_share_layers"] = True for fw, sess in framework_iterator(config, session=True): trainer = ppo.PPOTrainer(config=config, env="CartPole-v0") diff --git a/rllib/examples/cartpole_lstm.py b/rllib/examples/cartpole_lstm.py index de53c6ff1..35ea3689c 100644 --- a/rllib/examples/cartpole_lstm.py +++ b/rllib/examples/cartpole_lstm.py @@ -27,7 +27,9 @@ if __name__ == "__main__": configs = { "PPO": { "num_sgd_iter": 5, - "vf_share_layers": True, + "model": { + "vf_share_layers": True, + }, "vf_loss_coeff": 0.0001, }, "IMPALA": { diff --git a/rllib/examples/custom_env.py b/rllib/examples/custom_env.py index b9b060e43..6d311effe 100644 --- a/rllib/examples/custom_env.py +++ b/rllib/examples/custom_env.py @@ -118,8 +118,8 @@ if __name__ == "__main__": "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")), "model": { "custom_model": "my_model", + "vf_share_layers": True, }, - "vf_share_layers": True, "lr": grid_search([1e-2, 1e-4, 1e-6]), # try different lrs "num_workers": 1, # parallelism "framework": "torch" if args.torch else "tf", diff --git a/rllib/examples/mobilenet_v2_with_lstm.py b/rllib/examples/mobilenet_v2_with_lstm.py index cae074e0a..9da31b40a 100644 --- a/rllib/examples/mobilenet_v2_with_lstm.py +++ b/rllib/examples/mobilenet_v2_with_lstm.py @@ -51,8 +51,8 @@ if __name__ == "__main__": "cnn_shape": cnn_shape_torch if args.torch else cnn_shape, }, "max_seq_len": 20, + "vf_share_layers": True, }, - "vf_share_layers": True, # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0. "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")), "num_workers": 0, # no parallelism diff --git a/rllib/tests/test_lstm.py b/rllib/tests/test_lstm.py index 09b7aef73..cd13157d1 100644 --- a/rllib/tests/test_lstm.py +++ b/rllib/tests/test_lstm.py @@ -120,12 +120,12 @@ class TestRNNSequencing(unittest.TestCase): "rollout_fragment_length": 10, "train_batch_size": 10, "sgd_minibatch_size": 10, - "vf_share_layers": True, "simple_optimizer": True, "num_sgd_iter": 1, "model": { "custom_model": "rnn", "max_seq_len": 4, + "vf_share_layers": True, }, "framework": "tf", }) @@ -178,12 +178,12 @@ class TestRNNSequencing(unittest.TestCase): "rollout_fragment_length": 20, "train_batch_size": 20, "sgd_minibatch_size": 10, - "vf_share_layers": True, "simple_optimizer": False, "num_sgd_iter": 1, "model": { "custom_model": "rnn", "max_seq_len": 4, + "vf_share_layers": True, }, "framework": "tf", }) diff --git a/rllib/tests/test_model_imports.py b/rllib/tests/test_model_imports.py index 405b96b90..2a03b3789 100644 --- a/rllib/tests/test_model_imports.py +++ b/rllib/tests/test_model_imports.py @@ -185,8 +185,9 @@ class TestModelImport(unittest.TestCase): "PPO", config={ "num_workers": 0, - "vf_share_layers": True, - "model": {} + "model": { + "vf_share_layers": True, + }, }, env="CartPole-v0") diff --git a/rllib/tuned_examples/compact-regression-test.yaml b/rllib/tuned_examples/compact-regression-test.yaml index 7c5d633b8..765548225 100644 --- a/rllib/tuned_examples/compact-regression-test.yaml +++ b/rllib/tuned_examples/compact-regression-test.yaml @@ -43,7 +43,8 @@ atari-ppo-tf: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true num_gpus: 1 atari-ppo-torch: env: BreakoutNoFrameskip-v4 @@ -67,7 +68,8 @@ atari-ppo-torch: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true num_gpus: 1 apex: env: BreakoutNoFrameskip-v4 diff --git a/rllib/tuned_examples/ppo/atari-ddppo.yaml b/rllib/tuned_examples/ppo/atari-ddppo.yaml index 7bbaec98b..063d084c7 100644 --- a/rllib/tuned_examples/ppo/atari-ddppo.yaml +++ b/rllib/tuned_examples/ppo/atari-ddppo.yaml @@ -29,4 +29,5 @@ atari-ddppo: entropy_coeff: 0.01 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true diff --git a/rllib/tuned_examples/ppo/atari-ppo.yaml b/rllib/tuned_examples/ppo/atari-ppo.yaml index 8a08b9c65..1c92f4934 100644 --- a/rllib/tuned_examples/ppo/atari-ppo.yaml +++ b/rllib/tuned_examples/ppo/atari-ppo.yaml @@ -25,5 +25,6 @@ atari-ppo: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true + model: + vf_share_layers: true num_gpus: 1 diff --git a/rllib/tuned_examples/ppo/cartpole-ppo.yaml b/rllib/tuned_examples/ppo/cartpole-ppo.yaml index 5379ebe88..d86d11baa 100644 --- a/rllib/tuned_examples/ppo/cartpole-ppo.yaml +++ b/rllib/tuned_examples/ppo/cartpole-ppo.yaml @@ -12,8 +12,8 @@ cartpole-ppo: num_workers: 1 observation_filter: MeanStdFilter num_sgd_iter: 6 - vf_share_layers: true vf_loss_coeff: 0.01 model: - fcnet_hiddens: [32] - fcnet_activation: linear + fcnet_hiddens: [32] + fcnet_activation: linear + vf_share_layers: true diff --git a/rllib/tuned_examples/ppo/pong-ppo.yaml b/rllib/tuned_examples/ppo/pong-ppo.yaml index 3fd3e1d14..ddacd602c 100644 --- a/rllib/tuned_examples/ppo/pong-ppo.yaml +++ b/rllib/tuned_examples/ppo/pong-ppo.yaml @@ -22,7 +22,7 @@ pong-ppo: num_envs_per_worker: 5 batch_mode: truncate_episodes observation_filter: NoFilter - vf_share_layers: true num_gpus: 1 model: dim: 42 + vf_share_layers: true diff --git a/rllib/tuned_examples/ppo/repeatafterme-ppo-lstm.yaml b/rllib/tuned_examples/ppo/repeatafterme-ppo-lstm.yaml index 700e420d7..f783606fc 100644 --- a/rllib/tuned_examples/ppo/repeatafterme-ppo-lstm.yaml +++ b/rllib/tuned_examples/ppo/repeatafterme-ppo-lstm.yaml @@ -15,10 +15,10 @@ repeat-after-me-ppo-w-lstm: num_workers: 0 num_envs_per_worker: 20 num_sgd_iter: 5 - vf_share_layers: true entropy_coeff: 0.00001 model: use_lstm: true lstm_cell_size: 64 max_seq_len: 20 fcnet_hiddens: [64] + vf_share_layers: true