From 8fc2272f435a7e357949e60946b7001ae2ed14af Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Mon, 18 Nov 2019 10:39:07 -0800 Subject: [PATCH] [rllib] Reorganize trainer config, add warnings about high VF loss magnitude for PPO (#6181) --- doc/source/rllib-offline.rst | 2 +- rllib/agents/ppo/ppo.py | 63 +++++++---- rllib/agents/trainer.py | 214 ++++++++++++++++++++--------------- 3 files changed, 162 insertions(+), 117 deletions(-) diff --git a/doc/source/rllib-offline.rst b/doc/source/rllib-offline.rst index fc6c63fe4..4ce609d6d 100644 --- a/doc/source/rllib-offline.rst +++ b/doc/source/rllib-offline.rst @@ -173,7 +173,7 @@ You can configure experience output for an agent using the following options: .. literalinclude:: ../../rllib/agents/trainer.py :language: python :start-after: shuffle_buffer_size - :end-before: === Multiagent === + :end-before: Settings for Multi-Agent Environments The interface for a custom output writer is as follows: diff --git a/rllib/agents/ppo/ppo.py b/rllib/agents/ppo/ppo.py index 828402603..ee3a2eb20 100644 --- a/rllib/agents/ppo/ppo.py +++ b/rllib/agents/ppo/ppo.py @@ -19,49 +19,53 @@ DEFAULT_CONFIG = with_common_config({ # If true, use the Generalized Advantage Estimator (GAE) # with a value function, see https://arxiv.org/pdf/1506.02438.pdf. "use_gae": True, - # GAE(lambda) parameter + # The GAE(lambda) parameter. "lambda": 1.0, - # Initial coefficient for KL divergence + # Initial coefficient for KL divergence. "kl_coeff": 0.2, - # Size of batches collected from each worker + # Size of batches collected from each worker. "sample_batch_size": 200, - # Number of timesteps collected for each SGD round + # Number of timesteps collected for each SGD round. This defines the size + # of each SGD epoch. "train_batch_size": 4000, - # Total SGD batch size across all devices for SGD + # Total SGD batch size across all devices for SGD. This defines the + # minibatch size within each epoch. "sgd_minibatch_size": 128, - # Whether to shuffle sequences in the batch when training (recommended) + # Whether to shuffle sequences in the batch when training (recommended). "shuffle_sequences": True, - # Number of SGD iterations in each outer loop + # Number of SGD iterations in each outer loop (i.e., number of epochs to + # execute per train batch). "num_sgd_iter": 30, - # Stepsize of SGD + # Stepsize of SGD. "lr": 5e-5, - # Learning rate schedule + # Learning rate schedule. "lr_schedule": None, # Share layers for value function. If you set this to True, it's important # to tune vf_loss_coeff. "vf_share_layers": False, - # Coefficient of the value function loss. It's important to tune this if - # you set vf_share_layers: True + # Coefficient of the value function loss. IMPORTANT: you must tune this if + # you set vf_share_layers: True. "vf_loss_coeff": 1.0, - # Coefficient of the entropy regularizer + # Coefficient of the entropy regularizer. "entropy_coeff": 0.0, - # Decay schedule for the entropy regularizer + # Decay schedule for the entropy regularizer. "entropy_coeff_schedule": None, - # PPO clip parameter + # PPO clip parameter. "clip_param": 0.3, # Clip param for the value function. Note that this is sensitive to the # scale of the rewards. If your expected V is large, increase this. "vf_clip_param": 10.0, - # If specified, clip the global norm of gradients by this amount + # If specified, clip the global norm of gradients by this amount. "grad_clip": None, - # Target value for KL divergence + # Target value for KL divergence. "kl_target": 0.01, - # Whether to rollout "complete_episodes" or "truncate_episodes" + # Whether to rollout "complete_episodes" or "truncate_episodes". "batch_mode": "truncate_episodes", - # Which observation filter to apply to the observation + # Which observation filter to apply to the observation. "observation_filter": "NoFilter", - # Uses the sync samples optimizer instead of the multi-gpu one. This does - # not support minibatches. + # Uses the sync samples optimizer instead of the multi-gpu one. This is + # usually slower, but you might want to try it if you run into issues with + # the default optimizer. "simple_optimizer": False, }) # __sphinx_doc_end__ @@ -107,11 +111,26 @@ def update_kl(trainer, fetches): def warn_about_bad_reward_scales(trainer, result): + if result["policy_reward_mean"]: + return # Punt on handling multiagent case. + + # Warn about excessively high VF loss. + learner_stats = result["info"]["learner"] + if "default_policy" in learner_stats: + scaled_vf_loss = (trainer.config["vf_loss_coeff"] * + learner_stats["default_policy"]["vf_loss"]) + policy_loss = learner_stats["default_policy"]["policy_loss"] + if trainer.config["vf_share_layers"] and scaled_vf_loss > 100: + logger.warning( + "The magnitude of your value function loss is extremely large " + "({}) compared to the policy loss ({}). This can prevent the " + "policy from learning. Consider scaling down the VF loss by " + "reducing vf_loss_coeff, or disabling vf_share_layers.".format( + scaled_vf_loss, policy_loss)) + # Warn about bad clipping configs if trainer.config["vf_clip_param"] <= 0: rew_scale = float("inf") - elif result["policy_reward_mean"]: - rew_scale = 0 # punt on handling multiagent case else: rew_scale = round( abs(result["episode_reward_mean"]) / diff --git a/rllib/agents/trainer.py b/rllib/agents/trainer.py index a22e8e36e..e6c231135 100644 --- a/rllib/agents/trainer.py +++ b/rllib/agents/trainer.py @@ -40,13 +40,87 @@ MAX_WORKER_FAILURE_RETRIES = 3 # yapf: disable # __sphinx_doc_begin__ COMMON_CONFIG = { - # === Debugging === - # Whether to write episode stats and videos to the agent log dir + # === Settings for Rollout Worker processes === + # Number of rollout worker actors to create for parallel sampling. Setting + # this to 0 will force rollouts to be done in the trainer actor. + "num_workers": 2, + # Number of environments to evaluate vectorwise per worker. This enables + # model inference batching, which can improve performance for inference + # bottlenecked workloads. + "num_envs_per_worker": 1, + # Default sample batch size (unroll length). Batches of this size are + # collected from rollout workers until train_batch_size is met. When using + # multiple envs per worker, this is multiplied by num_envs_per_worker. + # + # For example, given sample_batch_size=100 and train_batch_size=1000: + # 1. RLlib will collect 10 batches of size 100 from the rollout workers. + # 2. These batches are concatenated and we perform an epoch of SGD. + # + # If we further set num_envs_per_worker=5, then the sample batches will be + # of size 5*100 = 500, and RLlib will only collect 2 batches per epoch. + # + # The exact workflow here can vary per algorithm. For example, PPO further + # divides the train batch into minibatches for multi-epoch SGD. + "sample_batch_size": 200, + # Whether to rollout "complete_episodes" or "truncate_episodes" to + # `sample_batch_size` length unrolls. Episode truncation guarantees more + # evenly sized batches, but increases variance as the reward-to-go will + # need to be estimated at truncation boundaries. + "batch_mode": "truncate_episodes", + + # === Settings for the Trainer process === + # Number of GPUs to allocate to the trainer process. Note that not all + # algorithms can take advantage of trainer GPUs. This can be fractional + # (e.g., 0.3 GPUs). + "num_gpus": 0, + # Training batch size, if applicable. Should be >= sample_batch_size. + # Samples batches will be concatenated together to a batch of this size, + # which is then passed to SGD. + "train_batch_size": 200, + # Arguments to pass to the policy model. See models/catalog.py for a full + # list of the available model options. + "model": MODEL_DEFAULTS, + # Arguments to pass to the policy optimizer. These vary by optimizer. + "optimizer": {}, + + # === Environment Settings === + # Discount factor of the MDP. + "gamma": 0.99, + # Number of steps after which the episode is forced to terminate. Defaults + # to `env.spec.max_episode_steps` (if present) for Gym envs. + "horizon": None, + # Calculate rewards but don't reset the environment when the horizon is + # hit. This allows value estimation and RNN state to span across logical + # episodes denoted by horizon. This only has an effect if horizon != inf. + "soft_horizon": False, + # Don't set 'done' at the end of the episode. Note that you still need to + # set this if soft_horizon=True, unless your env is actually running + # forever without returning done=True. + "no_done_at_end": False, + # Arguments to pass to the env creator. + "env_config": {}, + # Environment name can also be passed via config. + "env": None, + # Whether to clip rewards prior to experience postprocessing. Setting to + # None means clip for Atari only. + "clip_rewards": None, + # Whether to np.clip() actions to the action space low/high range spec. + "clip_actions": True, + # Whether to use rllib or deepmind preprocessors by default + "preprocessor_pref": "deepmind", + # The default learning rate. + "lr": 0.0001, + + # === Debug Settings === + # Whether to write episode stats and videos to the agent log dir. This is + # typically located in ~/ray_results. "monitor": False, # Set the ray.rllib.* log level for the agent process and its workers. # Should be one of DEBUG, INFO, WARN, or ERROR. The DEBUG level will also # periodically print out summaries of relevant internal dataflow (this is - # also printed out once at startup at the INFO level). + # also printed out once at startup at the INFO level). When using the + # `rllib train` command, you can also use the `-v` and `-vv` flags as + # shorthand for INFO and DEBUG. "log_level": "WARN", # Callbacks that will be run during various phases of training. These all # take a single "info" dict as an argument. For episode callbacks, custom @@ -66,11 +140,15 @@ COMMON_CONFIG = { # "all_pre_batches": (other agent ids), # } }, - # Whether to attempt to continue training if a worker crashes. + # Whether to attempt to continue training if a worker crashes. The number + # of currently healthy workers is reported as the "num_healthy_workers" + # metric. "ignore_worker_failures": False, - # Log system resource metrics to results. + # Log system resource metrics to results. This requires `psutil` to be + # installed for sys stats, and `gputil` for GPU metrics. "log_sys_usage": True, - # Enable TF eager execution (TF policies only). + # Enable TF eager execution (TF policies only). If using `rllib train`, + # this can also be enabled with the `--eager` flag. "eager": False, # Enable tracing in eager mode. This greatly improves performance, but # makes it slightly harder to debug since Python code won't be evaluated @@ -80,42 +158,7 @@ COMMON_CONFIG = { # only has an effect is eager is enabled. "no_eager_on_workers": False, - # === Policy === - # Arguments to pass to model. See models/catalog.py for a full list of the - # available model options. - "model": MODEL_DEFAULTS, - # Arguments to pass to the policy optimizer. These vary by optimizer. - "optimizer": {}, - - # === Environment === - # Discount factor of the MDP - "gamma": 0.99, - # Number of steps after which the episode is forced to terminate. Defaults - # to `env.spec.max_episode_steps` (if present) for Gym envs. - "horizon": None, - # Calculate rewards but don't reset the environment when the horizon is - # hit. This allows value estimation and RNN state to span across logical - # episodes denoted by horizon. This only has an effect if horizon != inf. - "soft_horizon": False, - # Don't set 'done' at the end of the episode. Note that you still need to - # set this if soft_horizon=True, unless your env is actually running - # forever without returning done=True. - "no_done_at_end": False, - # Arguments to pass to the env creator - "env_config": {}, - # Environment name can also be passed via config - "env": None, - # Whether to clip rewards prior to experience postprocessing. Setting to - # None means clip for Atari only. - "clip_rewards": None, - # Whether to np.clip() actions to the action space low/high range spec. - "clip_actions": True, - # Whether to use rllib or deepmind preprocessors by default - "preprocessor_pref": "deepmind", - # The default learning rate - "lr": 0.0001, - - # === Evaluation === + # === Evaluation Settings === # Evaluate with every `evaluation_interval` training iterations. # The evaluation stats will be reported under the "evaluation" metric key. # Note that evaluation is currently not parallelized, and that for Ape-X @@ -129,62 +172,15 @@ COMMON_CONFIG = { # TODO(kismuz): implement determ. actions and include relevant keys hints "evaluation_config": {}, - # === Resources === - # Number of actors used for parallelism - "num_workers": 2, - # Number of GPUs to allocate to the trainer process. Note that not all - # algorithms can take advantage of trainer GPUs. This can be fractional - # (e.g., 0.3 GPUs). - "num_gpus": 0, - # Number of CPUs to allocate per worker. - "num_cpus_per_worker": 1, - # Number of GPUs to allocate per worker. This can be fractional. - "num_gpus_per_worker": 0, - # Any custom resources to allocate per worker. - "custom_resources_per_worker": {}, - # Number of CPUs to allocate for the trainer. Note: this only takes effect - # when running in Tune. - "num_cpus_for_driver": 1, - - # === Memory quota === - # You can set these memory quotas to tell Ray to reserve memory for your - # training run. This guarantees predictable execution, but the tradeoff is - # if your workload exceeeds the memory quota it will fail. - # Heap memory to reserve for the trainer process (0 for unlimited). This - # can be large if your are using large train batches, replay buffers, etc. - "memory": 0, - # Object store memory to reserve for the trainer process. Being large - # enough to fit a few copies of the model weights should be sufficient. - # This is enabled by default since models are typically quite small. - "object_store_memory": 0, - # Heap memory to reserve for each worker. Should generally be small unless - # your environment is very heavyweight. - "memory_per_worker": 0, - # Object store memory to reserve for each worker. This only needs to be - # large enough to fit a few sample batches at a time. This is enabled - # by default since it almost never needs to be larger than ~200MB. - "object_store_memory_per_worker": 0, - - # === Execution === - # Number of environments to evaluate vectorwise per worker. - "num_envs_per_worker": 1, - # Default sample batch size (unroll length). Batches of this size are - # collected from workers until train_batch_size is met. When using - # multiple envs per worker, this is multiplied by num_envs_per_worker. - "sample_batch_size": 200, - # Training batch size, if applicable. Should be >= sample_batch_size. - # Samples batches will be concatenated together to this size for training. - "train_batch_size": 200, - # Whether to rollout "complete_episodes" or "truncate_episodes" - "batch_mode": "truncate_episodes", + # === Advanced Rollout Settings === # Use a background thread for sampling (slightly off-policy, usually not - # advisable to turn on unless your env specifically requires it) + # advisable to turn on unless your env specifically requires it). "sample_async": False, - # Element-wise observation filter, either "NoFilter" or "MeanStdFilter" + # Element-wise observation filter, either "NoFilter" or "MeanStdFilter". "observation_filter": "NoFilter", # Whether to synchronize the statistics of remote filters. "synchronize_filters": True, - # Configure TF for single-process operation by default + # Configures TF for single-process operation by default. "tf_session_args": { # note: overriden by `local_tf_session_args` "intra_op_parallelism_threads": 2, @@ -232,6 +228,36 @@ COMMON_CONFIG = { # results. This makes experiments reproducible. "seed": None, + # === Advanced Resource Settings === + # Number of CPUs to allocate per worker. + "num_cpus_per_worker": 1, + # Number of GPUs to allocate per worker. This can be fractional. This is + # usually needed only if your env itself requires a GPU (i.e., it is a + # GPU-intensive video game), or model inference is unusually expensive. + "num_gpus_per_worker": 0, + # Any custom Ray resources to allocate per worker. + "custom_resources_per_worker": {}, + # Number of CPUs to allocate for the trainer. Note: this only takes effect + # when running in Tune. Otherwise, the trainer runs in the main program. + "num_cpus_for_driver": 1, + # You can set these memory quotas to tell Ray to reserve memory for your + # training run. This guarantees predictable execution, but the tradeoff is + # if your workload exceeeds the memory quota it will fail. + # Heap memory to reserve for the trainer process (0 for unlimited). This + # can be large if your are using large train batches, replay buffers, etc. + "memory": 0, + # Object store memory to reserve for the trainer process. Being large + # enough to fit a few copies of the model weights should be sufficient. + # This is enabled by default since models are typically quite small. + "object_store_memory": 0, + # Heap memory to reserve for each worker. Should generally be small unless + # your environment is very heavyweight. + "memory_per_worker": 0, + # Object store memory to reserve for each worker. This only needs to be + # large enough to fit a few sample batches at a time. This is enabled + # by default since it almost never needs to be larger than ~200MB. + "object_store_memory_per_worker": 0, + # === Offline Datasets === # Specify how to generate experiences: # - "sampler": generate experiences via online simulation (default) @@ -269,7 +295,7 @@ COMMON_CONFIG = { # Max output file size before rolling over to a new file. "output_max_file_size": 64 * 1024 * 1024, - # === Multiagent === + # === Settings for Multi-Agent Environments === "multiagent": { # Map from policy ids to tuples of (policy_cls, obs_space, # act_space, config). See rollout_worker.py for more info.