[rllib] Set PPO observation filter to NoFilter by default (#4191)

python/ray/rllib/agents/agent.py

@@ -332,6 +332,7 @@ class Agent(Trainable):
         merged_config = deep_update(merged_config, config,
                                     self._allow_unknown_configs,
                                     self._allow_unknown_subkeys)
+        self.raw_user_config = config
         self.config = merged_config
         Agent._validate_config(self.config)
         if self.config.get("log_level"):

python/ray/rllib/agents/ppo/ppo.py

@@ -51,7 +51,7 @@ DEFAULT_CONFIG = with_common_config({
     # Whether to rollout "complete_episodes" or "truncate_episodes"
     "batch_mode": "truncate_episodes",
     # Which observation filter to apply to the observation
-    "observation_filter": "MeanStdFilter",
+    "observation_filter": "NoFilter",
     # Uses the sync samples optimizer instead of the multi-gpu one. This does
     # not support minibatches.
     "simple_optimizer": False,
@@ -99,6 +99,14 @@ class PPOAgent(Agent):
 
     @override(Agent)
     def _train(self):
+        if "observation_filter" not in self.raw_user_config:
+            # TODO(ekl) remove this message after a few releases
+            logger.info(
+                "Important! Since 0.7.0, observation normalization is no "
+                "longer enabled by default. To enable running-mean "
+                "normalization, set 'observation_filter': 'MeanStdFilter'. "
+                "You can ignore this message if your environment doesn't "
+                "require observation normalization.")
         prev_steps = self.optimizer.num_steps_sampled
         fetches = self.optimizer.step()
         if "kl" in fetches:
@@ -139,7 +147,6 @@ class PPOAgent(Agent):
                 "{} iterations for your value ".format(rew_scale) +
                 "function to converge. If this is not intended, consider "
                 "increasing `vf_clip_param`.")
-
         return res
 
     def _validate_config(self):
@@ -159,13 +166,7 @@ class PPOAgent(Agent):
                 "In multi-agent mode, policies will be optimized sequentially "
                 "by the multi-GPU optimizer. Consider setting "
                 "simple_optimizer=True if this doesn't work for you.")
-        if self.config["observation_filter"] != "NoFilter":
-            logger.warning(
-                "By default, observations will be normalized with {}. ".format(
-                    self.config["observation_filter"]) +
-                "If you are using image or discrete type observations, "
-                "consider disabling this with observation_filter=NoFilter.")
         if not self.config["vf_share_layers"]:
             logger.warning(
-                "By default, the value function will NOT share layers with "
-                "the policy model (vf_share_layers=False).")
+                "FYI: By default, the value function will not share layers "
+                "with the policy model ('vf_share_layers': False).")