[rllib] Fix broken check in eval mode for IMPALA #7217

rllib/agents/impala/vtrace_policy.py

@@ -258,7 +258,7 @@ def add_behaviour_logits(policy):
 
 
 def validate_config(policy, obs_space, action_space, config):
-    if config["vtrace"]:
+    if config["vtrace"] and not config["in_evaluation"]:
         assert config["batch_mode"] == "truncate_episodes", \
             "Must use `truncate_episodes` batch mode with V-trace."
 

rllib/agents/trainer.py

@@ -179,6 +179,8 @@ COMMON_CONFIG = {
     # Number of episodes to run per evaluation period. If using multiple
     # evaluation workers, we will run at least this many episodes total.
     "evaluation_num_episodes": 10,
+    # Internal flag that is set to True for evaluation workers.
+    "in_evaluation": False,
     # Typical usage is to pass extra args to evaluation env creator
     # and to disable exploration by computing deterministic actions
     "evaluation_config": {
@@ -576,6 +578,7 @@ class Trainer(Trainable):
                 extra_config.update({
                     "batch_mode": "complete_episodes",
                     "batch_steps": 1,
+                    "in_evaluation": True,
                 })
                 logger.debug(
                     "using evaluation_config: {}".format(extra_config))