diff --git a/rllib/agents/trainer.py b/rllib/agents/trainer.py
index 150cb31fb..f3baf2d2e 100644
--- a/rllib/agents/trainer.py
+++ b/rllib/agents/trainer.py
@@ -594,6 +594,9 @@ class Trainer(Trainable):
             if self.config.get("evaluation_interval"):
                 # Update env_config with evaluation settings:
                 extra_config = copy.deepcopy(self.config["evaluation_config"])
+                # Assert that user has not unset "in_evaluation".
+                assert "in_evaluation" not in extra_config or \
+                    extra_config["in_evaluation"] is True
                 extra_config.update({
                     "batch_mode": "complete_episodes",
                     "batch_steps": 1,
diff --git a/rllib/rollout.py b/rllib/rollout.py
index 4273d0fe4..f509d1b20 100755
--- a/rllib/rollout.py
+++ b/rllib/rollout.py
@@ -2,11 +2,12 @@
 
 import argparse
 import collections
+import copy
 import json
 import os
+from pathlib import Path
 import pickle
 import shelve
-from pathlib import Path
 
 import gym
 import ray
@@ -210,7 +211,8 @@ def create_parser(parser_creator=None):
         default="{}",
         type=json.loads,
         help="Algorithm-specific configuration (e.g. env, hyperparams). "
-        "Gets merged with loaded configuration from checkpoint file.")
+        "Gets merged with loaded configuration from checkpoint file and "
+        "`evaluation_config` settings therein.")
     parser.add_argument(
         "--save-info",
         default=False,
@@ -258,6 +260,9 @@ def run(args, parser):
     if "num_workers" in config:
         config["num_workers"] = min(2, config["num_workers"])
 
+    # Merge with `evaluation_config`.
+    evaluation_config = copy.deepcopy(config.get("evaluation_config", {}))
+    config = merge_dicts(config, evaluation_config)
     # Merge with command line `--config` settings.
     config = merge_dicts(config, args.config)
     if not args.env: