[RLlib] Issue 9402 MARWIL producing nan rewards. (#9429)

rllib/BUILD

@@ -469,7 +469,9 @@ py_test(
 py_test(
     name = "test_marwil",
     tags = ["agents_dir"],
-    size = "small",
+    size = "medium",
+    # Include the json data file.
+    data = ["tests/data/cartpole/large.json"],
     srcs = ["agents/marwil/tests/test_marwil.py"]
 )
 
@@ -696,12 +698,12 @@ py_test(
     tags = ["quick_train", "external_files"],
     size = "small",
     # Include the json data file.
-    data = glob(["tests/data/cartpole_small/**"]),
+    data = ["tests/data/cartpole/small.json"],
     args = [
         "--env", "CartPole-v0",
         "--run", "DQN",
         "--stop", "'{\"training_iteration\": 1}'",
-        "--config", "'{\"framework\": \"tf\", \"input\": \"tests/data/cartpole_small\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"exploration_config\": {\"type\": \"SoftQ\"}}'"
+        "--config", "'{\"framework\": \"tf\", \"input\": \"tests/data/cartpole\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"exploration_config\": {\"type\": \"SoftQ\"}}'"
         ]
 )
 
@@ -798,12 +800,12 @@ py_test(
     tags = ["quick_train", "external_files"],
     size = "small",
     # Include the json data file.
-    data = glob(["tests/data/cartpole_small/**"]),
+    data = ["tests/data/cartpole/small.json"],
     args = [
         "--env", "CartPole-v0",
         "--run", "MARWIL",
         "--stop", "'{\"training_iteration\": 1}'",
-        "--config", "'{\"framework\": \"tf\", \"input\": \"tests/data/cartpole_small\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"shuffle_buffer_size\": 10}'"
+        "--config", "'{\"framework\": \"tf\", \"input\": \"tests/data/cartpole\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"shuffle_buffer_size\": 10}'"
         ]
 )
 
@@ -814,12 +816,12 @@ py_test(
     tags = ["quick_train", "external_files"],
     size = "small",
     # Include the json data file.
-    data = glob(["tests/data/cartpole_small/**"]),
+    data = ["tests/data/cartpole/small.json"],
     args = [
         "--env", "CartPole-v0",
         "--run", "MARWIL",
         "--stop", "'{\"training_iteration\": 1}'",
-        "--config", "'{\"framework\": \"torch\", \"input\": \"tests/data/cartpole_small\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"shuffle_buffer_size\": 10}'"
+        "--config", "'{\"framework\": \"torch\", \"input\": \"tests/data/cartpole\", \"learning_starts\": 0, \"input_evaluation\": [\"wis\", \"is\"], \"shuffle_buffer_size\": 10}'"
         ]
 )
 
@@ -1649,9 +1651,9 @@ py_test(
     tags = ["examples", "examples_C"],
     size = "small",
     # Include the json data file.
-    data = glob(["tests/data/cartpole_small/**"]),
+    data = ["tests/data/cartpole/small.json"],
     srcs = ["examples/custom_loss.py"],
-    args = ["--stop-iters=2", "--input-files=tests/data/cartpole_small"]
+    args = ["--stop-iters=2", "--input-files=tests/data/cartpole"]
 )
 
 py_test(
@@ -1660,9 +1662,9 @@ py_test(
     tags = ["examples", "examples_C"],
     size = "small",
     # Include the json data file.
-    data = glob(["tests/data/cartpole_small/**"]),
+    data = ["tests/data/cartpole/small.json"],
     srcs = ["examples/custom_loss.py"],
-    args = ["--torch", "--stop-iters=2", "--input-files=tests/data/cartpole_small"]
+    args = ["--torch", "--stop-iters=2", "--input-files=tests/data/cartpole"]
 )
 
 py_test(

rllib/agents/marwil/tests/test_marwil.py

@@ -1,3 +1,5 @@
+import os
+from pathlib import Path
 import unittest
 
 import ray
@@ -18,19 +20,40 @@ class TestMARWIL(unittest.TestCase):
     def tearDownClass(cls):
         ray.shutdown()
 
-    def test_marwil_compilation(self):
-        """Test whether a MARWILTrainer can be built with all frameworks."""
+    def test_marwil_compilation_and_learning_from_offline_file(self):
+        """Test whether a MARWILTrainer can be built with all frameworks.
+
+        And learns from a historic-data file.
+        """
+        rllib_dir = Path(__file__).parent.parent.parent.parent
+        print("rllib dir={}".format(rllib_dir))
+        data_file = os.path.join(rllib_dir, "tests/data/cartpole/large.json")
+        print("data_file={} exists={}".format(
+            data_file, os.path.isfile(data_file)))
+
         config = marwil.DEFAULT_CONFIG.copy()
         config["num_workers"] = 0  # Run locally.
-        num_iterations = 2
+        config["evaluation_num_workers"] = 1
+        config["evaluation_interval"] = 1
+        config["evaluation_config"] = {"input": "sampler"}
+        config["input"] = [data_file]
+        num_iterations = 300
 
         # Test for all frameworks.
         for _ in framework_iterator(config):
             trainer = marwil.MARWILTrainer(config=config, env="CartPole-v0")
             for i in range(num_iterations):
-                trainer.train()
+                eval_results = trainer.train()["evaluation"]
+                print("iter={} R={}".format(
+                    i, eval_results["episode_reward_mean"]))
+                # Learn until some reward is reached on an actual live env.
+                if eval_results["episode_reward_mean"] > 60.0:
+                    print("learnt!")
+                    break
+
             check_compute_single_action(
                 trainer, include_prev_action_reward=True)
+
             trainer.stop()
 
 

rllib/examples/custom_loss.py

@@ -31,7 +31,7 @@ parser.add_argument(
     type=str,
     default=os.path.join(
         os.path.dirname(os.path.abspath(__file__)),
-        "../tests/data/cartpole_small"))
+        "../tests/data/cartpole/small"))
 
 if __name__ == "__main__":
     ray.init()

rllib/tuned_examples/marwil/cartpole-marwil.yaml

@@ -10,6 +10,14 @@ cartpole-marwil:
     config:
         # Works for both torch and tf.
         framework: tf
+        # In order to evaluate on an actual environment, use these following
+        # settings:
+        evaluation_num_workers: 1
+        evaluation_interval: 1
+        evaluation_config:
+            input: sampler
+        # Compare IL (beta=0) vs MARWIL.
         beta:
-            grid_search: [0, 1]   # compare IL (beta=0) vs MARWIL
+            grid_search: [0, 1]
+        # The historic (offline) data file from the PPO run (at the top).
         input: /tmp/out