[rllib, tune] TrainingResult -> Dict, Removes C408 from flake8 (#2565)

python/ray/rllib/agents/a3c/a3c.py

@@ -103,8 +103,8 @@ class A3CAgent(Agent):
         FilterManager.synchronize(self.local_evaluator.filters,
                                   self.remote_evaluators)
         result = self.optimizer.collect_metrics()
-        result = result._replace(
-            timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps)
+        result.update(timesteps_this_iter=self.optimizer.num_steps_sampled -
+                      prev_steps)
         return result
 
     def _stop(self):

python/ray/rllib/agents/agent.py

@@ -12,7 +12,6 @@ import tensorflow as tf
 from ray.rllib.evaluation.policy_evaluator import PolicyEvaluator
 from ray.rllib.utils import deep_update
 from ray.tune.registry import ENV_CREATOR, _global_registry
-from ray.tune.result import TrainingResult
 from ray.tune.trainable import Trainable
 
 COMMON_CONFIG = {
@@ -266,7 +265,7 @@ class _MockAgent(Agent):
         if self.config["mock_error"] and self.iteration == 1 \
                 and (self.config["persistent_error"] or not self.restored):
             raise Exception("mock error")
-        return TrainingResult(
+        return dict(
             episode_reward_mean=10,
             episode_len_mean=10,
             timesteps_this_iter=10,
@@ -310,7 +309,7 @@ class _SigmoidFakeData(_MockAgent):
         i = max(0, self.iteration - self.config["offset"])
         v = np.tanh(float(i) / self.config["width"])
         v *= self.config["height"]
-        return TrainingResult(
+        return dict(
             episode_reward_mean=v,
             episode_len_mean=v,
             timesteps_this_iter=self.config["iter_timesteps"],
@@ -330,7 +329,7 @@ class _ParameterTuningAgent(_MockAgent):
     }
 
     def _train(self):
-        return TrainingResult(
+        return dict(
             episode_reward_mean=self.config["reward_amt"] * self.iteration,
             episode_len_mean=self.config["reward_amt"],
             timesteps_this_iter=self.config["iter_timesteps"],

python/ray/rllib/agents/bc/bc.py

@@ -8,7 +8,6 @@ from ray.rllib.agents.bc.bc_evaluator import BCEvaluator, \
     GPURemoteBCEvaluator, RemoteBCEvaluator
 from ray.rllib.optimizers import AsyncGradientsOptimizer
 from ray.rllib.utils import merge_dicts
-from ray.tune.result import TrainingResult
 from ray.tune.trial import Resources
 
 DEFAULT_CONFIG = {
@@ -89,7 +88,7 @@ class BCAgent(Agent):
             for m in ray.get(metrics):
                 total_samples += m["num_samples"]
                 total_loss += m["loss"]
-        result = TrainingResult(
+        result = dict(
             mean_loss=total_loss / total_samples,
             timesteps_this_iter=total_samples,
         )

python/ray/rllib/agents/dqn/dqn.py

@@ -203,13 +203,14 @@ class DQNAgent(Agent):
             result = collect_metrics(self.local_evaluator,
                                      self.remote_evaluators)
 
-        return result._replace(
+        result.update(
             timesteps_this_iter=self.global_timestep - start_timestep,
             info=dict({
                 "min_exploration": min(exp_vals),
                 "max_exploration": max(exp_vals),
                 "num_target_updates": self.num_target_updates,
             }, **self.optimizer.stats()))
+        return result
 
     def _stop(self):
         # workaround for https://github.com/ray-project/ray/issues/1516

python/ray/rllib/agents/es/es.py

@@ -300,7 +300,7 @@ class ESAgent(Agent):
             "time_elapsed": step_tend - self.tstart
         }
 
-        result = ray.tune.result.TrainingResult(
+        result = dict(
             episode_reward_mean=eval_returns.mean(),
             episode_len_mean=eval_lengths.mean(),
             timesteps_this_iter=noisy_lengths.sum(),

python/ray/rllib/agents/impala/impala.py

@@ -93,8 +93,8 @@ class ImpalaAgent(Agent):
         FilterManager.synchronize(self.local_evaluator.filters,
                                   self.remote_evaluators)
         result = self.optimizer.collect_metrics()
-        result = result._replace(
-            timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps)
+        result.update(timesteps_this_iter=self.optimizer.num_steps_sampled -
+                      prev_steps)
         return result
 
     def _stop(self):

python/ray/rllib/agents/pg/pg.py

@@ -50,5 +50,7 @@ class PGAgent(Agent):
     def _train(self):
         prev_steps = self.optimizer.num_steps_sampled
         self.optimizer.step()
-        return self.optimizer.collect_metrics()._replace(
-            timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps)
+        result = self.optimizer.collect_metrics()
+        result.update(timesteps_this_iter=self.optimizer.num_steps_sampled -
+                      prev_steps)
+        return result

python/ray/rllib/agents/ppo/ppo.py

@@ -112,9 +112,9 @@ class PPOAgent(Agent):
         FilterManager.synchronize(self.local_evaluator.filters,
                                   self.remote_evaluators)
         res = self.optimizer.collect_metrics()
-        res = res._replace(
+        res.update(
             timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps,
-            info=dict(fetches, **res.info))
+            info=dict(fetches, **res.get("info", {})))
         return res
 
     def _stop(self):

python/ray/rllib/evaluation/metrics.py

@@ -6,7 +6,6 @@ import numpy as np
 import collections
 
 import ray
-from ray.tune.result import TrainingResult
 
 
 def collect_metrics(local_evaluator, remote_evaluators=[]):
@@ -38,7 +37,7 @@ def collect_metrics(local_evaluator, remote_evaluators=[]):
     for policy_id, rewards in policy_rewards.copy().items():
         policy_rewards[policy_id] = np.mean(rewards)
 
-    return TrainingResult(
+    return dict(
         episode_reward_max=max_reward,
         episode_reward_min=min_reward,
         episode_reward_mean=avg_reward,

python/ray/rllib/optimizers/policy_optimizer.py

@@ -82,11 +82,11 @@ class PolicyOptimizer(object):
         """Returns evaluator and optimizer stats.
 
         Returns:
-            res (TrainingResult): TrainingResult from evaluator metrics with
+            res (dict): A training result dict from evaluator metrics with
                 `info` replaced with stats from self.
         """
         res = collect_metrics(self.local_evaluator, self.remote_evaluators)
-        res = res._replace(info=self.stats())
+        res.update(info=self.stats())
         return res
 
     def save(self):

python/ray/rllib/test/test_multi_agent_env.py

@@ -313,8 +313,8 @@ class TestMultiAgentEnv(unittest.TestCase):
         for i in range(100):
             result = pg.train()
             print("Iteration {}, reward {}, timesteps {}".format(
-                i, result.episode_reward_mean, result.timesteps_total))
-            if result.episode_reward_mean >= 50 * n:
+                i, result["episode_reward_mean"], result["timesteps_total"]))
+            if result["episode_reward_mean"] >= 50 * n:
                 return
         raise Exception("failed to improve reward")
 
@@ -349,7 +349,7 @@ class TestMultiAgentEnv(unittest.TestCase):
         for i in range(10):
             result = pg.train()
             print("Iteration {}, reward {}, timesteps {}".format(
-                i, result.episode_reward_mean, result.timesteps_total))
+                i, result["episode_reward_mean"], result["timesteps_total"]))
         self.assertTrue(
             pg.compute_action([0, 0, 0, 0], policy_id="policy_1") in [0, 1])
         self.assertTrue(
@@ -407,9 +407,10 @@ class TestMultiAgentEnv(unittest.TestCase):
                 ev.foreach_policy(
                     lambda p, _: p.update_target()
                     if isinstance(p, DQNPolicyGraph) else None)
-                print("Iter {}, rew {}".format(i, result.policy_reward_mean))
-                print("Total reward", result.episode_reward_mean)
-            if result.episode_reward_mean >= 25 * n:
+                print("Iter {}, rew {}".format(i,
+                                               result["policy_reward_mean"]))
+                print("Total reward", result["episode_reward_mean"])
+            if result["episode_reward_mean"] >= 25 * n:
                 return
         print(result)
         raise Exception("failed to improve reward")
@@ -442,9 +443,10 @@ class TestMultiAgentEnv(unittest.TestCase):
         for i in range(100):
             optimizer.step()
             result = collect_metrics(ev)
-            print("Iteration {}, rew {}".format(i, result.policy_reward_mean))
-            print("Total reward", result.episode_reward_mean)
-            if result.episode_reward_mean >= 25 * n:
+            print("Iteration {}, rew {}".format(i,
+                                                result["policy_reward_mean"]))
+            print("Total reward", result["episode_reward_mean"])
+            if result["episode_reward_mean"] >= 25 * n:
                 return
         raise Exception("failed to improve reward")
 

python/ray/rllib/test/test_policy_evaluator.py

@@ -124,8 +124,8 @@ class TestPolicyEvaluator(unittest.TestCase):
         ev.sample()
         ray.get(remote_ev.sample.remote())
         result = collect_metrics(ev, [remote_ev])
-        self.assertEqual(result.episodes_total, 20)
-        self.assertEqual(result.episode_reward_mean, 10)
+        self.assertEqual(result["episodes_total"], 20)
+        self.assertEqual(result["episode_reward_mean"], 10)
 
     def testAsync(self):
         ev = PolicyEvaluator(
@@ -160,12 +160,12 @@ class TestPolicyEvaluator(unittest.TestCase):
             batch = ev.sample()
             self.assertEqual(batch.count, 16)
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 0)
+        self.assertEqual(result["episodes_total"], 0)
         for _ in range(8):
             batch = ev.sample()
             self.assertEqual(batch.count, 16)
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 8)
+        self.assertEqual(result["episodes_total"], 8)
         indices = []
         for env in ev.async_env.vector_env.envs:
             self.assertEqual(env.unwrapped.config.worker_index, 0)
@@ -191,10 +191,10 @@ class TestPolicyEvaluator(unittest.TestCase):
         batch = ev.sample()
         self.assertEqual(batch.count, 16)
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 0)
+        self.assertEqual(result["episodes_total"], 0)
         batch = ev.sample()
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 4)
+        self.assertEqual(result["episodes_total"], 4)
 
     def testVectorEnvSupport(self):
         ev = PolicyEvaluator(
@@ -206,12 +206,12 @@ class TestPolicyEvaluator(unittest.TestCase):
             batch = ev.sample()
             self.assertEqual(batch.count, 10)
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 0)
+        self.assertEqual(result["episodes_total"], 0)
         for _ in range(8):
             batch = ev.sample()
             self.assertEqual(batch.count, 10)
         result = collect_metrics(ev, [])
-        self.assertEqual(result.episodes_total, 8)
+        self.assertEqual(result["episodes_total"], 8)
 
     def testTruncateEpisodes(self):
         ev = PolicyEvaluator(

python/ray/rllib/test/test_serving_env.py

@@ -157,8 +157,8 @@ class TestServingEnv(unittest.TestCase):
         for i in range(100):
             result = dqn.train()
             print("Iteration {}, reward {}, timesteps {}".format(
-                i, result.episode_reward_mean, result.timesteps_total))
-            if result.episode_reward_mean >= 100:
+                i, result["episode_reward_mean"], result["timesteps_total"]))
+            if result["episode_reward_mean"] >= 100:
                 return
         raise Exception("failed to improve reward")
 
@@ -168,8 +168,8 @@ class TestServingEnv(unittest.TestCase):
         for i in range(100):
             result = pg.train()
             print("Iteration {}, reward {}, timesteps {}".format(
-                i, result.episode_reward_mean, result.timesteps_total))
-            if result.episode_reward_mean >= 100:
+                i, result["episode_reward_mean"], result["timesteps_total"]))
+            if result["episode_reward_mean"] >= 100:
                 return
         raise Exception("failed to improve reward")
 
@@ -180,8 +180,8 @@ class TestServingEnv(unittest.TestCase):
         for i in range(100):
             result = pg.train()
             print("Iteration {}, reward {}, timesteps {}".format(
-                i, result.episode_reward_mean, result.timesteps_total))
-            if result.episode_reward_mean >= 100:
+                i, result["episode_reward_mean"], result["timesteps_total"]))
+            if result["episode_reward_mean"] >= 100:
                 return
         raise Exception("failed to improve reward")
 

python/ray/rllib/tuned_examples/regression_tests/regression_test.py

@@ -27,9 +27,11 @@ def _evaulate_config(filename):
     trials = tune.run_experiments(experiments)
     results = defaultdict(list)
     for t in trials:
-        results["time_total_s"] += [t.last_result.time_total_s]
-        results["episode_reward_mean"] += [t.last_result.episode_reward_mean]
-        results["training_iteration"] += [t.last_result.training_iteration]
+        results["time_total_s"] += [t.last_result["time_total_s"]]
+        results["episode_reward_mean"] += [
+            t.last_result["episode_reward_mean"]
+        ]
+        results["training_iteration"] += [t.last_result["training_iteration"]]
 
     return {k: np.median(v) for k, v in results.items()}
 

python/ray/rllib/tuned_examples/run_regression_tests.py

@@ -23,7 +23,7 @@ if __name__ == '__main__':
 
     num_failures = 0
     for t in trials:
-        if (t.last_result.episode_reward_mean <
+        if (t.last_result["episode_reward_mean"] <
                 t.stopping_criterion["episode_reward_mean"]):
             num_failures += 1
 

python/ray/rllib/utils/__init__.py

@@ -5,12 +5,7 @@ from ray.rllib.utils.filter import Filter
 from ray.rllib.utils.policy_client import PolicyClient
 from ray.rllib.utils.policy_server import PolicyServer
 
-__all__ = [
-    "Filter",
-    "FilterManager",
-    "PolicyClient",
-    "PolicyServer",
-]
+__all__ = ["Filter", "FilterManager", "PolicyClient", "PolicyServer"]
 
 
 def merge_dicts(d1, d2):