From cdf94c18a44ed358b6cf421d44864b5ef6ad4237 Mon Sep 17 00:00:00 2001 From: Alok Singh <8325708+alok@users.noreply.github.com> Date: Thu, 3 May 2018 07:45:12 -0700 Subject: [PATCH] Clean up syntax for supported Python versions. (#1963) * Use set/dict literal syntax Ran code through [pyupgrade](https://github.com/asottile/pyupgrade). This is supported in every Python version 2.7+. * Drop unnecessary string format specification No need to specify 0,1.. if paramters are passed in order. * Revert "Drop unnecessary string format specification" This reverts commit efa5ec85d30ff69f34e5ed93e31343fea7647bcb. * Undo changes to cloudpickle Drop use of set literal until cloudpickle uses it. * Reformat code with YAPF We need to set up a git pre-push hook to automatically run this stuff. --- examples/resnet/resnet_main.py | 2 +- python/ray/common/test/test.py | 2 +- python/ray/dataframe/groupby.py | 2 +- python/ray/experimental/tfutils.py | 2 +- python/ray/plasma/test/test.py | 14 +++++++------- python/ray/tune/test/trial_scheduler_test.py | 16 ++++++++-------- python/ray/tune/trial_runner.py | 2 +- python/ray/worker.py | 8 +++++--- test/actor_test.py | 18 +++++++++--------- test/runtest.py | 6 +++--- test/stress_tests.py | 6 ++++-- 11 files changed, 41 insertions(+), 37 deletions(-) diff --git a/examples/resnet/resnet_main.py b/examples/resnet/resnet_main.py index 4b42fbee9..5175b6de3 100644 --- a/examples/resnet/resnet_main.py +++ b/examples/resnet/resnet_main.py @@ -230,7 +230,7 @@ def train(): # testing task with the current weights every 200 steps. acc = ray.get(acc_id) acc_id = test_actor.accuracy.remote(weight_id, step) - print("Step {0}: {1:.6f}".format(step - 200, acc)) + print("Step {}: {:.6f}".format(step - 200, acc)) except KeyboardInterrupt: pass diff --git a/python/ray/common/test/test.py b/python/ray/common/test/test.py index 5892d289f..4ae867ff9 100644 --- a/python/ray/common/test/test.py +++ b/python/ray/common/test/test.py @@ -133,7 +133,7 @@ class TestObjectID(unittest.TestCase): x = random_object_id() y = random_object_id() {x: y} - set([x, y]) + {x, y} class TestTask(unittest.TestCase): diff --git a/python/ray/dataframe/groupby.py b/python/ray/dataframe/groupby.py index 892bc8f74..94c8c4747 100644 --- a/python/ray/dataframe/groupby.py +++ b/python/ray/dataframe/groupby.py @@ -106,7 +106,7 @@ class DataFrameGroupBy(object): @property def groups(self): - return dict([(k, pd.Index(v)) for k, v in self._keys_and_values]) + return {k: pd.Index(v) for k, v in self._keys_and_values} def min(self, **kwargs): return self._apply_agg_function(lambda df: df.min(**kwargs)) diff --git a/python/ray/experimental/tfutils.py b/python/ray/experimental/tfutils.py index 10d5fb4bc..fc33900b8 100644 --- a/python/ray/experimental/tfutils.py +++ b/python/ray/experimental/tfutils.py @@ -49,7 +49,7 @@ class TensorFlowVariables(object): self.sess = sess queue = deque([loss]) variable_names = [] - explored_inputs = set([loss]) + explored_inputs = {loss} # We do a BFS on the dependency graph of the input function to find # the variables. diff --git a/python/ray/plasma/test/test.py b/python/ray/plasma/test/test.py index 8b0d62fe1..0ec424d6a 100644 --- a/python/ray/plasma/test/test.py +++ b/python/ray/plasma/test/test.py @@ -297,7 +297,7 @@ class TestPlasmaManager(unittest.TestCase): self.client1.seal(obj_id1) ready, waiting = self.client1.wait( [obj_id1], timeout=100, num_returns=1) - self.assertEqual(set(ready), set([obj_id1])) + self.assertEqual(set(ready), {obj_id1}) self.assertEqual(waiting, []) # Test wait if only one object available and only one object waited @@ -307,8 +307,8 @@ class TestPlasmaManager(unittest.TestCase): # Don't seal. ready, waiting = self.client1.wait( [obj_id2, obj_id1], timeout=100, num_returns=1) - self.assertEqual(set(ready), set([obj_id1])) - self.assertEqual(set(waiting), set([obj_id2])) + self.assertEqual(set(ready), {obj_id1}) + self.assertEqual(set(waiting), {obj_id2}) # Test wait if object is sealed later. obj_id3 = random_object_id() @@ -321,14 +321,14 @@ class TestPlasmaManager(unittest.TestCase): t.start() ready, waiting = self.client1.wait( [obj_id3, obj_id2, obj_id1], timeout=1000, num_returns=2) - self.assertEqual(set(ready), set([obj_id1, obj_id3])) - self.assertEqual(set(waiting), set([obj_id2])) + self.assertEqual(set(ready), {obj_id1, obj_id3}) + self.assertEqual(set(waiting), {obj_id2}) # Test if the appropriate number of objects is shown if some objects # are not ready. ready, waiting = self.client1.wait([obj_id3, obj_id2, obj_id1], 100, 3) - self.assertEqual(set(ready), set([obj_id1, obj_id3])) - self.assertEqual(set(waiting), set([obj_id2])) + self.assertEqual(set(ready), {obj_id1, obj_id3}) + self.assertEqual(set(waiting), {obj_id2}) # Don't forget to seal obj_id2. self.client1.seal(obj_id2) diff --git a/python/ray/tune/test/trial_scheduler_test.py b/python/ray/tune/test/trial_scheduler_test.py index b008af3c7..a15448db7 100644 --- a/python/ray/tune/test/trial_scheduler_test.py +++ b/python/ray/tune/test/trial_scheduler_test.py @@ -688,36 +688,36 @@ class PopulationBasedTestingSuite(unittest.TestCase): # Categorical case assertProduces( lambda: explore({"v": 4}, {"v": [3, 4, 8, 10]}, 0.0, lambda x: x), - set([3, 8])) + {3, 8}) assertProduces( lambda: explore({"v": 3}, {"v": [3, 4, 8, 10]}, 0.0, lambda x: x), - set([3, 4])) + {3, 4}) assertProduces( lambda: explore({"v": 10}, {"v": [3, 4, 8, 10]}, 0.0, lambda x: x), - set([8, 10])) + {8, 10}) assertProduces( lambda: explore({"v": 7}, {"v": [3, 4, 8, 10]}, 0.0, lambda x: x), - set([3, 4, 8, 10])) + {3, 4, 8, 10}) assertProduces( lambda: explore({"v": 4}, {"v": [3, 4, 8, 10]}, 1.0, lambda x: x), - set([3, 4, 8, 10])) + {3, 4, 8, 10}) # Continuous case assertProduces( lambda: explore( {"v": 100}, {"v": lambda: random.choice([10, 100])}, 0.0, lambda x: x), - set([80, 120])) + {80, 120}) assertProduces( lambda: explore( {"v": 100.0}, {"v": lambda: random.choice([10, 100])}, 0.0, lambda x: x), - set([80.0, 120.0])) + {80.0, 120.0}) assertProduces( lambda: explore( {"v": 100.0}, {"v": lambda: random.choice([10, 100])}, 1.0, lambda x: x), - set([10.0, 100.0])) + {10.0, 100.0}) def testYieldsTimeToOtherTrials(self): pbt, runner = self.basicSetup() diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 56474c119..3f07e7bb5 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -172,7 +172,7 @@ class TrialRunner(object): if max_debug == start_num: break - for local_dir in sorted(set([t.local_dir for t in self._trials])): + for local_dir in sorted({t.local_dir for t in self._trials}): messages.append("Result logdir: {}".format(local_dir)) for state, trials in sorted(states.items()): limit = limit_per_state[state] diff --git a/python/ray/worker.py b/python/ray/worker.py index a12f93a54..dc246ce8e 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -464,9 +464,11 @@ class Worker(object): final_results = self.retrieve_and_deserialize(plain_object_ids, 0) # Construct a dictionary mapping object IDs that we haven't gotten yet # to their original index in the object_ids argument. - unready_ids = dict((plain_object_ids[i].binary(), i) - for (i, val) in enumerate(final_results) - if val is plasma.ObjectNotAvailable) + unready_ids = { + plain_object_ids[i].binary(): i + for (i, val) in enumerate(final_results) + if val is plasma.ObjectNotAvailable + } was_blocked = (len(unready_ids) > 0) # Try reconstructing any objects we haven't gotten yet. Try to get them # until at least get_timeout_milliseconds milliseconds passes, then diff --git a/test/actor_test.py b/test/actor_test.py index 7e040185b..6114d2e1f 100644 --- a/test/actor_test.py +++ b/test/actor_test.py @@ -774,7 +774,7 @@ class ActorsWithGPUs(unittest.TestCase): # Make sure that no two actors are assigned to the same GPU. locations_and_ids = ray.get( [actor.get_location_and_ids.remote() for actor in actors]) - node_names = set([location for location, gpu_id in locations_and_ids]) + node_names = {location for location, gpu_id in locations_and_ids} self.assertEqual(len(node_names), num_local_schedulers) location_actor_combinations = [] for node_name in node_names: @@ -815,7 +815,7 @@ class ActorsWithGPUs(unittest.TestCase): # Make sure that no two actors are assigned to the same GPU. locations_and_ids = ray.get( [actor.get_location_and_ids.remote() for actor in actors1]) - node_names = set([location for location, gpu_id in locations_and_ids]) + node_names = {location for location, gpu_id in locations_and_ids} self.assertEqual(len(node_names), num_local_schedulers) # Keep track of which GPU IDs are being used for each location. @@ -847,9 +847,9 @@ class ActorsWithGPUs(unittest.TestCase): # Make sure that no two actors are assigned to the same GPU. locations_and_ids = ray.get( [actor.get_location_and_ids.remote() for actor in actors2]) - self.assertEqual( - node_names, - set([location for location, gpu_id in locations_and_ids])) + self.assertEqual(node_names, + {location + for location, gpu_id in locations_and_ids}) for location, gpu_ids in locations_and_ids: gpus_in_use[location].extend(gpu_ids) for node_name in node_names: @@ -887,7 +887,7 @@ class ActorsWithGPUs(unittest.TestCase): # Make sure that no two actors are assigned to the same GPU. locations_and_ids = ray.get( [actor.get_location_and_ids.remote() for actor in actors]) - node_names = set([location for location, gpu_id in locations_and_ids]) + node_names = {location for location, gpu_id in locations_and_ids} self.assertEqual(len(node_names), 2) for node_name in node_names: node_gpu_ids = [ @@ -896,8 +896,8 @@ class ActorsWithGPUs(unittest.TestCase): ] self.assertIn(len(node_gpu_ids), [5, 10]) self.assertEqual( - set(node_gpu_ids), - set([(i, ) for i in range(len(node_gpu_ids))])) + set(node_gpu_ids), {(i, ) + for i in range(len(node_gpu_ids))}) # Creating a new actor should fail because all of the GPUs are being # used. @@ -1942,7 +1942,7 @@ class ActorPlacementAndResources(unittest.TestCase): results = ray.get([result1, result2, result3]) self.assertEqual(results[0], results[2]) - self.assertEqual(set(results), set([0, 1])) + self.assertEqual(set(results), {0, 1}) # Make sure that when one actor goes out of scope a new actor is # created because some resources have been freed up. diff --git a/test/runtest.py b/test/runtest.py index a44543a21..1f95250cd 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -255,7 +255,7 @@ class SerializationTest(unittest.TestCase): # Test sets. self.assertEqual(ray.get(f.remote(set())), set()) - s = set([1, (1, 2, "hi")]) + s = {1, (1, 2, "hi")} self.assertEqual(ray.get(f.remote(s)), s) # Test types. @@ -1317,8 +1317,8 @@ class ResourcesTest(unittest.TestCase): self.assertEqual(list_of_ids, 10 * [[]]) list_of_ids = ray.get([f1.remote() for _ in range(10)]) - set_of_ids = set([tuple(gpu_ids) for gpu_ids in list_of_ids]) - self.assertEqual(set_of_ids, set([(i, ) for i in range(10)])) + set_of_ids = {tuple(gpu_ids) for gpu_ids in list_of_ids} + self.assertEqual(set_of_ids, {(i, ) for i in range(10)}) list_of_ids = ray.get([f2.remote(), f4.remote(), f4.remote()]) all_ids = [gpu_id for gpu_ids in list_of_ids for gpu_id in gpu_ids] diff --git a/test/stress_tests.py b/test/stress_tests.py index 62bf3604e..490a0149c 100644 --- a/test/stress_tests.py +++ b/test/stress_tests.py @@ -210,8 +210,10 @@ class ReconstructionTests(unittest.TestCase): state._initialize_global_state(self.redis_ip_address, self.redis_port) if os.environ.get('RAY_USE_NEW_GCS', False): tasks = state.task_table() - local_scheduler_ids = set( - task["LocalSchedulerID"] for task in tasks.values()) + local_scheduler_ids = { + task["LocalSchedulerID"] + for task in tasks.values() + } # Make sure that all nodes in the cluster were used by checking that # the set of local scheduler IDs that had a task scheduled or submitted