Use flake8-comprehensions (#1976)

* Add flake8 to Travis

* Add flake8-comprehensions

[flake8 plugin](https://github.com/adamchainz/flake8-comprehensions) that
checks for useless constructions.

* Use generators instead of lists where appropriate

A lot of the builtins can take in generators instead of lists.

This commit applies `flake8-comprehensions` to find them.

* Fix lint error

* Fix some string formatting

The rest can be fixed in another PR

* Fix compound literals syntax

This should probably be merged after #1963.

* dict() -> {}

* Use dict literal syntax

dict(...) -> {...}

* Rewrite nested dicts

* Fix hanging indent

* Add missing import

* Add missing quote

* fmt

* Add missing whitespace

* rm duplicate pip install

This is already installed in another file.

* Fix indent

* move `merge_dicts` into utils

* Bring up to date with `master`

* Add automatic syntax upgrade

* rm pyupgrade

In case users want to still use it on their own, the upgrade-syn.sh script was
left in the `.travis` dir.

python/ray/experimental/array/distributed/core.py

@@ -241,7 +241,7 @@ def subblocks(a, *ranges):
     result = DistArray(shape)
     for index in np.ndindex(*result.num_blocks):
         result.objectids[index] = a.objectids[tuple(
-            [ranges[i][index[i]] for i in range(a.ndim)])]
+            ranges[i][index[i]] for i in range(a.ndim))]
     return result
 
 

python/ray/experimental/state.py

@@ -360,7 +360,7 @@ class GlobalState(object):
         """
         self._check_connected()
         db_client_keys = self.redis_client.keys(DB_CLIENT_PREFIX + "*")
-        node_info = dict()
+        node_info = {}
         for key in db_client_keys:
             client_info = self.redis_client.hgetall(key)
             node_ip_address = decode(client_info[b"node_ip_address"])
@@ -403,7 +403,7 @@ class GlobalState(object):
         """
         relevant_files = self.redis_client.keys("LOGFILE*")
 
-        ip_filename_file = dict()
+        ip_filename_file = {}
 
         for filename in relevant_files:
             filename = filename.decode("ascii")
@@ -417,7 +417,7 @@ class GlobalState(object):
                 file_str.append(y)
 
             if ip_addr not in ip_filename_file:
-                ip_filename_file[ip_addr] = dict()
+                ip_filename_file[ip_addr] = {}
 
             ip_filename_file[ip_addr][filename] = file_str
 
@@ -445,7 +445,7 @@ class GlobalState(object):
                 list of profiling information for tasks where the events have
                 no task ID.
         """
-        task_info = dict()
+        task_info = {}
         event_log_sets = self.redis_client.keys("event_log*")
 
         # The heap is used to maintain the set of x tasks that occurred the
@@ -498,7 +498,7 @@ class GlobalState(object):
                 for event in event_dict:
                     if "task_id" in event[3]:
                         task_id = event[3]["task_id"]
-                task_info[task_id] = dict()
+                task_info[task_id] = {}
                 task_info[task_id]["score"] = score
                 # Add task to (min/max) heap by its start point.
                 # if fwd, we want to delete the largest elements, so -score
@@ -901,7 +901,7 @@ class GlobalState(object):
     def workers(self):
         """Get a dictionary mapping worker ID to worker information."""
         worker_keys = self.redis_client.keys("Worker*")
-        workers_data = dict()
+        workers_data = {}
 
         for worker_key in worker_keys:
             worker_info = self.redis_client.hgetall(worker_key)
@@ -927,7 +927,7 @@ class GlobalState(object):
 
     def actors(self):
         actor_keys = self.redis_client.keys("Actor:*")
-        actor_info = dict()
+        actor_info = {}
         for key in actor_keys:
             info = self.redis_client.hgetall(key)
             actor_id = key[len("Actor:"):]

python/ray/experimental/tfutils.py

@@ -84,8 +84,8 @@ class TensorFlowVariables(object):
         for v in variable_list:
             self.variables[v.op.node_def.name] = v
 
-        self.placeholders = dict()
-        self.assignment_nodes = dict()
+        self.placeholders = {}
+        self.assignment_nodes = {}
 
         # Create new placeholders to put in custom weights.
         for k, var in self.variables.items():
@@ -109,9 +109,8 @@ class TensorFlowVariables(object):
         Returns:
             The length of all flattened variables concatenated.
         """
-        return sum([
-            np.prod(v.get_shape().as_list()) for v in self.variables.values()
-        ])
+        return sum(
+            np.prod(v.get_shape().as_list()) for v in self.variables.values())
 
     def _check_sess(self):
         """Checks if the session is set, and if not throw an error message."""

python/ray/experimental/ui.py

@@ -580,8 +580,11 @@ def cpu_usage():
             y_range=[0, 1])
 
         # Create the data source that the plot will pull from
-        time_series_source = ColumnDataSource(
-            data=dict(left=[], right=[], top=[]))
+        time_series_source = ColumnDataSource(data={
+            'left': [],
+            'right': [],
+            'top': []
+        })
 
         # Plot the rectangles representing the distribution
         time_series_fig.quad(
@@ -731,7 +734,7 @@ def cluster_usage():
         earliest = time.time()
         latest = 0
 
-        node_to_tasks = dict()
+        node_to_tasks = {}
         # Determine which task has the earlest start time out of the ones
         # passed into the update function
         for task_id, data in tasks.items():