Use flake8-comprehensions (#1976)

* Add flake8 to Travis

* Add flake8-comprehensions

[flake8 plugin](https://github.com/adamchainz/flake8-comprehensions) that
checks for useless constructions.

* Use generators instead of lists where appropriate

A lot of the builtins can take in generators instead of lists.

This commit applies `flake8-comprehensions` to find them.

* Fix lint error

* Fix some string formatting

The rest can be fixed in another PR

* Fix compound literals syntax

This should probably be merged after #1963.

* dict() -> {}

* Use dict literal syntax

dict(...) -> {...}

* Rewrite nested dicts

* Fix hanging indent

* Add missing import

* Add missing quote

* fmt

* Add missing whitespace

* rm duplicate pip install

This is already installed in another file.

* Fix indent

* move `merge_dicts` into utils

* Bring up to date with `master`

* Add automatic syntax upgrade

* rm pyupgrade

In case users want to still use it on their own, the upgrade-syn.sh script was
left in the `.travis` dir.

python/ray/dataframe/concat.py

@@ -38,8 +38,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
                          "and ray.dataframe.DataFrame objs are "
                          "valid", type(type_check))
 
-    all_series = all([isinstance(obj, pandas.Series)
-                      for obj in objs])
+    all_series = all(isinstance(obj, pandas.Series)
+                     for obj in objs)
     if all_series:
         return pandas.concat(objs, axis, join, join_axes,
                              ignore_index, keys, levels, names,
@@ -47,8 +47,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
 
     if isinstance(objs, dict):
         raise NotImplementedError(
-              "Obj as dicts not implemented. To contribute to "
-              "Pandas on Ray, please visit github.com/ray-project/ray.")
+            "Obj as dicts not implemented. To contribute to "
+            "Pandas on Ray, please visit github.com/ray-project/ray.")
 
     axis = pandas.DataFrame()._get_axis_number(axis)
 

python/ray/dataframe/dataframe.py

@@ -668,7 +668,7 @@ class DataFrame(object):
             mismatch = len(by) != len(self) if axis == 0 \
                 else len(by) != len(self.columns)
 
-            if all([obj in self for obj in by]) and mismatch:
+            if all(obj in self for obj in by) and mismatch:
                 raise NotImplementedError(
                     "Groupby with lists of columns not yet supported.")
             elif mismatch:
@@ -2194,7 +2194,7 @@ class DataFrame(object):
             A Series with the index for each maximum value for the axis
                 specified.
         """
-        if not all([d != np.dtype('O') for d in self.dtypes]):
+        if not all(d != np.dtype('O') for d in self.dtypes):
             raise TypeError(
                 "reduction operation 'argmax' not allowed for this dtype")
 
@@ -2216,7 +2216,7 @@ class DataFrame(object):
             A Series with the index for each minimum value for the axis
                 specified.
         """
-        if not all([d != np.dtype('O') for d in self.dtypes]):
+        if not all(d != np.dtype('O') for d in self.dtypes):
             raise TypeError(
                 "reduction operation 'argmax' not allowed for this dtype")
 
@@ -3196,9 +3196,9 @@ class DataFrame(object):
             """
             # This if call prevents ValueErrors with object only partitions
             if (numeric_only and
-                    all([dtype == np.dtype('O') or
-                         is_timedelta64_dtype(dtype)
-                         for dtype in df.dtypes])):
+                    all(dtype == np.dtype('O') or
+                        is_timedelta64_dtype(dtype)
+                        for dtype in df.dtypes)):
                 return base_object
             else:
                 return df.quantile(q=q, axis=axis, numeric_only=numeric_only,
@@ -4224,16 +4224,28 @@ class DataFrame(object):
                tupleize_cols=None, date_format=None, doublequote=True,
                escapechar=None, decimal="."):
 
-        kwargs = dict(
-                path_or_buf=path_or_buf, sep=sep, na_rep=na_rep,
-                float_format=float_format, columns=columns, header=header,
-                index=index, index_label=index_label, mode=mode,
-                encoding=encoding, compression=compression, quoting=quoting,
-                quotechar=quotechar, line_terminator=line_terminator,
-                chunksize=chunksize, tupleize_cols=tupleize_cols,
-                date_format=date_format, doublequote=doublequote,
-                escapechar=escapechar, decimal=decimal
-                )
+        kwargs = {
+            'path_or_buf': path_or_buf,
+            'sep': sep,
+            'na_rep': na_rep,
+            'float_format': float_format,
+            'columns': columns,
+            'header': header,
+            'index': index,
+            'index_label': index_label,
+            'mode': mode,
+            'encoding': encoding,
+            'compression': compression,
+            'quoting': quoting,
+            'quotechar': quotechar,
+            'line_terminator': line_terminator,
+            'chunksize': chunksize,
+            'tupleize_cols': tupleize_cols,
+            'date_format': date_format,
+            'doublequote': doublequote,
+            'escapechar': escapechar,
+            'decimal': decimal
+        }
 
         if compression is not None:
             warnings.warn("Defaulting to Pandas implementation",

python/ray/dataframe/io.py

@@ -208,60 +208,61 @@ def read_csv(filepath_or_buffer,
         kwargs: Keyword arguments in pandas::from_csv
     """
 
-    kwargs = dict(
-        sep=sep,
-        delimiter=delimiter,
-        header=header,
-        names=names,
-        index_col=index_col,
-        usecols=usecols,
-        squeeze=squeeze,
-        prefix=prefix,
-        mangle_dupe_cols=mangle_dupe_cols,
-        dtype=dtype,
-        engine=engine,
-        converters=converters,
-        true_values=true_values,
-        false_values=false_values,
-        skipinitialspace=skipinitialspace,
-        skiprows=skiprows,
-        nrows=nrows,
-        na_values=na_values,
-        keep_default_na=keep_default_na,
-        na_filter=na_filter,
-        verbose=verbose,
-        skip_blank_lines=skip_blank_lines,
-        parse_dates=parse_dates,
-        infer_datetime_format=infer_datetime_format,
-        keep_date_col=keep_date_col,
-        date_parser=date_parser,
-        dayfirst=dayfirst,
-        iterator=iterator,
-        chunksize=chunksize,
-        compression=compression,
-        thousands=thousands,
-        decimal=decimal,
-        lineterminator=lineterminator,
-        quotechar=quotechar,
-        quoting=quoting,
-        escapechar=escapechar,
-        comment=comment,
-        encoding=encoding,
-        dialect=dialect,
-        tupleize_cols=tupleize_cols,
-        error_bad_lines=error_bad_lines,
-        warn_bad_lines=warn_bad_lines,
-        skipfooter=skipfooter,
-        skip_footer=skip_footer,
-        doublequote=doublequote,
-        delim_whitespace=delim_whitespace,
-        as_recarray=as_recarray,
-        compact_ints=compact_ints,
-        use_unsigned=use_unsigned,
-        low_memory=low_memory,
-        buffer_lines=buffer_lines,
-        memory_map=memory_map,
-        float_precision=float_precision)
+    kwargs = {
+        'sep': sep,
+        'delimiter': delimiter,
+        'header': header,
+        'names': names,
+        'index_col': index_col,
+        'usecols': usecols,
+        'squeeze': squeeze,
+        'prefix': prefix,
+        'mangle_dupe_cols': mangle_dupe_cols,
+        'dtype': dtype,
+        'engine': engine,
+        'converters': converters,
+        'true_values': true_values,
+        'false_values': false_values,
+        'skipinitialspace': skipinitialspace,
+        'skiprows': skiprows,
+        'nrows': nrows,
+        'na_values': na_values,
+        'keep_default_na': keep_default_na,
+        'na_filter': na_filter,
+        'verbose': verbose,
+        'skip_blank_lines': skip_blank_lines,
+        'parse_dates': parse_dates,
+        'infer_datetime_format': infer_datetime_format,
+        'keep_date_col': keep_date_col,
+        'date_parser': date_parser,
+        'dayfirst': dayfirst,
+        'iterator': iterator,
+        'chunksize': chunksize,
+        'compression': compression,
+        'thousands': thousands,
+        'decimal': decimal,
+        'lineterminator': lineterminator,
+        'quotechar': quotechar,
+        'quoting': quoting,
+        'escapechar': escapechar,
+        'comment': comment,
+        'encoding': encoding,
+        'dialect': dialect,
+        'tupleize_cols': tupleize_cols,
+        'error_bad_lines': error_bad_lines,
+        'warn_bad_lines': warn_bad_lines,
+        'skipfooter': skipfooter,
+        'skip_footer': skip_footer,
+        'doublequote': doublequote,
+        'delim_whitespace': delim_whitespace,
+        'as_recarray': as_recarray,
+        'compact_ints': compact_ints,
+        'use_unsigned': use_unsigned,
+        'low_memory': low_memory,
+        'buffer_lines': buffer_lines,
+        'memory_map': memory_map,
+        'float_precision': float_precision,
+    }
 
     # Default to Pandas read_csv for non-serializable objects
     if not isinstance(filepath_or_buffer, str) or \

python/ray/dataframe/test/test_dataframe.py

@@ -1783,7 +1783,7 @@ def test_fillna_dtype_conversion(num_partitions=2):
     )
 
     # equiv of replace
-    df = pd.DataFrame(dict(A=[1, np.nan], B=[1., 2.]))
+    df = pd.DataFrame({'A': [1, np.nan], 'B': [1., 2.]})
     ray_df = from_pandas(df, num_partitions)
     for v in ['', 1, np.nan, 1.0]:
         assert ray_df_equals_pandas(

python/ray/dataframe/utils.py

@@ -9,7 +9,7 @@ import ray
 from . import get_npartitions
 
 
-_NAN_BLOCKS = dict()
+_NAN_BLOCKS = {}
 
 
 def _get_nan_block_id(n_row=1, n_col=1, transpose=False):
@@ -225,7 +225,7 @@ def _map_partitions(func, partitions, *argslists):
         return [_deploy_func.remote(func, part, argslists[0])
                 for part in partitions]
     else:
-        assert(all([len(args) == len(partitions) for args in argslists]))
+        assert(all(len(args) == len(partitions) for args in argslists))
         return [_deploy_func.remote(func, *args)
                 for args in zip(partitions, *argslists)]