diff --git a/python/ray/dataframe/dataframe.py b/python/ray/dataframe/dataframe.py index 10b8df9ec..df3be2a54 100644 --- a/python/ray/dataframe/dataframe.py +++ b/python/ray/dataframe/dataframe.py @@ -1444,9 +1444,31 @@ class DataFrame(object): return result def diff(self, periods=1, axis=0): - raise NotImplementedError( - "To contribute to Pandas on Ray, please visit " - "github.com/ray-project/ray.") + """Finds the difference between elements on the axis requested + + Args: + periods: Periods to shift for forming difference + axis: Take difference over rows or columns + + Returns: + DataFrame with the diff applied + """ + axis = pd.DataFrame()._get_axis_number(axis) + partitions = (self._col_partitions if + axis == 0 else self._row_partitions) + + result = _map_partitions(lambda df: + df.diff(axis=axis, periods=periods), + partitions) + + if (axis == 1): + return DataFrame(row_partitions=result, + columns=self.columns, + index=self.index) + if (axis == 0): + return DataFrame(col_partitions=result, + columns=self.columns, + index=self.index) def div(self, other, axis='columns', level=None, fill_value=None): """Divides this DataFrame against another DataFrame/Series/scalar. @@ -1661,7 +1683,7 @@ class DataFrame(object): # TODO: group series here into full df partitions to reduce # the number of remote calls to helper other_series = other_df.iloc[idx['index_within_partition']] - curr_index = self._row_metadata._coord_df.iloc[i] + curr_index = self._row_metadata._coord_df.loc[i] curr_df = self._row_partitions[int(curr_index['partition'])] results.append(_deploy_func.remote(helper, curr_df, diff --git a/python/ray/dataframe/test/test_dataframe.py b/python/ray/dataframe/test/test_dataframe.py index 817d849b0..73581fcd5 100644 --- a/python/ray/dataframe/test/test_dataframe.py +++ b/python/ray/dataframe/test/test_dataframe.py @@ -226,6 +226,7 @@ def test_int_dataframe(): test_quantile(ray_df, pandas_df, .5) test_quantile(ray_df, pandas_df, .75) test_describe(ray_df, pandas_df) + test_diff(ray_df, pandas_df) test_all(ray_df, pandas_df) test_any(ray_df, pandas_df) @@ -390,6 +391,7 @@ def test_float_dataframe(): test_quantile(ray_df, pandas_df, .5) test_quantile(ray_df, pandas_df, .75) test_describe(ray_df, pandas_df) + test_diff(ray_df, pandas_df) test_all(ray_df, pandas_df) test_any(ray_df, pandas_df) @@ -715,6 +717,7 @@ def test_nan_dataframe(): test_quantile(ray_df, pandas_df, .5) test_quantile(ray_df, pandas_df, .75) test_describe(ray_df, pandas_df) + test_diff(ray_df, pandas_df) test_all(ray_df, pandas_df) test_any(ray_df, pandas_df) @@ -1170,11 +1173,12 @@ def test_describe(ray_df, pandas_df): assert(ray_df.describe().equals(pandas_df.describe())) -def test_diff(): - ray_df = create_test_dataframe() - - with pytest.raises(NotImplementedError): - ray_df.diff() +@pytest.fixture +def test_diff(ray_df, pandas_df): + assert(ray_df_equals_pandas(ray_df.diff(), pandas_df.diff())) + assert(ray_df_equals_pandas(ray_df.diff(axis=1), pandas_df.diff(axis=1))) + assert(ray_df_equals_pandas(ray_df.diff(periods=1), + pandas_df.diff(periods=1))) def test_div():