wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-06 05:16:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

[DataFrame] Implement diff (#1996)

Browse Source 
* added diff method

* sanity checks

* flake8

* updated sanity checks'

* rebase and style updates

* updated diff tests and cleaned up code

* updated tests for periods

* flake8
This commit is contained in:
Rohan Singh
2018-05-05 00:44:52 -07:00
committed by Devin Petersohn
parent a58629f53f
commit 8509a51291
2 changed files with 35 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/dataframe/dataframe.py
+26 -4
View File
@@ -1444,9 +1444,31 @@ class DataFrame(object):
return result
def diff(self, periods=1, axis=0):
raise NotImplementedError(
"To contribute to Pandas on Ray, please visit "
"github.com/ray-project/ray.")
"""Finds the difference between elements on the axis requested
Args:
periods: Periods to shift for forming difference
axis: Take difference over rows or columns
Returns:
DataFrame with the diff applied
"""
axis = pd.DataFrame()._get_axis_number(axis)
partitions = (self._col_partitions if
axis == 0 else self._row_partitions)
result = _map_partitions(lambda df:
df.diff(axis=axis, periods=periods),
partitions)
if (axis == 1):
return DataFrame(row_partitions=result,
columns=self.columns,
index=self.index)
if (axis == 0):
return DataFrame(col_partitions=result,
columns=self.columns,
index=self.index)
def div(self, other, axis='columns', level=None, fill_value=None):
"""Divides this DataFrame against another DataFrame/Series/scalar.
@@ -1661,7 +1683,7 @@ class DataFrame(object):
# TODO: group series here into full df partitions to reduce
# the number of remote calls to helper
other_series = other_df.iloc[idx['index_within_partition']]
curr_index = self._row_metadata._coord_df.iloc[i]
curr_index = self._row_metadata._coord_df.loc[i]
curr_df = self._row_partitions[int(curr_index['partition'])]
results.append(_deploy_func.remote(helper,
curr_df,
python/ray/dataframe/test/test_dataframe.py
+9 -5
View File
@@ -226,6 +226,7 @@ def test_int_dataframe():
test_quantile(ray_df, pandas_df, .5)
test_quantile(ray_df, pandas_df, .75)
test_describe(ray_df, pandas_df)
test_diff(ray_df, pandas_df)
test_all(ray_df, pandas_df)
test_any(ray_df, pandas_df)
@@ -390,6 +391,7 @@ def test_float_dataframe():
test_quantile(ray_df, pandas_df, .5)
test_quantile(ray_df, pandas_df, .75)
test_describe(ray_df, pandas_df)
test_diff(ray_df, pandas_df)
test_all(ray_df, pandas_df)
test_any(ray_df, pandas_df)
@@ -715,6 +717,7 @@ def test_nan_dataframe():
test_quantile(ray_df, pandas_df, .5)
test_quantile(ray_df, pandas_df, .75)
test_describe(ray_df, pandas_df)
test_diff(ray_df, pandas_df)
test_all(ray_df, pandas_df)
test_any(ray_df, pandas_df)
@@ -1170,11 +1173,12 @@ def test_describe(ray_df, pandas_df):
assert(ray_df.describe().equals(pandas_df.describe()))
def test_diff():
ray_df = create_test_dataframe()
with pytest.raises(NotImplementedError):
ray_df.diff()
@pytest.fixture
def test_diff(ray_df, pandas_df):
assert(ray_df_equals_pandas(ray_df.diff(), pandas_df.diff()))
assert(ray_df_equals_pandas(ray_df.diff(axis=1), pandas_df.diff(axis=1)))
assert(ray_df_equals_pandas(ray_df.diff(periods=1),
pandas_df.diff(periods=1)))
def test_div():