From e70490a415cc35468f27f623aa1e36a2255ed91a Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Tue, 28 Jun 2016 17:20:09 -0400 Subject: [PATCH 1/5] BUG: Correct AverageDollarVolume NaN handling `AverageDollarVolume` used `nanmean`, which discards NaNs before averaging, giving an ADV which is too high for any equities that have any NaNs. Changing the method to `nansum` divided by window length so that the denominator is the same no matter whether there are NaNs or not. --- zipline/pipeline/factors/technical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zipline/pipeline/factors/technical.py b/zipline/pipeline/factors/technical.py index 10666a8a..c7e26efe 100644 --- a/zipline/pipeline/factors/technical.py +++ b/zipline/pipeline/factors/technical.py @@ -156,7 +156,7 @@ class AverageDollarVolume(CustomFactor): inputs = [USEquityPricing.close, USEquityPricing.volume] def compute(self, today, assets, out, close, volume): - out[:] = nanmean(close * volume, axis=0) + out[:] = nansum(close * volume, axis=0) / len(close) class _ExponentialWeightedFactor(SingleInputMixin, CustomFactor): From ebbcca73e817c9aa326ee9eff7680952600bbe47 Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Wed, 29 Jun 2016 10:12:35 -0400 Subject: [PATCH 2/5] TST: Add NaN cases to `AverageDollarVolume` factor test. --- tests/pipeline/test_engine.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/pipeline/test_engine.py b/tests/pipeline/test_engine.py index d3a8501e..dfd7fc7d 100644 --- a/tests/pipeline/test_engine.py +++ b/tests/pipeline/test_engine.py @@ -1056,7 +1056,12 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): index=dates, columns=cls.asset_finder.retrieve_all(sids), ) + cls.raw_data_with_nans = cls.raw_data.where(cls.raw_data % 3 != 0) + open_loader = DataFrameLoader( + USEquityPricing.open, + cls.raw_data_with_nans, + ) close_loader = DataFrameLoader(USEquityPricing.close, cls.raw_data) volume_loader = DataFrameLoader( USEquityPricing.volume, @@ -1065,6 +1070,7 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): cls.engine = SimplePipelineEngine( { + USEquityPricing.open: open_loader, USEquityPricing.close: close_loader, USEquityPricing.volume: volume_loader, }.__getitem__, @@ -1195,6 +1201,14 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): columns={ 'dv1': AverageDollarVolume(window_length=1), 'dv5': AverageDollarVolume(window_length=5), + 'dv1_nan': AverageDollarVolume( + window_length=1, + inputs=[USEquityPricing.open, USEquityPricing.volume], + ), + 'dv5_nan': AverageDollarVolume( + window_length=5, + inputs=[USEquityPricing.open, USEquityPricing.volume], + ), } ), self.dates[5], @@ -1207,6 +1221,15 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): expected_5 = rolling_mean((self.raw_data ** 2) * 2, window=5)[5:] assert_frame_equal(results['dv5'].unstack(), expected_5) + expected_1_nan = (self.raw_data_with_nans[5:] + * self.raw_data[5:] * 2).fillna(0) + assert_frame_equal(results['dv1_nan'].unstack(), expected_1_nan) + + expected_5_nan = rolling_mean((self.raw_data_with_nans + * self.raw_data * 2).fillna(0), + window=5)[5:] + assert_frame_equal(results['dv5_nan'].unstack(), expected_5_nan) + @parameter_space(returns_length=[2, 3], correlation_length=[3, 4]) def test_correlation_factors(self, returns_length, correlation_length): """ From e67b5e5516332bbcb5b6c104540d5bed2f98b515 Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Wed, 29 Jun 2016 11:16:39 -0400 Subject: [PATCH 3/5] TST: Change `AverageDollarVolume` test to check case of partial NaNs --- tests/pipeline/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_engine.py b/tests/pipeline/test_engine.py index dfd7fc7d..99b42165 100644 --- a/tests/pipeline/test_engine.py +++ b/tests/pipeline/test_engine.py @@ -1056,7 +1056,7 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): index=dates, columns=cls.asset_finder.retrieve_all(sids), ) - cls.raw_data_with_nans = cls.raw_data.where(cls.raw_data % 3 != 0) + cls.raw_data_with_nans = cls.raw_data.where((cls.raw_data % 2) != 0) open_loader = DataFrameLoader( USEquityPricing.open, From 985e6bafeeeb9a0c971f7860e6f88f6a7853a8e4 Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Wed, 29 Jun 2016 11:34:21 -0400 Subject: [PATCH 4/5] DOC: Add comment explaining ADV NaN test expected result calculation. --- tests/pipeline/test_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pipeline/test_engine.py b/tests/pipeline/test_engine.py index 99b42165..c42f4a44 100644 --- a/tests/pipeline/test_engine.py +++ b/tests/pipeline/test_engine.py @@ -1221,6 +1221,9 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase): expected_5 = rolling_mean((self.raw_data ** 2) * 2, window=5)[5:] assert_frame_equal(results['dv5'].unstack(), expected_5) + # The following two use USEquityPricing.open and .volume as inputs. + # The former uses self.raw_data_with_nans, and the latter uses + # .raw_data * 2. Thus we multiply instead of squaring as above. expected_1_nan = (self.raw_data_with_nans[5:] * self.raw_data[5:] * 2).fillna(0) assert_frame_equal(results['dv1_nan'].unstack(), expected_1_nan) From e0e18bc328d15c199f6e3280f6c3316bff04fdfa Mon Sep 17 00:00:00 2001 From: Nathan Wolfe Date: Wed, 29 Jun 2016 13:47:56 -0400 Subject: [PATCH 5/5] DOC: Add `AverageDollarVolume` change to release notes. --- docs/source/whatsnew/1.0.2.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/whatsnew/1.0.2.txt b/docs/source/whatsnew/1.0.2.txt index 348dc076..f88bf8c8 100644 --- a/docs/source/whatsnew/1.0.2.txt +++ b/docs/source/whatsnew/1.0.2.txt @@ -15,7 +15,10 @@ Enhancements Bug Fixes ~~~~~~~~~ -None +- Changes :class:`~zipline.pipeline.factors.AverageDollarVolume` built-in + factor to treat missing close or volume values as 0. Previously, NaNs were + simply discarded before averaging, giving the remaining values too much + weight (:issue:`1309`). Documentation ~~~~~~~~~~~~~