From d7b631617c59fb26febaab3c3f81eaae41381800 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Wed, 20 Jul 2016 17:10:08 -0400 Subject: [PATCH 01/11] ENH: made filters window safe --- zipline/pipeline/filters/filter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index c184e0d6..63138cbc 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -167,6 +167,10 @@ class Filter(RestrictedDTypeMixin, ComputableTerm): output of a Pipeline and for reducing memory consumption of Pipeline results. """ + + # make filters window safe + window_safe = True + ALLOWED_DTYPES = (bool_dtype,) # Used by RestrictedDTypeMixin dtype = bool_dtype From 98be158c207c024fca8d75f473c46d85c18fe007 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Thu, 21 Jul 2016 08:49:41 -0400 Subject: [PATCH 02/11] ENH: storing commits. test case added --- tests/pipeline/test_filter.py | 19 +++++++++++++++++++ zipline/pipeline/filters/filter.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index 0f8c4f3b..9c1cbdae 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -18,6 +18,7 @@ from numpy import ( ones, ones_like, putmask, + sum ) from numpy.random import randn, seed as random_seed @@ -379,3 +380,21 @@ class FilterTestCase(BasePipelineTestCase): initial_workspace={self.f: data}, ) check_arrays(results['isfinite'], isfinite(data)) + + def test_window_safe(self): + """ + Rationale : CustomFactors are *not all* window safe, while + CustomFilters are + """ + data = array([[True, False, False, True], + [False, True, False, True]], dtype=bool) + + class CustomFilterBool(Filter): + window_length = 0 + inputs = () + + filter_results = self.run_graph( + TermGraph({'factor': sum(CustomFilterBool, axis=0)}), + initial_workspace={CustomFilterBool: data}, + ) + check_arrays(filter_results['factor'], array([1, 1, 0, 2])) diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index 63138cbc..8267b95e 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -170,7 +170,7 @@ class Filter(RestrictedDTypeMixin, ComputableTerm): # make filters window safe window_safe = True - + ALLOWED_DTYPES = (bool_dtype,) # Used by RestrictedDTypeMixin dtype = bool_dtype From 21b33f03f99a39e33d06cadbb32d20730887503f Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Thu, 21 Jul 2016 10:21:19 -0400 Subject: [PATCH 03/11] ENH: test filter up and running --- tests/pipeline/test_filter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index 9c1cbdae..08eab31c 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -18,7 +18,7 @@ from numpy import ( ones, ones_like, putmask, - sum + nansum ) from numpy.random import randn, seed as random_seed @@ -380,11 +380,10 @@ class FilterTestCase(BasePipelineTestCase): initial_workspace={self.f: data}, ) check_arrays(results['isfinite'], isfinite(data)) - + def test_window_safe(self): """ - Rationale : CustomFactors are *not all* window safe, while - CustomFilters are + This computation is now possible as filters are windowsafe """ data = array([[True, False, False, True], [False, True, False, True]], dtype=bool) @@ -394,7 +393,7 @@ class FilterTestCase(BasePipelineTestCase): inputs = () filter_results = self.run_graph( - TermGraph({'factor': sum(CustomFilterBool, axis=0)}), + TermGraph({'filter': nansum(CustomFilterBool, axis=0)}), initial_workspace={CustomFilterBool: data}, ) - check_arrays(filter_results['factor'], array([1, 1, 0, 2])) + check_arrays(filter_results['filter'], array([1, 1, 0, 2])) From c31d3b7904fc7b65faf087e825c33694f5c4b320 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Thu, 21 Jul 2016 12:25:02 -0400 Subject: [PATCH 04/11] ENH: Test now works --- tests/pipeline/test_filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index 08eab31c..a527cee5 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -393,7 +393,7 @@ class FilterTestCase(BasePipelineTestCase): inputs = () filter_results = self.run_graph( - TermGraph({'filter': nansum(CustomFilterBool, axis=0)}), - initial_workspace={CustomFilterBool: data}, + TermGraph({'filter': CustomFilterBool()}), + initial_workspace={CustomFilterBool(): data}, ) - check_arrays(filter_results['filter'], array([1, 1, 0, 2])) + check_arrays(nansum(filter_results['filter'], axis=0), array([1, 1, 0, 2])) From 5e991a16d3f3df9a47d93cb4eb3711a1d47fb4ec Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Fri, 22 Jul 2016 14:30:35 -0400 Subject: [PATCH 05/11] ENH: Test added and runs without error --- tests/pipeline/test_filter.py | 43 +++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index a527cee5..363ad306 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -11,6 +11,7 @@ from numpy import ( eye, float64, full_like, + full, inf, isfinite, nan, @@ -18,12 +19,13 @@ from numpy import ( ones, ones_like, putmask, - nansum + nansum, ) from numpy.random import randn, seed as random_seed from zipline.errors import BadPercentileBounds from zipline.pipeline import Filter, Factor, TermGraph +from zipline.pipeline.factors import CustomFactor from zipline.testing import check_arrays from zipline.utils.numpy_utils import float64_dtype from .base import BasePipelineTestCase, with_default_shape @@ -382,18 +384,35 @@ class FilterTestCase(BasePipelineTestCase): check_arrays(results['isfinite'], isfinite(data)) def test_window_safe(self): - """ - This computation is now possible as filters are windowsafe - """ - data = array([[True, False, False, True], - [False, True, False, True]], dtype=bool) + # all true data set of (days, securities) + data = full(self.default_shape, True, dtype=bool) - class CustomFilterBool(Filter): - window_length = 0 + # rolling window length for TestFactor + k = 8 + + class InputFilter(Filter): inputs = () + window_length = 0 - filter_results = self.run_graph( - TermGraph({'filter': CustomFilterBool()}), - initial_workspace={CustomFilterBool(): data}, + class TestFactor(CustomFactor): + dtype = float64_dtype + inputs = (InputFilter(), ) + window_length = k + + def compute(self, today, assets, out, filter_): + # sum for each column + out[:] = nansum(filter_, axis=0) + + results = self.run_graph( + TermGraph({'windowsafe': TestFactor()}), + initial_workspace={InputFilter(): data}, ) - check_arrays(nansum(filter_results['filter'], axis=0), array([1, 1, 0, 2])) + + # number of securities in default_shape + securities = self.default_shape[1] + + # number of days in default_shape + n = self.default_shape[0] + + # assert that these two arrays are the same + check_arrays(results['windowsafe'], full(shape=((n - k + 1), securities), fill_value=k)) From 36a727f4afcb3c02bfefe07708855ddabd727a7c Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Fri, 22 Jul 2016 14:56:59 -0400 Subject: [PATCH 06/11] ENH: sum vs nansum cleared up --- tests/pipeline/test_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index 363ad306..d34dfa4a 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -19,7 +19,7 @@ from numpy import ( ones, ones_like, putmask, - nansum, + sum, ) from numpy.random import randn, seed as random_seed @@ -401,7 +401,7 @@ class FilterTestCase(BasePipelineTestCase): def compute(self, today, assets, out, filter_): # sum for each column - out[:] = nansum(filter_, axis=0) + out[:] = sum(filter_, axis=0) results = self.run_graph( TermGraph({'windowsafe': TestFactor()}), From b4aa0aecbb7c4a98601d6e378f018d08203bc127 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Fri, 22 Jul 2016 15:08:34 -0400 Subject: [PATCH 07/11] STY: Flake8 --- tests/pipeline/test_filter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index d34dfa4a..c2a6673d 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -382,7 +382,7 @@ class FilterTestCase(BasePipelineTestCase): initial_workspace={self.f: data}, ) check_arrays(results['isfinite'], isfinite(data)) - + def test_window_safe(self): # all true data set of (days, securities) data = full(self.default_shape, True, dtype=bool) @@ -414,5 +414,8 @@ class FilterTestCase(BasePipelineTestCase): # number of days in default_shape n = self.default_shape[0] + # shape of output array + output_shape = ((n - k + 1), securities) + # assert that these two arrays are the same - check_arrays(results['windowsafe'], full(shape=((n - k + 1), securities), fill_value=k)) + check_arrays(results['windowsafe'], full(output_shape, k)) From 9c6aa0cd583a62308f98a9f424acf34449dad028 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Fri, 22 Jul 2016 15:21:18 -0400 Subject: [PATCH 08/11] DOC: Added documentation about window-safe filters --- docs/source/whatsnew/1.0.2.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/whatsnew/1.0.2.txt b/docs/source/whatsnew/1.0.2.txt index 303f849e..076f5776 100644 --- a/docs/source/whatsnew/1.0.2.txt +++ b/docs/source/whatsnew/1.0.2.txt @@ -18,6 +18,9 @@ Enhancements - Allow correlations and regressions to be computed between two 2D factors by doing computations asset-wise (:issue:`1307`). +- Filters have been made window_safe by default. Now they can be passed in as + arguments to other Filters, Factors and Classifiers (:issue:`1338`). + Bug Fixes ~~~~~~~~~ From ea01fb074a1ed2a9dfbf378441364c9dbb346ad2 Mon Sep 17 00:00:00 2001 From: Gil Wassermann Date: Fri, 22 Jul 2016 16:08:33 -0400 Subject: [PATCH 09/11] STY: style changes --- tests/pipeline/test_filter.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index c2a6673d..6a33c251 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -19,7 +19,7 @@ from numpy import ( ones, ones_like, putmask, - sum, + sum as np_sum ) from numpy.random import randn, seed as random_seed @@ -401,21 +401,16 @@ class FilterTestCase(BasePipelineTestCase): def compute(self, today, assets, out, filter_): # sum for each column - out[:] = sum(filter_, axis=0) + out[:] = np_sum(filter_, axis=0) results = self.run_graph( TermGraph({'windowsafe': TestFactor()}), initial_workspace={InputFilter(): data}, ) - # number of securities in default_shape - securities = self.default_shape[1] - # number of days in default_shape n = self.default_shape[0] # shape of output array - output_shape = ((n - k + 1), securities) - - # assert that these two arrays are the same + output_shape = ((n - k + 1), self.default_shape[1]) check_arrays(results['windowsafe'], full(output_shape, k)) From be857ead0e135bf9e714746b1cb56d720f1088f0 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Sun, 24 Jul 2016 21:17:16 -0400 Subject: [PATCH 10/11] DOC: Clarify default window-safety for Filters. --- zipline/pipeline/filters/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index 8267b95e..be5d6ada 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -167,8 +167,8 @@ class Filter(RestrictedDTypeMixin, ComputableTerm): output of a Pipeline and for reducing memory consumption of Pipeline results. """ - - # make filters window safe + # Filters are window-safe by default, since a yes/no decision means the + # same thing from all temporal perspectives. window_safe = True ALLOWED_DTYPES = (bool_dtype,) # Used by RestrictedDTypeMixin From 3cc1cf078a5d4a11368b2ebe66fc8d12f344f5c6 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Sun, 24 Jul 2016 21:21:40 -0400 Subject: [PATCH 11/11] TEST: Parameterize over window_length. --- tests/pipeline/test_filter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/pipeline/test_filter.py b/tests/pipeline/test_filter.py index 6a33c251..c403e909 100644 --- a/tests/pipeline/test_filter.py +++ b/tests/pipeline/test_filter.py @@ -26,7 +26,7 @@ from numpy.random import randn, seed as random_seed from zipline.errors import BadPercentileBounds from zipline.pipeline import Filter, Factor, TermGraph from zipline.pipeline.factors import CustomFactor -from zipline.testing import check_arrays +from zipline.testing import check_arrays, parameter_space from zipline.utils.numpy_utils import float64_dtype from .base import BasePipelineTestCase, with_default_shape @@ -383,13 +383,11 @@ class FilterTestCase(BasePipelineTestCase): ) check_arrays(results['isfinite'], isfinite(data)) - def test_window_safe(self): + @parameter_space(factor_len=[2, 3, 4]) + def test_window_safe(self, factor_len): # all true data set of (days, securities) data = full(self.default_shape, True, dtype=bool) - # rolling window length for TestFactor - k = 8 - class InputFilter(Filter): inputs = () window_length = 0 @@ -397,7 +395,7 @@ class FilterTestCase(BasePipelineTestCase): class TestFactor(CustomFactor): dtype = float64_dtype inputs = (InputFilter(), ) - window_length = k + window_length = factor_len def compute(self, today, assets, out, filter_): # sum for each column @@ -412,5 +410,8 @@ class FilterTestCase(BasePipelineTestCase): n = self.default_shape[0] # shape of output array - output_shape = ((n - k + 1), self.default_shape[1]) - check_arrays(results['windowsafe'], full(output_shape, k)) + output_shape = ((n - factor_len + 1), self.default_shape[1]) + check_arrays( + results['windowsafe'], + full(output_shape, factor_len, dtype=float64) + )