diff --git a/tests/pipeline/test_factor.py b/tests/pipeline/test_factor.py index 2d81dc18..46245222 100644 --- a/tests/pipeline/test_factor.py +++ b/tests/pipeline/test_factor.py @@ -22,8 +22,9 @@ from numpy import ( ) from numpy.random import randn, seed import pandas as pd +from scipy.stats.mstats import winsorize as scipy_winsorize -from zipline.errors import UnknownRankMethod +from zipline.errors import BadPercentileBounds, UnknownRankMethod from zipline.lib.labelarray import LabelArray from zipline.lib.rank import masked_rankdata_2d from zipline.lib.normalize import naive_grouped_rowwise_apply as grouped_apply @@ -709,11 +710,157 @@ class FactorTestCase(BasePipelineTestCase): check=partial(check_allclose, atol=0.001), ) + def test_winsorize_hand_computed(self): + """ + Test the hand-computed example in factor.winsorize. + """ + f = self.f + m = Mask() + c = C() + str_c = C(dtype=categorical_dtype, missing_value=None) + + factor_data = array([ + [1., 2., 3., 4., 5., 6.], + [1., 8., 27., 64., 125., 216.], + [6., 5., 4., 3., 2., 1.] + ]) + filter_data = array( + [[False, True, True, True, True, True], + [True, False, True, True, True, True], + [True, True, False, True, True, True]], + dtype=bool, + ) + classifier_data = array( + [[1, 1, 1, 2, 2, 2], + [1, 1, 1, 2, 2, 2], + [1, 1, 1, 2, 2, 2]], + dtype=int64_dtype, + ) + string_classifier_data = LabelArray( + classifier_data.astype(str).astype(object), + missing_value=None, + ) + + terms = { + 'winsor_1': f.winsorize( + min_percentile=0.33, + max_percentile=0.67 + ), + 'winsor_2': f.winsorize( + min_percentile=0.49, + max_percentile=1 + ), + 'winsor_3': f.winsorize( + min_percentile=0, + max_percentile=.67 + ), + 'masked': f.winsorize( + min_percentile=0.33, + max_percentile=0.67, + mask=m + ), + 'grouped': f.winsorize( + min_percentile=0.34, + max_percentile=0.66, + groupby=c + ), + 'grouped_str': f.winsorize( + min_percentile=0.34, + max_percentile=0.66, + groupby=str_c + ), + 'grouped_masked': f.winsorize( + min_percentile=0.34, + max_percentile=0.66, + mask=m, + groupby=c + ), + 'grouped_masked_str': f.winsorize( + min_percentile=0.34, + max_percentile=0.66, + mask=m, + groupby=str_c + ), + } + expected = { + 'winsor_1': array([ + [2., 2., 3., 4., 5., 5.], + [8., 8., 27., 64., 125., 125.], + [5., 5., 4., 3., 2., 2.] + ]), + 'winsor_2': array([ + [3.0, 3., 3., 4., 5., 6.], + [27., 27., 27., 64., 125., 216.], + [6.0, 5., 4., 3., 3., 3.] + ]), + 'winsor_3': array([ + [1., 2., 3., 4., 5., 5.], + [1., 8., 27., 64., 125., 125.], + [5., 5., 4., 3., 2., 1.] + ]), + 'masked': array([ + [nan, 3., 3., 4., 5., 5.], + [27., nan, 27., 64., 125., 125.], + [5.0, 5., nan, 3., 2., 2.] + ]), + 'grouped': array([ + [2., 2., 2., 5., 5., 5.], + [8., 8., 8., 125., 125., 125.], + [5., 5., 5., 2., 2., 2.] + ]), + 'grouped_masked': array([ + [nan, 2., 3., 5., 5., 5.], + [1.0, nan, 27., 125., 125., 125.], + [6.0, 5., nan, 2., 2., 2.] + ]), + } + # Changing the classifier dtype shouldn't affect anything. + expected['grouped_str'] = expected['grouped'] + expected['grouped_masked_str'] = expected['grouped_masked'] + + self.check_terms( + terms, + expected, + initial_workspace={ + f: factor_data, + c: classifier_data, + str_c: string_classifier_data, + m: filter_data, + }, + mask=self.build_mask(self.ones_mask(shape=factor_data.shape)), + check=partial(check_allclose, atol=0.001), + ) + + def test_winsorize_bad_bounds(self): + """ + Test out of bounds input for factor.winsorize. + """ + f = self.f + + bad_percentiles = [ + (-.1, 1), + (0, 95), + (5, 95), + (5, 5), + (.6, .4) + ] + for min_, max_ in bad_percentiles: + with self.assertRaises(BadPercentileBounds): + f.winsorize(min_percentile=min_, max_percentile=max_) + @parameter_space( seed_value=range(1, 2), normalizer_name_and_func=[ - ('demean', lambda row: row - nanmean(row)), - ('zscore', lambda row: (row - nanmean(row)) / nanstd(row)), + ('demean', {}, lambda row: row - nanmean(row)), + ('zscore', {}, lambda row: (row - nanmean(row)) / nanstd(row)), + ( + 'winsorize', + {"min_percentile": 0.25, "max_percentile": 0.75}, + lambda row: scipy_winsorize( + row, + limits=0.25, + ) + ), ], add_nulls_to_factor=(False, True,), ) @@ -722,9 +869,9 @@ class FactorTestCase(BasePipelineTestCase): normalizer_name_and_func, add_nulls_to_factor): - name, func = normalizer_name_and_func + name, kwargs, func = normalizer_name_and_func - shape = (7, 7) + shape = (20, 20) # All Trues. nomask = self.ones_mask(shape=shape) @@ -755,7 +902,7 @@ class FactorTestCase(BasePipelineTestCase): c = C() c_with_nulls = OtherC() m = Mask() - method = getattr(f, name) + method = partial(getattr(f, name), **kwargs) terms = { 'vanilla': method(), 'masked': method(mask=m), @@ -1051,6 +1198,10 @@ class ShortReprTestCase(TestCase): r = F().zscore().short_repr() self.assertEqual(r, "GroupedRowTransform('zscore')") + def test_winsorize(self): + r = F().winsorize(min_percentile=.05, max_percentile=.95).short_repr() + self.assertEqual(r, "GroupedRowTransform('winsorize')") + class TestWindowSafety(TestCase): @@ -1062,6 +1213,23 @@ class TestWindowSafety(TestCase): self.assertFalse(F(window_safe=False).demean().window_safe) self.assertTrue(F(window_safe=True).demean().window_safe) + def test_winsorize_is_window_safe_if_input_is_window_safe(self): + self.assertFalse( + F().winsorize(min_percentile=.05, max_percentile=.95).window_safe + ) + self.assertFalse( + F(window_safe=False).winsorize( + min_percentile=.05, + max_percentile=.95 + ).window_safe + ) + self.assertTrue( + F(window_safe=True).winsorize( + min_percentile=.05, + max_percentile=.95 + ).window_safe + ) + class TestPostProcessAndToWorkSpaceValue(ZiplineTestCase): @parameter_space(dtype_=(float64_dtype, datetime64ns_dtype)) diff --git a/zipline/errors.py b/zipline/errors.py index 656f554e..17626df3 100644 --- a/zipline/errors.py +++ b/zipline/errors.py @@ -570,8 +570,8 @@ class BadPercentileBounds(ZiplineError): are invalid. """ msg = ( - "Percentile bounds must fall between 0.0 and 100.0, and min must be " - "less than max." + "Percentile bounds must fall between 0.0 and {upper_bound}, and min " + "must be less than max." "\nInputs were min={min_percentile}, max={max_percentile}." ) diff --git a/zipline/pipeline/factors/factor.py b/zipline/pipeline/factors/factor.py index 793282d1..9b552ef9 100644 --- a/zipline/pipeline/factors/factor.py +++ b/zipline/pipeline/factors/factor.py @@ -4,11 +4,12 @@ factor.py from functools import wraps from operator import attrgetter from numbers import Number +from math import ceil from numpy import empty_like, inf, nan, where from scipy.stats import rankdata -from zipline.errors import UnknownRankMethod +from zipline.errors import BadPercentileBounds, UnknownRankMethod from zipline.lib.normalize import naive_grouped_rowwise_apply from zipline.lib.rank import masked_rankdata_2d, rankdata_1d_descending from zipline.pipeline.api_utils import restrict_to_dtype @@ -833,6 +834,104 @@ class Factor(RestrictedDTypeMixin, ComputableTerm): mask=mask, ) + @expect_types( + min_percentile=(int, float), + max_percentile=(int, float), + mask=(Filter, NotSpecifiedType), + groupby=(Classifier, NotSpecifiedType), + ) + @float64_only + def winsorize(self, + min_percentile, + max_percentile, + mask=NotSpecified, + groupby=NotSpecified): + """ + Construct a Factor returns a winsorized row. Winsorizing changes values + ranked less than the minimum percentile to to value at the minimum + percentile. Similarly, values ranking above the maximum percentile will + be changed to the value at the maximum percentile. This is useful + when limiting the impact of extreme values. + + If ``mask`` is supplied, ignore values where ``mask`` returns False + when computing row means and standard deviations, and output NaN + anywhere the mask is False. + + If ``groupby`` is supplied, compute by partitioning each row based on + the values produced by ``groupby``, winsorizing the partitioned arrays, + and stitching the sub-results back together. + + Parameters + ---------- + min_percentile: float, int + Entries with values at or below this percentile will be replaced + with the (len(inp) * min_percentile)th lowest value. If low values + should not be clipped, use 0. + max_percentile: float, int + Entries with values at or above this percentile will be replaced + with the (len(inp) * max_percentile)th lowest value. If high + values should not be clipped, use 1. + mask : zipline.pipeline.Filter, optional + A Filter defining values to ignore when winsorizing. + groupby : zipline.pipeline.Classifier, optional + A classifier defining partitions over which to winsorize. + + Returns + ------- + winsorized : zipline.pipeline.Factor + A Factor producing a winsorized version of self. + + Example + ------- + + price = USEquityPricing.close.latest + columns={ + 'PRICE': price, + 'WINSOR_1: price.winsorize( + min_percentile=0.25, max_percentile=0.75 + ), + 'WINSOR_2': price.winsorize( + min_percentile=0.50, max_percentile=1.0 + ), + 'WINSOR_3': price.winsorize( + min_percentile=0.0, max_percentile=0.5 + ), + + } + + Given a pipeline with columns, defined above, the result for a + given day could look like: + + 'PRICE' 'WINSOR_1' 'WINSOR_2' 'WINSOR_3' + Asset_1 1 2 4 3 + Asset_2 2 2 4 3 + Asset_3 3 3 4 3 + Asset_4 4 4 4 4 + Asset_5 5 5 5 4 + Asset_6 6 5 5 4 + + See Also + -------- + :func:`scipy.stats.mstats.winsorize` + :meth:`pandas.DataFrame.groupby` + """ + if not 0.0 <= min_percentile < max_percentile <= 1.0: + raise BadPercentileBounds( + min_percentile=min_percentile, + max_percentile=max_percentile, + upper_bound=1.0, + ) + return GroupedRowTransform( + transform=winsorize, + transform_args=(min_percentile, max_percentile), + factor=self, + groupby=groupby, + dtype=self.dtype, + missing_value=self.missing_value, + mask=mask, + window_safe=self.window_safe, + ) + @expect_types(bins=int, mask=(Filter, NotSpecifiedType)) def quantiles(self, bins, mask=NotSpecified): """ @@ -1530,3 +1629,23 @@ def demean(row): def zscore(row): return (row - nanmean(row)) / nanstd(row) + + +def winsorize(row, min_percentile, max_percentile): + """ + This implementation is based on scipy.stats.mstats.winsorize + """ + a = row.copy() + num = a.size + idx = a.argsort() + if min_percentile > 0: + lowidx = int(min_percentile * num) + a[idx[:lowidx]] = a[idx[lowidx]] + if max_percentile < 1: + upidx = ceil(num * max_percentile) + # upidx could return as the length of the array, in this case + # no modification to the right tail is necessary. + if upidx < num: + a[idx[upidx:]] = a[idx[upidx - 1]] + + return a diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index 864e18f9..b4e3809d 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -334,6 +334,7 @@ class PercentileFilter(SingleInputMixin, Filter): raise BadPercentileBounds( min_percentile=self._min_percentile, max_percentile=self._max_percentile, + upper_bound=100.0 ) return super(PercentileFilter, self)._validate()