From 872b84e09a1bb78ff1b7d53ad405188411b43283 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Mon, 21 Mar 2016 22:26:58 -0400 Subject: [PATCH] ENH: Implement Factor.quantiles. --- tests/pipeline/test_factor.py | 188 +++++++++++++++++++-- zipline/lib/quantiles.py | 17 ++ zipline/pipeline/classifiers/__init__.py | 9 +- zipline/pipeline/classifiers/classifier.py | 26 ++- zipline/pipeline/factors/factor.py | 33 +++- zipline/testing/core.py | 9 +- 6 files changed, 261 insertions(+), 21 deletions(-) create mode 100644 zipline/lib/quantiles.py diff --git a/tests/pipeline/test_factor.py b/tests/pipeline/test_factor.py index e7dab7cc..3a049bc1 100644 --- a/tests/pipeline/test_factor.py +++ b/tests/pipeline/test_factor.py @@ -1,10 +1,11 @@ """ Tests for Factor terms. """ +from functools import partial from itertools import product -from six import iteritems from nose_parameterized import parameterized +from toolz import compose from numpy import ( apply_along_axis, arange, @@ -13,9 +14,8 @@ from numpy import ( empty, eye, nan, - nanmean, - nanstd, ones, + rot90, where, ) from numpy.random import randn, seed @@ -32,6 +32,7 @@ from zipline.testing import ( check_allclose, check_arrays, parameter_space, + permute_rows, ) from zipline.utils.functional import dzip_exact from zipline.utils.numpy_utils import ( @@ -40,6 +41,7 @@ from zipline.utils.numpy_utils import ( int64_dtype, NaTns, ) +from zipline.utils.math_utils import nanmean, nanstd from .base import BasePipelineTestCase @@ -50,6 +52,12 @@ class F(Factor): window_length = 0 +class OtherF(Factor): + dtype = float64_dtype + inputs = () + window_length = 0 + + class C(Classifier): dtype = int64_dtype missing_value = -1 @@ -498,7 +506,7 @@ class FactorTestCase(BasePipelineTestCase): mask=self.build_mask(self.ones_mask(shape=factor_data.shape)), ) - for key, (res, exp) in iteritems(dzip_exact(results, expected)): + for key, (res, exp) in dzip_exact(results, expected).items(): check_allclose( res, exp, @@ -516,7 +524,7 @@ class FactorTestCase(BasePipelineTestCase): ('demean', lambda row: row - nanmean(row)), ('zscore', lambda row: (row - nanmean(row)) / nanstd(row)), ], - add_nulls_to_factor=(False, True,) + add_nulls_to_factor=(False, True,), ) def test_normalizations_randomized(self, seed_value, @@ -532,9 +540,9 @@ class FactorTestCase(BasePipelineTestCase): # Falses on main diagonal. eyemask = self.eye_mask(shape=shape) # Falses on other diagonal. - eyemask_T = eyemask.T + eyemask90 = rot90(eyemask) # Falses on both diagonals. - xmask = eyemask & eyemask_T + xmask = eyemask & eyemask90 # Block of random data. factor_data = self.randn_data(seed=seed_value, shape=shape) @@ -548,7 +556,7 @@ class FactorTestCase(BasePipelineTestCase): # With -1s on main diagonal. classifier_data_eyenulls = where(eyemask, classifier_data, -1) # With -1s on opposite diagonal. - classifier_data_eyenulls_T = where(eyemask_T, classifier_data, -1) + classifier_data_eyenulls90 = where(eyemask90, classifier_data, -1) # With -1s on both diagonals. classifier_data_xnulls = where(xmask, classifier_data, -1) @@ -581,8 +589,8 @@ class FactorTestCase(BasePipelineTestCase): # If the classifier has nulls, we should get NaNs in the # corresponding locations in the output. 'grouped_with_nulls': where( - eyemask_T, - grouped_apply(factor_data, classifier_data_eyenulls_T, func), + eyemask90, + grouped_apply(factor_data, classifier_data_eyenulls90, func), nan, ), # Passing a mask with a classifier should behave as though the @@ -613,7 +621,7 @@ class FactorTestCase(BasePipelineTestCase): initial_workspace={ f: factor_data, c: classifier_data, - c_with_nulls: classifier_data_eyenulls_T, + c_with_nulls: classifier_data_eyenulls90, Mask(): eyemask, }, mask=self.build_mask(nomask), @@ -640,3 +648,161 @@ class FactorTestCase(BasePipelineTestCase): ).format(normalizer=method_name) self.assertEqual(errmsg, expected) + + @parameter_space(seed=[1, 2, 3]) + def test_quantiles_unmasked(self, seed): + permute = partial(permute_rows, seed) + + shape = (6, 6) + + # Shuffle the input rows to verify that we don't depend on the order. + # Div by 2 to ensure that we don't depend on inputs being integral. + factor_data = permute(arange(36, dtype=float).reshape(shape)) / 2.0 + + f = self.f + terms = { + '2': f.quantiles(bins=2), + '3': f.quantiles(bins=3), + '6': f.quantiles(bins=6), + } + + # Apply the same shuffle we applied to the input rows to our + # expectations. Doing it this way makes it obvious that our + # expectation corresponds to our input, while still testing against + # a range of input orderings. + permuted_array = compose(permute, partial(array, dtype=int)) + expected = { + # The values in the input are all increasing, so the first half of + # each row should be in the bottom bucket, and the second half + # should be in the top bucket. + '2': permuted_array([[0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1]]), + # Similar for three buckets. + '3': permuted_array([[0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2]]), + # In the limiting case, we just have every column different. + '6': permuted_array([[0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5], + [0, 1, 2, 3, 4, 5]]), + } + + graph = TermGraph(terms) + results = self.run_graph( + graph, + initial_workspace={ + f: factor_data, + }, + mask=self.build_mask(self.ones_mask(shape=shape)), + ) + + for key, (res, exp) in dzip_exact(results, expected).items(): + check_arrays(res, exp) + + @parameter_space(seed=[1, 2, 3]) + def test_quantiles_masked(self, seed): + permute = partial(permute_rows, seed) + + # 7 x 7 so that we divide evenly into 2/3/6-tiles after including the + # nan value in each row. + shape = (7, 7) + + # Shuffle the input rows to verify that we don't depend on the order. + # Div by 2 to ensure that we don't depend on inputs being integral. + factor_data = permute(arange(49, dtype=float).reshape(shape)) / 2.0 + factor_data_w_nans = where( + permute(rot90(self.eye_mask(shape=shape))), + factor_data, + nan, + ) + mask_data = permute(self.eye_mask(shape=shape)) + + f = F() + f_nans = OtherF() + m = Mask() + + terms = { + '2_masked': f.quantiles(bins=2, mask=m), + '3_masked': f.quantiles(bins=3, mask=m), + '6_masked': f.quantiles(bins=6, mask=m), + '2_nans': f_nans.quantiles(bins=2), + '3_nans': f_nans.quantiles(bins=3), + '6_nans': f_nans.quantiles(bins=6), + } + + # Apply the same shuffle we applied to the input rows to our + # expectations. Doing it this way makes it obvious that our + # expectation corresponds to our input, while still testing against + # a range of input orderings. + permuted_array = compose(permute, partial(array, dtype=int)) + expected = { + # Expected results here are the same as in test_quantiles_masked, + # except with diagonals of -1s interpolated to match the effects of + # masking and/or input nans. + '2_masked': permuted_array([[-1, 0, 0, 0, 1, 1, 1], + [0, -1, 0, 0, 1, 1, 1], + [0, 0, -1, 0, 1, 1, 1], + [0, 0, 0, -1, 1, 1, 1], + [0, 0, 0, 1, -1, 1, 1], + [0, 0, 0, 1, 1, -1, 1], + [0, 0, 0, 1, 1, 1, -1]]), + '3_masked': permuted_array([[-1, 0, 0, 1, 1, 2, 2], + [0, -1, 0, 1, 1, 2, 2], + [0, 0, -1, 1, 1, 2, 2], + [0, 0, 1, -1, 1, 2, 2], + [0, 0, 1, 1, -1, 2, 2], + [0, 0, 1, 1, 2, -1, 2], + [0, 0, 1, 1, 2, 2, -1]]), + '6_masked': permuted_array([[-1, 0, 1, 2, 3, 4, 5], + [0, -1, 1, 2, 3, 4, 5], + [0, 1, -1, 2, 3, 4, 5], + [0, 1, 2, -1, 3, 4, 5], + [0, 1, 2, 3, -1, 4, 5], + [0, 1, 2, 3, 4, -1, 5], + [0, 1, 2, 3, 4, 5, -1]]), + '2_nans': permuted_array([[0, 0, 0, 1, 1, 1, -1], + [0, 0, 0, 1, 1, -1, 1], + [0, 0, 0, 1, -1, 1, 1], + [0, 0, 0, -1, 1, 1, 1], + [0, 0, -1, 0, 1, 1, 1], + [0, -1, 0, 0, 1, 1, 1], + [-1, 0, 0, 0, 1, 1, 1]]), + '3_nans': permuted_array([[0, 0, 1, 1, 2, 2, -1], + [0, 0, 1, 1, 2, -1, 2], + [0, 0, 1, 1, -1, 2, 2], + [0, 0, 1, -1, 1, 2, 2], + [0, 0, -1, 1, 1, 2, 2], + [0, -1, 0, 1, 1, 2, 2], + [-1, 0, 0, 1, 1, 2, 2]]), + '6_nans': permuted_array([[0, 1, 2, 3, 4, 5, -1], + [0, 1, 2, 3, 4, -1, 5], + [0, 1, 2, 3, -1, 4, 5], + [0, 1, 2, -1, 3, 4, 5], + [0, 1, -1, 2, 3, 4, 5], + [0, -1, 1, 2, 3, 4, 5], + [-1, 0, 1, 2, 3, 4, 5]]), + } + + graph = TermGraph(terms) + results = self.run_graph( + graph, + initial_workspace={ + f: factor_data, + f_nans: factor_data_w_nans, + m: mask_data, + }, + mask=self.build_mask(self.ones_mask(shape=shape)), + ) + + for key, (res, exp) in dzip_exact(results, expected).items(): + check_arrays(res, exp) diff --git a/zipline/lib/quantiles.py b/zipline/lib/quantiles.py new file mode 100644 index 00000000..470153d0 --- /dev/null +++ b/zipline/lib/quantiles.py @@ -0,0 +1,17 @@ +""" +Algorithms for computing quantiles on numpy arrays. +""" +from numpy.lib import apply_along_axis +from pandas import qcut + + +def quantiles(data, nbins_or_partition_bounds): + """ + Compute rowwise array quantiles on an input. + """ + return apply_along_axis( + qcut, + 1, + data, + q=nbins_or_partition_bounds, labels=False, + ) diff --git a/zipline/pipeline/classifiers/__init__.py b/zipline/pipeline/classifiers/__init__.py index b512100f..cc2e242b 100644 --- a/zipline/pipeline/classifiers/__init__.py +++ b/zipline/pipeline/classifiers/__init__.py @@ -1,8 +1,15 @@ -from .classifier import Classifier, CustomClassifier, Everything, Latest +from .classifier import ( + Classifier, + CustomClassifier, + Quantiles, + Everything, + Latest, +) __all__ = [ 'Classifier', 'CustomClassifier', 'Everything', 'Latest', + 'Quantiles', ] diff --git a/zipline/pipeline/classifiers/classifier.py b/zipline/pipeline/classifiers/classifier.py index 6777e03e..0d6db642 100644 --- a/zipline/pipeline/classifiers/classifier.py +++ b/zipline/pipeline/classifiers/classifier.py @@ -1,8 +1,9 @@ """ classifier.py """ -from numpy import zeros, where +from numpy import where, isnan, nan, zeros +from zipline.lib.quantiles import quantiles from zipline.pipeline.term import ComputableTerm from zipline.utils.numpy_utils import int64_dtype @@ -10,7 +11,8 @@ from ..mixins import ( CustomTermMixin, LatestMixin, PositiveWindowLengthMixin, - RestrictedDTypeMixin + RestrictedDTypeMixin, + SingleInputMixin, ) @@ -44,6 +46,26 @@ class Everything(Classifier): ) +class Quantiles(SingleInputMixin, Classifier): + """ + A classifier computing quantiles over an input. + """ + params = ('bins',) + dtype = int64_dtype + window_length = 0 + missing_value = -1 + + def _compute(self, arrays, dates, assets, mask): + data = arrays[0] + bins = self.params['bins'] + to_bin = where(mask, data, nan) + result = quantiles(to_bin, bins) + # Write self.missing_value into nan locations, whether they were + # generated by our input mask or not. + result[isnan(result)] = self.missing_value + return result.astype(int64_dtype) + + class CustomClassifier(PositiveWindowLengthMixin, CustomTermMixin, Classifier): """ Base class for user-defined Classifiers. diff --git a/zipline/pipeline/factors/factor.py b/zipline/pipeline/factors/factor.py index b9054e5d..774ed7e4 100644 --- a/zipline/pipeline/factors/factor.py +++ b/zipline/pipeline/factors/factor.py @@ -5,13 +5,13 @@ from functools import wraps from operator import attrgetter from numbers import Number -from numpy import inf, where, nanstd +from numpy import inf, where from toolz import curry from zipline.errors import UnknownRankMethod from zipline.lib.normalize import naive_grouped_rowwise_apply from zipline.lib.rank import masked_rankdata_2d -from zipline.pipeline.classifiers import Classifier, Everything +from zipline.pipeline.classifiers import Classifier, Everything, Quantiles from zipline.pipeline.mixins import ( CustomTermMixin, LatestMixin, @@ -43,7 +43,7 @@ from zipline.pipeline.filters import ( NullFilter, ) from zipline.utils.input_validation import expect_types -from zipline.utils.math_utils import nanmean +from zipline.utils.math_utils import nanmean, nanstd from zipline.utils.numpy_utils import ( bool_dtype, coerce_to_dtype, @@ -685,6 +685,33 @@ class Factor(RestrictedDTypeMixin, ComputableTerm): """ return Rank(self, method=method, ascending=ascending, mask=mask) + @expect_types(bins=int, mask=(Filter, NotSpecifiedType)) + def quantiles(self, bins, mask=NotSpecified): + """ + Construct a Classifier computing quantiles of the output of ``self``. + + Every non-NaN data point the output is labelled with an integer value + from 0 to (bins - 1). NaNs are labelled with -1. + + If ``mask`` is supplied, ignore data points in locations for which + ``mask`` produces False, and emit a label of -1 at those locations. + + Parameters + ---------- + bins : int + Number of bins labels to compute. + mask : zipline.pipeline.Filter, optional + Mask of values to ignore when computing quantiles. + + Returns + ------- + quantiles : zipline.pipeline.classifiers.Quantiles + A Classifier producing integer labels ranging from 0 to (bins - 1). + """ + if mask is NotSpecified: + mask = self.mask + return Quantiles(inputs=(self,), bins=bins, mask=mask) + def top(self, N, mask=NotSpecified): """ Construct a Filter matching the top N asset values of self each day. diff --git a/zipline/testing/core.py b/zipline/testing/core.py index c759480b..3d96c4ec 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -409,7 +409,7 @@ def make_trade_panel_for_asset_info(dates, volume_step_by_date, volume_step_by_sid): """ - Convert an asset info frame into a panel of trades, writing NaNs for + locations where assets did not exist. """ sids = list(asset_info.index) @@ -579,7 +579,7 @@ def check_allclose(actual, ) -def check_arrays(x, y, err_msg='', verbose=True): +def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True): """ Wrapper around np.testing.assert_array_equal that also verifies that inputs are ndarrays. @@ -588,8 +588,9 @@ def check_arrays(x, y, err_msg='', verbose=True): -------- np.assert_array_equal """ - if type(x) != type(y): - raise AssertionError("%s != %s" % (type(x), type(y))) + assert type(x) == type(y), "{x} != {y}".format(x=type(x), y=type(y)) + assert x.dtype == y.dtype, "{x.dtype} != {y.dtype}".format(x=x, y=y) + return assert_array_equal(x, y, err_msg=err_msg, verbose=True)