wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-18 11:50:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

ENH: Implement Factor.quantiles.

Browse Source
This commit is contained in:
Scott Sanderson
2016-03-25 15:11:18 -04:00
parent 16c5aecba6
commit 872b84e09a
6 changed files with 261 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/pipeline/test_factor.py
+177 -11
View File
@@ -1,10 +1,11 @@
"""
Tests for Factor terms.
"""
from functools import partial
from itertools import product
from six import iteritems
from nose_parameterized import parameterized
from toolz import compose
from numpy import (
apply_along_axis,
arange,
@@ -13,9 +14,8 @@ from numpy import (
empty,
eye,
nan,
nanmean,
nanstd,
ones,
rot90,
where,
)
from numpy.random import randn, seed
@@ -32,6 +32,7 @@ from zipline.testing import (
check_allclose,
check_arrays,
parameter_space,
permute_rows,
)
from zipline.utils.functional import dzip_exact
from zipline.utils.numpy_utils import (
@@ -40,6 +41,7 @@ from zipline.utils.numpy_utils import (
int64_dtype,
NaTns,
)
from zipline.utils.math_utils import nanmean, nanstd
from .base import BasePipelineTestCase
@@ -50,6 +52,12 @@ class F(Factor):
window_length = 0
class OtherF(Factor):
dtype = float64_dtype
inputs = ()
window_length = 0
class C(Classifier):
dtype = int64_dtype
missing_value = -1
@@ -498,7 +506,7 @@ class FactorTestCase(BasePipelineTestCase):
mask=self.build_mask(self.ones_mask(shape=factor_data.shape)),
)
for key, (res, exp) in iteritems(dzip_exact(results, expected)):
for key, (res, exp) in dzip_exact(results, expected).items():
check_allclose(
res,
exp,
@@ -516,7 +524,7 @@ class FactorTestCase(BasePipelineTestCase):
('demean', lambda row: row - nanmean(row)),
('zscore', lambda row: (row - nanmean(row)) / nanstd(row)),
],
add_nulls_to_factor=(False, True,)
add_nulls_to_factor=(False, True,),
)
def test_normalizations_randomized(self,
seed_value,
@@ -532,9 +540,9 @@ class FactorTestCase(BasePipelineTestCase):
# Falses on main diagonal.
eyemask = self.eye_mask(shape=shape)
# Falses on other diagonal.
eyemask_T = eyemask.T
eyemask90 = rot90(eyemask)
# Falses on both diagonals.
xmask = eyemask & eyemask_T
xmask = eyemask & eyemask90
# Block of random data.
factor_data = self.randn_data(seed=seed_value, shape=shape)
@@ -548,7 +556,7 @@ class FactorTestCase(BasePipelineTestCase):
# With -1s on main diagonal.
classifier_data_eyenulls = where(eyemask, classifier_data, -1)
# With -1s on opposite diagonal.
classifier_data_eyenulls_T = where(eyemask_T, classifier_data, -1)
classifier_data_eyenulls90 = where(eyemask90, classifier_data, -1)
# With -1s on both diagonals.
classifier_data_xnulls = where(xmask, classifier_data, -1)
@@ -581,8 +589,8 @@ class FactorTestCase(BasePipelineTestCase):
# If the classifier has nulls, we should get NaNs in the
# corresponding locations in the output.
'grouped_with_nulls': where(
eyemask_T,
grouped_apply(factor_data, classifier_data_eyenulls_T, func),
eyemask90,
grouped_apply(factor_data, classifier_data_eyenulls90, func),
nan,
),
# Passing a mask with a classifier should behave as though the
@@ -613,7 +621,7 @@ class FactorTestCase(BasePipelineTestCase):
initial_workspace={
f: factor_data,
c: classifier_data,
c_with_nulls: classifier_data_eyenulls_T,
c_with_nulls: classifier_data_eyenulls90,
Mask(): eyemask,
},
mask=self.build_mask(nomask),
@@ -640,3 +648,161 @@ class FactorTestCase(BasePipelineTestCase):
).format(normalizer=method_name)
self.assertEqual(errmsg, expected)
@parameter_space(seed=[1, 2, 3])
def test_quantiles_unmasked(self, seed):
permute = partial(permute_rows, seed)
shape = (6, 6)
# Shuffle the input rows to verify that we don't depend on the order.
# Div by 2 to ensure that we don't depend on inputs being integral.
factor_data = permute(arange(36, dtype=float).reshape(shape)) / 2.0
f = self.f
terms = {
'2': f.quantiles(bins=2),
'3': f.quantiles(bins=3),
'6': f.quantiles(bins=6),
}
# Apply the same shuffle we applied to the input rows to our
# expectations. Doing it this way makes it obvious that our
# expectation corresponds to our input, while still testing against
# a range of input orderings.
permuted_array = compose(permute, partial(array, dtype=int))
expected = {
# The values in the input are all increasing, so the first half of
# each row should be in the bottom bucket, and the second half
# should be in the top bucket.
'2': permuted_array([[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]]),
# Similar for three buckets.
'3': permuted_array([[0, 0, 1, 1, 2, 2],
[0, 0, 1, 1, 2, 2],
[0, 0, 1, 1, 2, 2],
[0, 0, 1, 1, 2, 2],
[0, 0, 1, 1, 2, 2],
[0, 0, 1, 1, 2, 2]]),
# In the limiting case, we just have every column different.
'6': permuted_array([[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5],
[0, 1, 2, 3, 4, 5]]),
}
graph = TermGraph(terms)
results = self.run_graph(
graph,
initial_workspace={
f: factor_data,
},
mask=self.build_mask(self.ones_mask(shape=shape)),
)
for key, (res, exp) in dzip_exact(results, expected).items():
check_arrays(res, exp)
@parameter_space(seed=[1, 2, 3])
def test_quantiles_masked(self, seed):
permute = partial(permute_rows, seed)
# 7 x 7 so that we divide evenly into 2/3/6-tiles after including the
# nan value in each row.
shape = (7, 7)
# Shuffle the input rows to verify that we don't depend on the order.
# Div by 2 to ensure that we don't depend on inputs being integral.
factor_data = permute(arange(49, dtype=float).reshape(shape)) / 2.0
factor_data_w_nans = where(
permute(rot90(self.eye_mask(shape=shape))),
factor_data,
nan,
)
mask_data = permute(self.eye_mask(shape=shape))
f = F()
f_nans = OtherF()
m = Mask()
terms = {
'2_masked': f.quantiles(bins=2, mask=m),
'3_masked': f.quantiles(bins=3, mask=m),
'6_masked': f.quantiles(bins=6, mask=m),
'2_nans': f_nans.quantiles(bins=2),
'3_nans': f_nans.quantiles(bins=3),
'6_nans': f_nans.quantiles(bins=6),
}
# Apply the same shuffle we applied to the input rows to our
# expectations. Doing it this way makes it obvious that our
# expectation corresponds to our input, while still testing against
# a range of input orderings.
permuted_array = compose(permute, partial(array, dtype=int))
expected = {
# Expected results here are the same as in test_quantiles_masked,
# except with diagonals of -1s interpolated to match the effects of
# masking and/or input nans.
'2_masked': permuted_array([[-1, 0, 0, 0, 1, 1, 1],
[0, -1, 0, 0, 1, 1, 1],
[0, 0, -1, 0, 1, 1, 1],
[0, 0, 0, -1, 1, 1, 1],
[0, 0, 0, 1, -1, 1, 1],
[0, 0, 0, 1, 1, -1, 1],
[0, 0, 0, 1, 1, 1, -1]]),
'3_masked': permuted_array([[-1, 0, 0, 1, 1, 2, 2],
[0, -1, 0, 1, 1, 2, 2],
[0, 0, -1, 1, 1, 2, 2],
[0, 0, 1, -1, 1, 2, 2],
[0, 0, 1, 1, -1, 2, 2],
[0, 0, 1, 1, 2, -1, 2],
[0, 0, 1, 1, 2, 2, -1]]),
'6_masked': permuted_array([[-1, 0, 1, 2, 3, 4, 5],
[0, -1, 1, 2, 3, 4, 5],
[0, 1, -1, 2, 3, 4, 5],
[0, 1, 2, -1, 3, 4, 5],
[0, 1, 2, 3, -1, 4, 5],
[0, 1, 2, 3, 4, -1, 5],
[0, 1, 2, 3, 4, 5, -1]]),
'2_nans': permuted_array([[0, 0, 0, 1, 1, 1, -1],
[0, 0, 0, 1, 1, -1, 1],
[0, 0, 0, 1, -1, 1, 1],
[0, 0, 0, -1, 1, 1, 1],
[0, 0, -1, 0, 1, 1, 1],
[0, -1, 0, 0, 1, 1, 1],
[-1, 0, 0, 0, 1, 1, 1]]),
'3_nans': permuted_array([[0, 0, 1, 1, 2, 2, -1],
[0, 0, 1, 1, 2, -1, 2],
[0, 0, 1, 1, -1, 2, 2],
[0, 0, 1, -1, 1, 2, 2],
[0, 0, -1, 1, 1, 2, 2],
[0, -1, 0, 1, 1, 2, 2],
[-1, 0, 0, 1, 1, 2, 2]]),
'6_nans': permuted_array([[0, 1, 2, 3, 4, 5, -1],
[0, 1, 2, 3, 4, -1, 5],
[0, 1, 2, 3, -1, 4, 5],
[0, 1, 2, -1, 3, 4, 5],
[0, 1, -1, 2, 3, 4, 5],
[0, -1, 1, 2, 3, 4, 5],
[-1, 0, 1, 2, 3, 4, 5]]),
}
graph = TermGraph(terms)
results = self.run_graph(
graph,
initial_workspace={
f: factor_data,
f_nans: factor_data_w_nans,
m: mask_data,
},
mask=self.build_mask(self.ones_mask(shape=shape)),
)
for key, (res, exp) in dzip_exact(results, expected).items():
check_arrays(res, exp)
zipline/lib/quantiles.py
+17
View File
@@ -0,0 +1,17 @@
"""
Algorithms for computing quantiles on numpy arrays.
"""
from numpy.lib import apply_along_axis
from pandas import qcut
def quantiles(data, nbins_or_partition_bounds):
"""
Compute rowwise array quantiles on an input.
"""
return apply_along_axis(
qcut,
1,
data,
q=nbins_or_partition_bounds, labels=False,
)
zipline/pipeline/classifiers/__init__.py
+8 -1
View File
@@ -1,8 +1,15 @@
from .classifier import Classifier, CustomClassifier, Everything, Latest
from .classifier import (
Classifier,
CustomClassifier,
Quantiles,
Everything,
Latest,
)
__all__ = [
'Classifier',
'CustomClassifier',
'Everything',
'Latest',
'Quantiles',
]
zipline/pipeline/classifiers/classifier.py
+24 -2
View File
@@ -1,8 +1,9 @@
"""
classifier.py
"""
from numpy import zeros, where
from numpy import where, isnan, nan, zeros
from zipline.lib.quantiles import quantiles
from zipline.pipeline.term import ComputableTerm
from zipline.utils.numpy_utils import int64_dtype
@@ -10,7 +11,8 @@ from ..mixins import (
CustomTermMixin,
LatestMixin,
PositiveWindowLengthMixin,
RestrictedDTypeMixin
RestrictedDTypeMixin,
SingleInputMixin,
)
@@ -44,6 +46,26 @@ class Everything(Classifier):
)
class Quantiles(SingleInputMixin, Classifier):
"""
A classifier computing quantiles over an input.
"""
params = ('bins',)
dtype = int64_dtype
window_length = 0
missing_value = -1
def _compute(self, arrays, dates, assets, mask):
data = arrays[0]
bins = self.params['bins']
to_bin = where(mask, data, nan)
result = quantiles(to_bin, bins)
# Write self.missing_value into nan locations, whether they were
# generated by our input mask or not.
result[isnan(result)] = self.missing_value
return result.astype(int64_dtype)
class CustomClassifier(PositiveWindowLengthMixin, CustomTermMixin, Classifier):
"""
Base class for user-defined Classifiers.
zipline/pipeline/factors/factor.py
+30 -3
View File
@@ -5,13 +5,13 @@ from functools import wraps
from operator import attrgetter
from numbers import Number
from numpy import inf, where, nanstd
from numpy import inf, where
from toolz import curry
from zipline.errors import UnknownRankMethod
from zipline.lib.normalize import naive_grouped_rowwise_apply
from zipline.lib.rank import masked_rankdata_2d
from zipline.pipeline.classifiers import Classifier, Everything
from zipline.pipeline.classifiers import Classifier, Everything, Quantiles
from zipline.pipeline.mixins import (
CustomTermMixin,
LatestMixin,
@@ -43,7 +43,7 @@ from zipline.pipeline.filters import (
NullFilter,
)
from zipline.utils.input_validation import expect_types
from zipline.utils.math_utils import nanmean
from zipline.utils.math_utils import nanmean, nanstd
from zipline.utils.numpy_utils import (
bool_dtype,
coerce_to_dtype,
@@ -685,6 +685,33 @@ class Factor(RestrictedDTypeMixin, ComputableTerm):
"""
return Rank(self, method=method, ascending=ascending, mask=mask)
@expect_types(bins=int, mask=(Filter, NotSpecifiedType))
def quantiles(self, bins, mask=NotSpecified):
"""
Construct a Classifier computing quantiles of the output of ``self``.
Every non-NaN data point the output is labelled with an integer value
from 0 to (bins - 1). NaNs are labelled with -1.
If ``mask`` is supplied, ignore data points in locations for which
``mask`` produces False, and emit a label of -1 at those locations.
Parameters
----------
bins : int
Number of bins labels to compute.
mask : zipline.pipeline.Filter, optional
Mask of values to ignore when computing quantiles.
Returns
-------
quantiles : zipline.pipeline.classifiers.Quantiles
A Classifier producing integer labels ranging from 0 to (bins - 1).
"""
if mask is NotSpecified:
mask = self.mask
return Quantiles(inputs=(self,), bins=bins, mask=mask)
def top(self, N, mask=NotSpecified):
"""
Construct a Filter matching the top N asset values of self each day.
zipline/testing/core.py
+5 -4
View File
@@ -409,7 +409,7 @@ def make_trade_panel_for_asset_info(dates,
volume_step_by_date,
volume_step_by_sid):
"""
Convert an asset info frame into a panel of trades, writing NaNs for
locations where assets did not exist.
"""
sids = list(asset_info.index)
@@ -579,7 +579,7 @@ def check_allclose(actual,
)
def check_arrays(x, y, err_msg='', verbose=True):
def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True):
"""
Wrapper around np.testing.assert_array_equal that also verifies that inputs
are ndarrays.
@@ -588,8 +588,9 @@ def check_arrays(x, y, err_msg='', verbose=True):
--------
np.assert_array_equal
"""
if type(x) != type(y):
raise AssertionError("%s != %s" % (type(x), type(y)))
assert type(x) == type(y), "{x} != {y}".format(x=type(x), y=type(y))
assert x.dtype == y.dtype, "{x.dtype} != {y.dtype}".format(x=x, y=y)
return assert_array_equal(x, y, err_msg=err_msg, verbose=True)