wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-09 05:32:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

ENH: NaN semantics for LabelArray missing values.

Browse Source
This commit is contained in:
Scott Sanderson
2016-05-03 15:51:50 -04:00
parent 2395cbb671
commit 8de45540f2
6 changed files with 190 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_labelarray.py
+85 -21
View File
@@ -33,7 +33,7 @@ class LabelArrayTestCase(ZiplineTestCase):
super(LabelArrayTestCase, cls).init_class_fixtures()
cls.rowvalues = row = ['', 'a', 'b', 'ab', 'a', '', 'b', 'ab', 'z']
cls.strs = np.array([rotN(row, i) for i in range(3)])
cls.strs = np.array([rotN(row, i) for i in range(3)], dtype=object)
def test_fail_on_direct_construction(self):
# See http://docs.scipy.org/doc/numpy-1.10.0/user/basics.subclassing.html#simple-example-adding-an-extra-attribute-to-ndarray # noqa
@@ -48,32 +48,82 @@ class LabelArrayTestCase(ZiplineTestCase):
@parameter_space(
__fail_fast=True,
s=['', 'a', 'z', 'aa', 'not in the array'],
shape=[(27,), (9, 3), (3, 9), (3, 3, 3)],
compval=['', 'a', 'z', 'not in the array'],
shape=[(27,), (3, 9), (3, 3, 3)],
array_astype=(bytes, unicode, object),
scalar_astype=(bytes, unicode, object),
missing_value=('', 'a', 'not in the array', None),
)
def test_compare_to_str(self, s, shape, array_astype, scalar_astype):
def test_compare_to_str(self,
compval,
shape,
array_astype,
missing_value):
strs = self.strs.reshape(shape).astype(array_astype)
arr = LabelArray(strs, missing_value='')
check_arrays(strs == s, arr == s)
check_arrays(strs != s, arr != s)
if missing_value is None:
# As of numpy 1.9.2, object array != None returns just False
# instead of an array, with a deprecation warning saying the
# behavior will change in the future. Work around that by just
# using the ufunc.
notmissing = np.not_equal(strs, missing_value)
else:
notmissing = (strs != missing_value)
np_startswith = np.vectorize(lambda elem: elem.startswith(s))
check_arrays(arr.startswith(s), np_startswith(strs))
arr = LabelArray(strs, missing_value=missing_value)
np_endswith = np.vectorize(lambda elem: elem.endswith(s))
check_arrays(arr.endswith(s), np_endswith(strs))
# arr.missing_value should behave like NaN.
check_arrays(
arr == compval,
(strs == compval) & notmissing,
)
check_arrays(
arr != compval,
(strs != compval) & notmissing,
)
np_contains = np.vectorize(lambda elem: s in elem)
check_arrays(arr.has_substring(s), np_contains(strs))
np_startswith = np.vectorize(lambda elem: elem.startswith(compval))
check_arrays(
arr.startswith(compval),
np_startswith(strs) & notmissing,
)
def test_compare_to_str_array(self):
np_endswith = np.vectorize(lambda elem: elem.endswith(compval))
check_arrays(
arr.endswith(compval),
np_endswith(strs) & notmissing,
)
np_contains = np.vectorize(lambda elem: compval in elem)
check_arrays(
arr.has_substring(compval),
np_contains(strs) & notmissing,
)
@parameter_space(
__fail_fast=True,
missing_value=('', 'a', 'not in the array', None),
)
def test_compare_to_str_array(self, missing_value):
strs = self.strs
shape = strs.shape
arr = LabelArray(strs, missing_value='')
check_arrays(strs == arr, np.full_like(strs, True, dtype=bool))
check_arrays(strs != arr, np.full_like(strs, False, dtype=bool))
arr = LabelArray(strs, missing_value=missing_value)
if missing_value is None:
# As of numpy 1.9.2, object array != None returns just False
# instead of an array, with a deprecation warning saying the
# behavior will change in the future. Work around that by just
# using the ufunc.
notmissing = np.not_equal(strs, missing_value)
else:
notmissing = (strs != missing_value)
check_arrays(arr.not_missing(), notmissing)
check_arrays(arr.is_missing(), ~notmissing)
# The arrays are equal everywhere, but comparisons against the
# missing_value should always produce False
check_arrays(strs == arr, notmissing)
check_arrays(strs != arr, np.zeros_like(strs, dtype=bool))
def broadcastable_row(value, dtype):
return np.full((shape[0], 1), value, dtype=strs.dtype)
@@ -81,20 +131,22 @@ class LabelArrayTestCase(ZiplineTestCase):
def broadcastable_col(value, dtype):
return np.full((1, shape[1]), value, dtype=strs.dtype)
# Test comparison between arr and a like-shap 2D array, a column
# vector, and a row vector.
for comparator, dtype, value in product((eq, ne),
(bytes, unicode, object),
set(self.rowvalues)):
check_arrays(
comparator(arr, np.full_like(strs, value)),
comparator(strs, value),
comparator(strs, value) & notmissing,
)
check_arrays(
comparator(arr, broadcastable_row(value, dtype=dtype)),
comparator(strs, value),
comparator(strs, value) & notmissing,
)
check_arrays(
comparator(arr, broadcastable_col(value, dtype=dtype)),
comparator(strs, value),
comparator(strs, value) & notmissing,
)
@parameter_space(
@@ -122,6 +174,10 @@ class LabelArrayTestCase(ZiplineTestCase):
self.assertIs(sliced.missing_value, arr.missing_value)
def test_infer_categories(self):
"""
Test that categories are inferred in sorted order if they're not
explicitly passed.
"""
arr1d = LabelArray(self.strs, missing_value='')
codes1d = arr1d.as_int_array()
self.assertEqual(arr1d.shape, self.strs.shape)
@@ -144,6 +200,14 @@ class LabelArrayTestCase(ZiplineTestCase):
arr1d.as_int_array() == idx,
)
# It should be equivalent to pass the same set of categories manually.
arr1d_explicit_categories = LabelArray(
self.strs,
missing_value='',
categories=arr1d.categories,
)
check_arrays(arr1d, arr1d_explicit_categories)
for shape in (9, 3), (3, 9), (3, 3, 3):
strs2d = self.strs.reshape(shape)
arr2d = LabelArray(strs2d, missing_value='')
zipline/lib/adjusted_array.py
+2 -3
View File
@@ -18,7 +18,6 @@ from zipline.errors import (
WindowLengthTooLong,
)
from zipline.lib.labelarray import LabelArray
from zipline.utils.compat import unicode
from zipline.utils.numpy_utils import (
datetime64ns_dtype,
float64_dtype,
@@ -111,10 +110,10 @@ def _normalize_array(data, missing_value):
elif data_dtype in INT_DTYPES:
return data.astype(int64), {'dtype': dtype(int64)}
elif is_categorical(data_dtype):
if not isinstance(missing_value, (bytes, unicode)):
if not isinstance(missing_value, LabelArray.SUPPORTED_SCALAR_TYPES):
raise TypeError(
"Invalid missing_value for categorical array.\n"
"Expected bytes or unicode. Got %r." % missing_value,
"Expected None, bytes or unicode. Got %r." % missing_value,
)
return LabelArray(data, missing_value), {}
elif data_dtype.kind == 'M':
zipline/lib/labelarray.py
+84 -42
View File
@@ -2,7 +2,6 @@
An ndarray subclass for working with arrays of strings.
"""
from functools import partial
from numbers import Number
from operator import eq, ne
import re
@@ -19,7 +18,11 @@ from zipline.utils.input_validation import (
expect_types,
optional,
)
from zipline.utils.numpy_utils import int_dtype_with_size_in_bytes, is_object
from zipline.utils.numpy_utils import (
bool_dtype,
int_dtype_with_size_in_bytes,
is_object,
)
from ._factorize import (
factorize_strings,
@@ -45,6 +48,18 @@ def _make_unsupported_method(name):
return method
class MissingValueMismatch(ValueError):
"""
Error raised on attempt to perform operations between LabelArrays with
mismatched missing_values.
"""
def __init__(self, left, right):
super(MissingValueMismatch, self).__init__(
"LabelArray missing_values don't match:"
" left={}, right={}".format(left, right)
)
class CategoryMismatch(ValueError):
"""
Error raised on attempt to perform operations between LabelArrays with
@@ -95,7 +110,7 @@ class LabelArray(ndarray):
reverse_categories : dict[str -> int]
Reverse lookup table for ``categories``. Stores the index in
``categories`` at which each entry each unique entry is found.
missing_value : str
missing_value : str or None
A sentinel missing value with NaN semantics for comparisons.
Notes
@@ -117,11 +132,15 @@ class LabelArray(ndarray):
--------
http://docs.scipy.org/doc/numpy-1.10.0/user/basics.subclassing.html
"""
SUPPORTED_SCALAR_TYPES = (bytes, unicode, type(None))
@preprocess(
values=coerce(list, partial(np.asarray, dtype=object)),
categories=coerce(np.ndarray, list),
)
@expect_types(
values=np.ndarray,
missing_value=SUPPORTED_SCALAR_TYPES,
categories=optional(list),
)
@expect_kinds(values=("O", "S", "U"))
@@ -301,9 +320,9 @@ class LabelArray(ndarray):
else:
raise CategoryMismatch(self_categories, value_categories)
elif isinstance(value, (bytes, unicode)):
value_code = self.reverse_categories.get(value, None)
if value_code is None:
elif isinstance(value, self.SUPPORTED_SCALAR_TYPES):
value_code = self.reverse_categories.get(value, -1)
if value_code < 0:
raise ValueError("%r is not in LabelArray categories." % value)
self.as_int_array()[indexer] = value_code
else:
@@ -314,47 +333,54 @@ class LabelArray(ndarray):
),
)
def is_missing(self):
"""
Like isnan, but checks for locations where we store missing values.
"""
return (
self.as_int_array() == self.reverse_categories[self.missing_value]
)
def not_missing(self):
"""
Like ~isnan, but checks for locations where we store missing values.
"""
return (
self.as_int_array() != self.reverse_categories[self.missing_value]
)
def _equality_check(op):
"""
Shared code for __eq__ and __ne__, parameterized on the actual
comparison operator to use.
"""
# What value should we return if we compare against a value not in our
# categories?
if op is eq:
COMPARE_TO_UNKNOWN = False
elif op is ne:
COMPARE_TO_UNKNOWN = True
else:
raise AssertionError("_make_equality_check called with %s" % op)
def method(self, other):
self_categories = self.categories
if isinstance(other, LabelArray):
self_mv = self.missing_value
other_mv = other.missing_value
if self_mv != other_mv:
raise MissingValueMismatch(self_mv, other_mv)
self_categories = self.categories
other_categories = other.categories
if compare_arrays(self_categories, other_categories):
return op(self.as_int_array(), other.as_int_array())
else:
if not compare_arrays(self_categories, other_categories):
raise CategoryMismatch(self_categories, other_categories)
return (
op(self.as_int_array(), other.as_int_array())
& self.not_missing()
& other.not_missing()
)
elif isinstance(other, ndarray):
# Compare to ndarrays as though we were an array of strings.
# This is fairly expensive, and should generally be avoided.
return op(self.as_string_array(), other)
return op(self.as_string_array(), other) & self.not_missing()
elif isinstance(other, (bytes, unicode)):
i = self._reverse_categories.get(other, None)
if i is None:
# Requested string isn't in our categories. Short circuit.
# This isn't full_like because that would try to return a
# LabelArray.
return np.full(self.shape, COMPARE_TO_UNKNOWN, dtype=bool)
return op(self.as_int_array(), i)
elif isinstance(other, Number):
return NotImplemented
elif isinstance(other, self.SUPPORTED_SCALAR_TYPES):
i = self._reverse_categories.get(other, -1)
return op(self.as_int_array(), i) & self.not_missing()
return op(super(LabelArray, self), other)
return method
@@ -450,14 +476,30 @@ class LabelArray(ndarray):
missing_value=self.missing_value,
)
def apply(self, f, dtype):
def map_predicate(self, f):
"""
Map a function elementwise over entries in ``self``.
Map a function from str -> bool element-wise over ``self``.
``f`` will be applied exactly once to each unique value in ``self``.
``f`` will be applied exactly once to each non-missing unique value in
``self``. Missing values will always return False.
"""
vf = np.vectorize(f, otypes=[dtype])
return vf(self.categories)[self.as_int_array()]
# Functions passed to this are of type str -> bool. Don't ever call
# them on None, which is the only non-str value we ever store in
# categories.
if self.missing_value is None:
f_to_use = lambda x: False if x is None else f(x)
else:
f_to_use = f
# Call f on each unique value in our categories.
results = np.vectorize(f_to_use, otypes=[bool_dtype])(self.categories)
# missing_value should produce False no matter what
results[self.reverse_categories[self.missing_value]] = False
# unpack the results form each unique value into their corresponding
# locations in our indices.
return results[self.as_int_array()]
def startswith(self, prefix):
"""
@@ -473,7 +515,7 @@ class LabelArray(ndarray):
An array with the same shape as self indicating whether each
element of self started with ``prefix``.
"""
return self.apply(lambda elem: elem.startswith(prefix), dtype=bool)
return self.map_predicate(lambda elem: elem.startswith(prefix))
def endswith(self, suffix):
"""
@@ -489,7 +531,7 @@ class LabelArray(ndarray):
An array with the same shape as self indicating whether each
element of self ended with ``suffix``
"""
return self.apply(lambda elem: elem.endswith(suffix), dtype=bool)
return self.map_predicate(lambda elem: elem.endswith(suffix))
def has_substring(self, substring):
"""
@@ -505,7 +547,7 @@ class LabelArray(ndarray):
An array with the same shape as self indicating whether each
element of self ended with ``suffix``.
"""
return self.apply(lambda elem: substring in elem, dtype=bool)
return self.map_predicate(lambda elem: substring in elem)
@preprocess(pattern=coerce(from_=(bytes, unicode), to=re.compile))
def matches(self, pattern):
@@ -522,7 +564,7 @@ class LabelArray(ndarray):
An array with the same shape as self indicating whether each
element of self was matched by ``pattern``.
"""
return self.apply(compose(bool, pattern.match), dtype=bool)
return self.map_predicate(compose(bool, pattern.match))
# These types all implement an O(N) __contains__, so pre-emptively
# coerce to `set`.
@@ -543,4 +585,4 @@ class LabelArray(ndarray):
An array with the same shape as self indicating whether each
element of self was an element of ``container``.
"""
return self.apply(container.__contains__, dtype=bool)
return self.map_predicate(container.__contains__)
zipline/pipeline/classifiers/classifier.py
-1
View File
@@ -347,7 +347,6 @@ class StringPredicate(SingleInputMixin, Filter):
data = arrays[0]
return (
self._op(data, self._compval)
& (data != self.inputs[0].missing_value)
& mask
)
zipline/pipeline/data/testing.py
+5
View File
@@ -25,6 +25,11 @@ class TestingDataSet(DataSet):
int_col = Column(dtype=int64_dtype, missing_value=0)
categorical_col = Column(dtype=categorical_dtype, missing_value=u'')
categorical_default_None = Column(
dtype=categorical_dtype,
missing_value=None,
)
categorical_default_NULL = Column(
dtype=categorical_dtype,
missing_value=u'<<NULL>>',
zipline/testing/core.py
+14 -1
View File
@@ -40,6 +40,7 @@ from zipline.data.us_equity_pricing import (
)
from zipline.finance.trading import TradingEnvironment
from zipline.finance.order import ORDER_STATUS
from zipline.lib.labelarray import LabelArray
from zipline.pipeline.engine import SimplePipelineEngine
from zipline.pipeline.loaders.testing import make_seeded_random_loader
from zipline.utils import security_list
@@ -394,7 +395,19 @@ def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True):
assert type(x) == type(y), "{x} != {y}".format(x=type(x), y=type(y))
assert x.dtype == y.dtype, "{x.dtype} != {y.dtype}".format(x=x, y=y)
return assert_array_equal(x, y, err_msg=err_msg, verbose=True)
if isinstance(x, LabelArray):
# Check that both arrays have missing values in the same locations...
assert_array_equal(
x.is_missing(),
y.is_missing(),
err_msg=err_msg,
verbose=verbose,
)
# ...then check the actual values as well.
x = x.as_string_array()
y = y.as_string_array()
return assert_array_equal(x, y, err_msg=err_msg, verbose=verbose)
class UnexpectedAttributeAccess(Exception):