diff --git a/tests/test_labelarray.py b/tests/test_labelarray.py index f7a32f40..9c2462e6 100644 --- a/tests/test_labelarray.py +++ b/tests/test_labelarray.py @@ -33,7 +33,7 @@ class LabelArrayTestCase(ZiplineTestCase): super(LabelArrayTestCase, cls).init_class_fixtures() cls.rowvalues = row = ['', 'a', 'b', 'ab', 'a', '', 'b', 'ab', 'z'] - cls.strs = np.array([rotN(row, i) for i in range(3)]) + cls.strs = np.array([rotN(row, i) for i in range(3)], dtype=object) def test_fail_on_direct_construction(self): # See http://docs.scipy.org/doc/numpy-1.10.0/user/basics.subclassing.html#simple-example-adding-an-extra-attribute-to-ndarray # noqa @@ -48,32 +48,82 @@ class LabelArrayTestCase(ZiplineTestCase): @parameter_space( __fail_fast=True, - s=['', 'a', 'z', 'aa', 'not in the array'], - shape=[(27,), (9, 3), (3, 9), (3, 3, 3)], + compval=['', 'a', 'z', 'not in the array'], + shape=[(27,), (3, 9), (3, 3, 3)], array_astype=(bytes, unicode, object), - scalar_astype=(bytes, unicode, object), + missing_value=('', 'a', 'not in the array', None), ) - def test_compare_to_str(self, s, shape, array_astype, scalar_astype): + def test_compare_to_str(self, + compval, + shape, + array_astype, + missing_value): + strs = self.strs.reshape(shape).astype(array_astype) - arr = LabelArray(strs, missing_value='') - check_arrays(strs == s, arr == s) - check_arrays(strs != s, arr != s) + if missing_value is None: + # As of numpy 1.9.2, object array != None returns just False + # instead of an array, with a deprecation warning saying the + # behavior will change in the future. Work around that by just + # using the ufunc. + notmissing = np.not_equal(strs, missing_value) + else: + notmissing = (strs != missing_value) - np_startswith = np.vectorize(lambda elem: elem.startswith(s)) - check_arrays(arr.startswith(s), np_startswith(strs)) + arr = LabelArray(strs, missing_value=missing_value) - np_endswith = np.vectorize(lambda elem: elem.endswith(s)) - check_arrays(arr.endswith(s), np_endswith(strs)) + # arr.missing_value should behave like NaN. + check_arrays( + arr == compval, + (strs == compval) & notmissing, + ) + check_arrays( + arr != compval, + (strs != compval) & notmissing, + ) - np_contains = np.vectorize(lambda elem: s in elem) - check_arrays(arr.has_substring(s), np_contains(strs)) + np_startswith = np.vectorize(lambda elem: elem.startswith(compval)) + check_arrays( + arr.startswith(compval), + np_startswith(strs) & notmissing, + ) - def test_compare_to_str_array(self): + np_endswith = np.vectorize(lambda elem: elem.endswith(compval)) + check_arrays( + arr.endswith(compval), + np_endswith(strs) & notmissing, + ) + + np_contains = np.vectorize(lambda elem: compval in elem) + check_arrays( + arr.has_substring(compval), + np_contains(strs) & notmissing, + ) + + @parameter_space( + __fail_fast=True, + missing_value=('', 'a', 'not in the array', None), + ) + def test_compare_to_str_array(self, missing_value): strs = self.strs shape = strs.shape - arr = LabelArray(strs, missing_value='') - check_arrays(strs == arr, np.full_like(strs, True, dtype=bool)) - check_arrays(strs != arr, np.full_like(strs, False, dtype=bool)) + arr = LabelArray(strs, missing_value=missing_value) + + if missing_value is None: + # As of numpy 1.9.2, object array != None returns just False + # instead of an array, with a deprecation warning saying the + # behavior will change in the future. Work around that by just + # using the ufunc. + notmissing = np.not_equal(strs, missing_value) + else: + notmissing = (strs != missing_value) + + check_arrays(arr.not_missing(), notmissing) + check_arrays(arr.is_missing(), ~notmissing) + + # The arrays are equal everywhere, but comparisons against the + # missing_value should always produce False + check_arrays(strs == arr, notmissing) + check_arrays(strs != arr, np.zeros_like(strs, dtype=bool)) def broadcastable_row(value, dtype): return np.full((shape[0], 1), value, dtype=strs.dtype) @@ -81,20 +131,22 @@ class LabelArrayTestCase(ZiplineTestCase): def broadcastable_col(value, dtype): return np.full((1, shape[1]), value, dtype=strs.dtype) + # Test comparison between arr and a like-shap 2D array, a column + # vector, and a row vector. for comparator, dtype, value in product((eq, ne), (bytes, unicode, object), set(self.rowvalues)): check_arrays( comparator(arr, np.full_like(strs, value)), - comparator(strs, value), + comparator(strs, value) & notmissing, ) check_arrays( comparator(arr, broadcastable_row(value, dtype=dtype)), - comparator(strs, value), + comparator(strs, value) & notmissing, ) check_arrays( comparator(arr, broadcastable_col(value, dtype=dtype)), - comparator(strs, value), + comparator(strs, value) & notmissing, ) @parameter_space( @@ -122,6 +174,10 @@ class LabelArrayTestCase(ZiplineTestCase): self.assertIs(sliced.missing_value, arr.missing_value) def test_infer_categories(self): + """ + Test that categories are inferred in sorted order if they're not + explicitly passed. + """ arr1d = LabelArray(self.strs, missing_value='') codes1d = arr1d.as_int_array() self.assertEqual(arr1d.shape, self.strs.shape) @@ -144,6 +200,14 @@ class LabelArrayTestCase(ZiplineTestCase): arr1d.as_int_array() == idx, ) + # It should be equivalent to pass the same set of categories manually. + arr1d_explicit_categories = LabelArray( + self.strs, + missing_value='', + categories=arr1d.categories, + ) + check_arrays(arr1d, arr1d_explicit_categories) + for shape in (9, 3), (3, 9), (3, 3, 3): strs2d = self.strs.reshape(shape) arr2d = LabelArray(strs2d, missing_value='') diff --git a/zipline/lib/adjusted_array.py b/zipline/lib/adjusted_array.py index b75ac918..40c6803e 100644 --- a/zipline/lib/adjusted_array.py +++ b/zipline/lib/adjusted_array.py @@ -18,7 +18,6 @@ from zipline.errors import ( WindowLengthTooLong, ) from zipline.lib.labelarray import LabelArray -from zipline.utils.compat import unicode from zipline.utils.numpy_utils import ( datetime64ns_dtype, float64_dtype, @@ -111,10 +110,10 @@ def _normalize_array(data, missing_value): elif data_dtype in INT_DTYPES: return data.astype(int64), {'dtype': dtype(int64)} elif is_categorical(data_dtype): - if not isinstance(missing_value, (bytes, unicode)): + if not isinstance(missing_value, LabelArray.SUPPORTED_SCALAR_TYPES): raise TypeError( "Invalid missing_value for categorical array.\n" - "Expected bytes or unicode. Got %r." % missing_value, + "Expected None, bytes or unicode. Got %r." % missing_value, ) return LabelArray(data, missing_value), {} elif data_dtype.kind == 'M': diff --git a/zipline/lib/labelarray.py b/zipline/lib/labelarray.py index f33a9021..fe27ecbe 100644 --- a/zipline/lib/labelarray.py +++ b/zipline/lib/labelarray.py @@ -2,7 +2,6 @@ An ndarray subclass for working with arrays of strings. """ from functools import partial -from numbers import Number from operator import eq, ne import re @@ -19,7 +18,11 @@ from zipline.utils.input_validation import ( expect_types, optional, ) -from zipline.utils.numpy_utils import int_dtype_with_size_in_bytes, is_object +from zipline.utils.numpy_utils import ( + bool_dtype, + int_dtype_with_size_in_bytes, + is_object, +) from ._factorize import ( factorize_strings, @@ -45,6 +48,18 @@ def _make_unsupported_method(name): return method +class MissingValueMismatch(ValueError): + """ + Error raised on attempt to perform operations between LabelArrays with + mismatched missing_values. + """ + def __init__(self, left, right): + super(MissingValueMismatch, self).__init__( + "LabelArray missing_values don't match:" + " left={}, right={}".format(left, right) + ) + + class CategoryMismatch(ValueError): """ Error raised on attempt to perform operations between LabelArrays with @@ -95,7 +110,7 @@ class LabelArray(ndarray): reverse_categories : dict[str -> int] Reverse lookup table for ``categories``. Stores the index in ``categories`` at which each entry each unique entry is found. - missing_value : str + missing_value : str or None A sentinel missing value with NaN semantics for comparisons. Notes @@ -117,11 +132,15 @@ class LabelArray(ndarray): -------- http://docs.scipy.org/doc/numpy-1.10.0/user/basics.subclassing.html """ + SUPPORTED_SCALAR_TYPES = (bytes, unicode, type(None)) + @preprocess( values=coerce(list, partial(np.asarray, dtype=object)), + categories=coerce(np.ndarray, list), ) @expect_types( values=np.ndarray, + missing_value=SUPPORTED_SCALAR_TYPES, categories=optional(list), ) @expect_kinds(values=("O", "S", "U")) @@ -301,9 +320,9 @@ class LabelArray(ndarray): else: raise CategoryMismatch(self_categories, value_categories) - elif isinstance(value, (bytes, unicode)): - value_code = self.reverse_categories.get(value, None) - if value_code is None: + elif isinstance(value, self.SUPPORTED_SCALAR_TYPES): + value_code = self.reverse_categories.get(value, -1) + if value_code < 0: raise ValueError("%r is not in LabelArray categories." % value) self.as_int_array()[indexer] = value_code else: @@ -314,47 +333,54 @@ class LabelArray(ndarray): ), ) + def is_missing(self): + """ + Like isnan, but checks for locations where we store missing values. + """ + return ( + self.as_int_array() == self.reverse_categories[self.missing_value] + ) + + def not_missing(self): + """ + Like ~isnan, but checks for locations where we store missing values. + """ + return ( + self.as_int_array() != self.reverse_categories[self.missing_value] + ) + def _equality_check(op): """ Shared code for __eq__ and __ne__, parameterized on the actual comparison operator to use. """ - # What value should we return if we compare against a value not in our - # categories? - if op is eq: - COMPARE_TO_UNKNOWN = False - elif op is ne: - COMPARE_TO_UNKNOWN = True - else: - raise AssertionError("_make_equality_check called with %s" % op) - def method(self, other): - self_categories = self.categories if isinstance(other, LabelArray): + self_mv = self.missing_value + other_mv = other.missing_value + if self_mv != other_mv: + raise MissingValueMismatch(self_mv, other_mv) + + self_categories = self.categories other_categories = other.categories - if compare_arrays(self_categories, other_categories): - return op(self.as_int_array(), other.as_int_array()) - else: + if not compare_arrays(self_categories, other_categories): raise CategoryMismatch(self_categories, other_categories) + return ( + op(self.as_int_array(), other.as_int_array()) + & self.not_missing() + & other.not_missing() + ) + elif isinstance(other, ndarray): # Compare to ndarrays as though we were an array of strings. # This is fairly expensive, and should generally be avoided. - return op(self.as_string_array(), other) + return op(self.as_string_array(), other) & self.not_missing() - elif isinstance(other, (bytes, unicode)): - i = self._reverse_categories.get(other, None) - if i is None: - # Requested string isn't in our categories. Short circuit. - # This isn't full_like because that would try to return a - # LabelArray. - return np.full(self.shape, COMPARE_TO_UNKNOWN, dtype=bool) - - return op(self.as_int_array(), i) - - elif isinstance(other, Number): - return NotImplemented + elif isinstance(other, self.SUPPORTED_SCALAR_TYPES): + i = self._reverse_categories.get(other, -1) + return op(self.as_int_array(), i) & self.not_missing() return op(super(LabelArray, self), other) return method @@ -450,14 +476,30 @@ class LabelArray(ndarray): missing_value=self.missing_value, ) - def apply(self, f, dtype): + def map_predicate(self, f): """ - Map a function elementwise over entries in ``self``. + Map a function from str -> bool element-wise over ``self``. - ``f`` will be applied exactly once to each unique value in ``self``. + ``f`` will be applied exactly once to each non-missing unique value in + ``self``. Missing values will always return False. """ - vf = np.vectorize(f, otypes=[dtype]) - return vf(self.categories)[self.as_int_array()] + # Functions passed to this are of type str -> bool. Don't ever call + # them on None, which is the only non-str value we ever store in + # categories. + if self.missing_value is None: + f_to_use = lambda x: False if x is None else f(x) + else: + f_to_use = f + + # Call f on each unique value in our categories. + results = np.vectorize(f_to_use, otypes=[bool_dtype])(self.categories) + + # missing_value should produce False no matter what + results[self.reverse_categories[self.missing_value]] = False + + # unpack the results form each unique value into their corresponding + # locations in our indices. + return results[self.as_int_array()] def startswith(self, prefix): """ @@ -473,7 +515,7 @@ class LabelArray(ndarray): An array with the same shape as self indicating whether each element of self started with ``prefix``. """ - return self.apply(lambda elem: elem.startswith(prefix), dtype=bool) + return self.map_predicate(lambda elem: elem.startswith(prefix)) def endswith(self, suffix): """ @@ -489,7 +531,7 @@ class LabelArray(ndarray): An array with the same shape as self indicating whether each element of self ended with ``suffix`` """ - return self.apply(lambda elem: elem.endswith(suffix), dtype=bool) + return self.map_predicate(lambda elem: elem.endswith(suffix)) def has_substring(self, substring): """ @@ -505,7 +547,7 @@ class LabelArray(ndarray): An array with the same shape as self indicating whether each element of self ended with ``suffix``. """ - return self.apply(lambda elem: substring in elem, dtype=bool) + return self.map_predicate(lambda elem: substring in elem) @preprocess(pattern=coerce(from_=(bytes, unicode), to=re.compile)) def matches(self, pattern): @@ -522,7 +564,7 @@ class LabelArray(ndarray): An array with the same shape as self indicating whether each element of self was matched by ``pattern``. """ - return self.apply(compose(bool, pattern.match), dtype=bool) + return self.map_predicate(compose(bool, pattern.match)) # These types all implement an O(N) __contains__, so pre-emptively # coerce to `set`. @@ -543,4 +585,4 @@ class LabelArray(ndarray): An array with the same shape as self indicating whether each element of self was an element of ``container``. """ - return self.apply(container.__contains__, dtype=bool) + return self.map_predicate(container.__contains__) diff --git a/zipline/pipeline/classifiers/classifier.py b/zipline/pipeline/classifiers/classifier.py index 332fc206..d35a5135 100644 --- a/zipline/pipeline/classifiers/classifier.py +++ b/zipline/pipeline/classifiers/classifier.py @@ -347,7 +347,6 @@ class StringPredicate(SingleInputMixin, Filter): data = arrays[0] return ( self._op(data, self._compval) - & (data != self.inputs[0].missing_value) & mask ) diff --git a/zipline/pipeline/data/testing.py b/zipline/pipeline/data/testing.py index 23712000..82579f7d 100644 --- a/zipline/pipeline/data/testing.py +++ b/zipline/pipeline/data/testing.py @@ -25,6 +25,11 @@ class TestingDataSet(DataSet): int_col = Column(dtype=int64_dtype, missing_value=0) categorical_col = Column(dtype=categorical_dtype, missing_value=u'') + categorical_default_None = Column( + dtype=categorical_dtype, + missing_value=None, + ) + categorical_default_NULL = Column( dtype=categorical_dtype, missing_value=u'<>', diff --git a/zipline/testing/core.py b/zipline/testing/core.py index ad130a79..ab60d516 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -40,6 +40,7 @@ from zipline.data.us_equity_pricing import ( ) from zipline.finance.trading import TradingEnvironment from zipline.finance.order import ORDER_STATUS +from zipline.lib.labelarray import LabelArray from zipline.pipeline.engine import SimplePipelineEngine from zipline.pipeline.loaders.testing import make_seeded_random_loader from zipline.utils import security_list @@ -394,7 +395,19 @@ def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True): assert type(x) == type(y), "{x} != {y}".format(x=type(x), y=type(y)) assert x.dtype == y.dtype, "{x.dtype} != {y.dtype}".format(x=x, y=y) - return assert_array_equal(x, y, err_msg=err_msg, verbose=True) + if isinstance(x, LabelArray): + # Check that both arrays have missing values in the same locations... + assert_array_equal( + x.is_missing(), + y.is_missing(), + err_msg=err_msg, + verbose=verbose, + ) + # ...then check the actual values as well. + x = x.as_string_array() + y = y.as_string_array() + + return assert_array_equal(x, y, err_msg=err_msg, verbose=verbose) class UnexpectedAttributeAccess(Exception):