diff --git a/tests/pipeline/test_classifier.py b/tests/pipeline/test_classifier.py index f52e801e..522639df 100644 --- a/tests/pipeline/test_classifier.py +++ b/tests/pipeline/test_classifier.py @@ -21,7 +21,7 @@ unicode_dtype = np.dtype('U3') class ClassifierTestCase(BasePipelineTestCase): @parameter_space(mv=[-1, 0, 1, 999]) - def test_isnull(self, mv): + def test_integral_isnull(self, mv): class C(Classifier): dtype = int64_dtype @@ -51,6 +51,41 @@ class ClassifierTestCase(BasePipelineTestCase): mask=self.build_mask(self.ones_mask(shape=data.shape)), ) + @parameter_space(mv=['0', None]) + def test_string_isnull(self, mv): + + class C(Classifier): + dtype = categorical_dtype + missing_value = mv + inputs = () + window_length = 0 + + c = C() + + # There's no significance to the values here other than that they + # contain a mix of missing and non-missing values. + raw = np.asarray( + [['', 'a', 'ab', 'ba'], + ['z', 'ab', 'a', 'ab'], + ['aa', 'ab', '', 'ab'], + ['aa', 'a', 'ba', 'ba']], + dtype=categorical_dtype, + ) + data = LabelArray(raw, missing_value=mv) + + self.check_terms( + terms={ + 'isnull': c.isnull(), + 'notnull': c.notnull() + }, + expected={ + 'isnull': np.equal(raw, mv), + 'notnull': np.not_equal(raw, mv), + }, + initial_workspace={c: data}, + mask=self.build_mask(self.ones_mask(shape=data.shape)), + ) + @parameter_space(compval=[0, 1, 999]) def test_eq(self, compval): diff --git a/zipline/lib/rank.pyx b/zipline/lib/rank.pyx index e9a045a4..3f529f4d 100644 --- a/zipline/lib/rank.pyx +++ b/zipline/lib/rank.pyx @@ -28,9 +28,9 @@ from zipline.utils.numpy_utils import ( import_array() -cpdef ismissing(ndarray data, object missing_value): +cpdef is_missing(ndarray data, object missing_value): """ - Generic ismissing function that handles quirks with NaN. + Generic is_missing function that handles quirks with NaN. """ if is_float(data) and isnan(missing_value): return isnan(data) @@ -51,7 +51,7 @@ def masked_rankdata_2d(ndarray data, "Can't compute rankdata on array of dtype %r." % dtype_name ) - cdef ndarray missing_locations = (~mask | ismissing(data, missing_value)) + cdef ndarray missing_locations = (~mask | is_missing(data, missing_value)) # Interpret the bytes of integral data as floats for sorting. data = data.copy().view(float64) diff --git a/zipline/pipeline/filters/filter.py b/zipline/pipeline/filters/filter.py index 360667f4..053b794e 100644 --- a/zipline/pipeline/filters/filter.py +++ b/zipline/pipeline/filters/filter.py @@ -14,7 +14,8 @@ from zipline.errors import ( BadPercentileBounds, UnsupportedDataType, ) -from zipline.lib.rank import ismissing +from zipline.lib.labelarray import LabelArray +from zipline.lib.rank import is_missing from zipline.pipeline.mixins import ( CustomTermMixin, LatestMixin, @@ -230,19 +231,22 @@ class NullFilter(SingleInputMixin, Filter): Parameters ---------- - factor : zipline.pipeline.Factor + factor : zipline.pipeline.Term The factor to compare against its missing_value. """ window_length = 0 - def __new__(cls, factor): + def __new__(cls, term): return super(NullFilter, cls).__new__( cls, - inputs=(factor,), + inputs=(term,), ) def _compute(self, arrays, dates, assets, mask): - return ismissing(arrays[0], self.inputs[0].missing_value) + data = arrays[0] + if isinstance(data, LabelArray): + return data.is_missing() + return is_missing(arrays[0], self.inputs[0].missing_value) class PercentileFilter(SingleInputMixin, Filter):