MAINT: Remove support for custom string Column missing values.

Pandas 0.18 deprecated passing "null-ish" values to pd.categorical.  The
expectation, instead, is that you use categorical's native support for
missing data, which means the user will always get NaN's for missing
entries of the categorical.

A follow-up to this change should probably drop support for custom
missing values entirely and to use LabelArray/categorical for integer
data.

tests/pipeline/test_classifier.py

@@ -51,7 +51,7 @@ class ClassifierTestCase(BasePipelineTestCase):
             mask=self.build_mask(self.ones_mask(shape=data.shape)),
         )
 
-    @parameter_space(mv=['0', None])
+    @parameter_space(mv=[None])
     def test_string_isnull(self, mv):
 
         class C(Classifier):
@@ -126,7 +126,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         class C(Classifier):
             dtype = categorical_dtype
-            missing_value = ''
+            missing_value = None
             inputs = ()
             window_length = 0
 
@@ -162,7 +162,7 @@ class ClassifierTestCase(BasePipelineTestCase):
     )
     def test_disallow_comparison_to_missing_value(self, missing, dtype_):
         if dtype_ == categorical_dtype:
-            missing = str(missing)
+            missing = None
 
         class C(Classifier):
             dtype = dtype_
@@ -224,7 +224,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         class C(Classifier):
             dtype = categorical_dtype
-            missing_value = missing
+            missing_value = None
             inputs = ()
             window_length = 0
 
@@ -245,7 +245,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         expected = (
             (data.as_int_array() != data.reverse_categories.get(compval, -1)) &
-            (data.as_int_array() != data.reverse_categories[C.missing_value])
+            (data.as_int_array() != data.reverse_categories[missing])
         )
 
         self.check_terms(
@@ -271,7 +271,6 @@ class ClassifierTestCase(BasePipelineTestCase):
                                            labelarray_dtype):
         if labelarray_dtype == bytes_dtype:
             compval = compval.encode('utf-8')
-            missing = missing.encode('utf-8')
 
             startswith_re = b'^' + compval + b'.*'
             endswith_re = b'.*' + compval + b'$'
@@ -283,7 +282,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         class C(Classifier):
             dtype = categorical_dtype
-            missing_value = missing
+            missing_value = None
             inputs = ()
             window_length = 0
 
@@ -338,7 +337,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         class C(Classifier):
             dtype = categorical_dtype
-            missing_value = missing
+            missing_value = None
             inputs = ()
             window_length = 0
 
@@ -418,7 +417,7 @@ class ClassifierTestCase(BasePipelineTestCase):
         Test that element_of raises a useful error if we attempt to pass it an
         array of choices that include the classifier's missing_value.
         """
-        missing = "not in the array"
+        missing = None
 
         class C(Classifier):
             dtype = categorical_dtype
@@ -433,7 +432,7 @@ class ClassifierTestCase(BasePipelineTestCase):
                 c.element_of(bad_elems)
             errmsg = str(e.exception)
             expected = (
-                "Found self.missing_value ('not in the array') in choices"
+                "Found self.missing_value (None) in choices"
                 " supplied to C.element_of().\n"
                 "Missing values have NaN semantics, so the requested"
                 " comparison would always produce False.\n"
@@ -447,7 +446,7 @@ class ClassifierTestCase(BasePipelineTestCase):
 
         class C(Classifier):
             dtype = dtype_
-            missing_value = dtype.type('1')
+            missing_value = None if dtype_ is categorical_dtype else -1
             inputs = ()
             window_length = 0
 

tests/pipeline/test_events.py

@@ -55,15 +55,6 @@ class EventDataSet(DataSet):
     previous_string = Column(dtype=categorical_dtype, missing_value=None)
     next_string = Column(dtype=categorical_dtype, missing_value=None)
 
-    previous_string_custom_missing = Column(
-        dtype=categorical_dtype,
-        missing_value=u"<<NULL>>",
-    )
-    next_string_custom_missing = Column(
-        dtype=categorical_dtype,
-        missing_value=u"<<NULL>>",
-    )
-
 
 critical_dates = pd.to_datetime([
     '2014-01-05',
@@ -289,7 +280,6 @@ class EventsLoaderTestCase(WithAssetFinder,
             EventDataSet.next_float: 'float',
             EventDataSet.next_int: 'int',
             EventDataSet.next_string: 'string',
-            EventDataSet.next_string_custom_missing: 'string'
         }
         cls.previous_value_columns = {
             EventDataSet.previous_datetime: 'datetime',
@@ -297,7 +287,6 @@ class EventsLoaderTestCase(WithAssetFinder,
             EventDataSet.previous_float: 'float',
             EventDataSet.previous_int: 'int',
             EventDataSet.previous_string: 'string',
-            EventDataSet.previous_string_custom_missing: 'string'
         }
         cls.loader = cls.make_loader(
             events=cls.raw_events,
@@ -377,6 +366,11 @@ class EventsLoaderTestCase(WithAssetFinder,
                     # If we've seen event 1 but not event 2, event 1 should
                     # win.
                     self.assertEqual(computed_value, v1)
+                elif column.dtype == categorical_dtype:
+                    # XXX: The value in the output from pandas will be np.nan,
+                    #      but we currently only support None as the missing
+                    #      value for string columns.
+                    self.assertTrue(np.isnan(computed_value))
                 else:
                     # If we haven't seen either event, then we should have
                     # column.missing_value.
@@ -414,6 +408,11 @@ class EventsLoaderTestCase(WithAssetFinder,
                     # If we've seen event 1 but not event 2, event 1 should
                     # win.
                     self.assertEqual(computed_value, v2)
+                elif column.dtype == categorical_dtype:
+                    # XXX: The value in the output from pandas will be np.nan,
+                    #      but we currently only support None as the missing
+                    #      value for string columns.
+                    self.assertTrue(np.isnan(computed_value))
                 else:
                     # If we haven't seen either event, then we should have
                     # column.missing_value.

tests/pipeline/test_term.py

@@ -742,7 +742,7 @@ class SubDataSetTestCase(TestCase):
             window_length = 5
             inputs = [SomeDataSet.foo, SomeDataSet.bar]
             outputs = outputs_
-            missing_value = dtype_.type('123')
+            missing_value = None if dtype_ is categorical_dtype else -1
 
         expected_error = (
             "SomeClassifier does not support custom outputs, "

zipline/lib/labelarray.py

@@ -284,11 +284,28 @@ class LabelArray(ndarray):
         """
         if len(self.shape) > 1:
             raise ValueError("Can't convert a 2D array to a categorical.")
+
+        missing_code = self.reverse_categories[self.missing_value]
+        raw_codes = self.as_int_array()
+        # As of pandas 0.18, putting null values in pandas categoricals is
+        # deprecated. The preferred representation is to pass -1 as the code
+        # for missing values.
+        if missing_code == 0:
+            # This is just a performance optimization. It should produce the
+            # same results as below.
+            codes = raw_codes - 1
+            categories = self.categories[1:]
+        else:
+            # subtract 1 for anything greater than the missing code, and set
+            # the missing code to -1.
+            codes = raw_codes.copy()
+            codes[codes > missing_code] -= 1
+            codes[codes == missing_code] = -1
+            categories = self.categories[self.categories != self.missing_value]
+
         return pd.Categorical.from_codes(
-            self.as_int_array(),
-            # We need to make a copy because pandas >= 0.17 fails if this
-            # buffer isn't writeable.
-            self.categories.copy(),
+            codes,
+            categories,
             ordered=False,
             name=name,
         )

zipline/pipeline/data/testing.py

@@ -32,7 +32,3 @@ class TestingDataSet(DataSet):
         dtype=categorical_dtype,
         missing_value=None,
     )
-    categorical_default_NULL_string = Column(
-        dtype=categorical_dtype,
-        missing_value=u'<<NULL>>',
-    )

zipline/pipeline/term.py

@@ -28,7 +28,6 @@ from zipline.errors import (
     WindowLengthNotSpecified,
 )
 from zipline.lib.adjusted_array import can_represent_dtype
-from zipline.lib.labelarray import LabelArray
 from zipline.utils.input_validation import expect_types
 from zipline.utils.memoize import lazyval
 from zipline.utils.numpy_utils import (
@@ -756,12 +755,10 @@ def _assert_valid_categorical_missing_value(value):
 
     Raises a TypeError if the value is cannot be used as the missing_value for
     a categorical_dtype Term.
+
+    Currently, only None is supported as a missing value.
     """
-    label_types = LabelArray.SUPPORTED_SCALAR_TYPES
-    if not isinstance(value, label_types):
+    if value is not None:
         raise TypeError(
-            "Categorical terms must have missing values of type "
-            "{types}.".format(
-                types=' or '.join([t.__name__ for t in label_types]),
-            )
+            "Categorical terms must have missing values of None."
         )