From e0aeda4c3eaac814a182d3f02792f26ce50b1cec Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Thu, 5 May 2016 01:30:57 -0400 Subject: [PATCH] BUG: Fix bytes/unicode issues in py3. --- tests/pipeline/test_adjusted_array.py | 40 +++++++++++++++------------ tests/pipeline/test_classifier.py | 23 ++++++++++----- tests/test_labelarray.py | 6 ++++ zipline/lib/_factorize.pyx | 14 +++++++--- 4 files changed, 54 insertions(+), 29 deletions(-) diff --git a/tests/pipeline/test_adjusted_array.py b/tests/pipeline/test_adjusted_array.py index b645079d..cea6f090 100644 --- a/tests/pipeline/test_adjusted_array.py +++ b/tests/pipeline/test_adjusted_array.py @@ -27,6 +27,7 @@ from zipline.lib.adjustment import ( from zipline.lib.adjusted_array import AdjustedArray, NOMASK from zipline.lib.labelarray import LabelArray from zipline.testing import check_arrays, parameter_space +from zipline.utils.compat import unicode from zipline.utils.numpy_utils import ( coerce_to_dtype, datetime64ns_dtype, @@ -84,7 +85,7 @@ def as_labelarray(initial_dtype, missing_value, array): """ return LabelArray( array.astype(initial_dtype), - missing_value=initial_dtype.type(''), + missing_value=initial_dtype.type(missing_value), ) @@ -367,9 +368,9 @@ class AdjustedArrayTestCase(TestCase): ), _gen_unadjusted_cases( 'object_ndarray', - make_input=lambda a: a.astype(str).astype(object), - make_expected_output=as_labelarray(bytes_dtype, b''), - missing_value=b'', + make_input=lambda a: a.astype(unicode).astype(object), + make_expected_output=as_labelarray(unicode_dtype, u''), + missing_value='', ), # Test passing a LabelArray directly to AdjustedArray. _gen_unadjusted_cases( @@ -380,17 +381,17 @@ class AdjustedArrayTestCase(TestCase): ), _gen_unadjusted_cases( 'unicode_labelarray', - make_input=as_labelarray(unicode_dtype, u''), - make_expected_output=as_labelarray(bytes_dtype, u''), + make_input=as_labelarray(unicode_dtype, None), + make_expected_output=as_labelarray(unicode_dtype, None), missing_value=u'', ), _gen_unadjusted_cases( 'object_labelarray', make_input=( - lambda a: LabelArray(a.astype(str).astype(object), b'') + lambda a: LabelArray(a.astype(unicode).astype(object), u'') ), - make_expected_output=as_labelarray(bytes_dtype, b''), - missing_value=b'', + make_expected_output=as_labelarray(unicode_dtype, ''), + missing_value='', ), ) ) @@ -442,8 +443,8 @@ class AdjustedArrayTestCase(TestCase): ), ), # There are six cases here: - # Using np.bytes/np.unicode/python string arrays as inputs. - # Passing np.bytes/np.unicode/python string arrays to LabelArray, + # Using np.bytes/np.unicode/object arrays as inputs. + # Passing np.bytes/np.unicode/object arrays to LabelArray, # and using those as input. # # The outputs should always be LabelArrays. @@ -461,9 +462,9 @@ class AdjustedArrayTestCase(TestCase): ), _gen_unadjusted_cases( 'object_ndarray', - make_input=lambda a: a.astype(str).astype(object), - make_expected_output=as_labelarray(bytes_dtype, b''), - missing_value=b'', + make_input=lambda a: a.astype(unicode).astype(object), + make_expected_output=as_labelarray(unicode_dtype, u''), + missing_value=u'', ), _gen_unadjusted_cases( 'bytes_labelarray', @@ -474,16 +475,19 @@ class AdjustedArrayTestCase(TestCase): _gen_unadjusted_cases( 'unicode_labelarray', make_input=as_labelarray(unicode_dtype, u''), - make_expected_output=as_labelarray(bytes_dtype, u''), + make_expected_output=as_labelarray(unicode_dtype, u''), missing_value=u'', ), _gen_unadjusted_cases( 'object_labelarray', make_input=( - lambda a: LabelArray(a.astype(str).astype(object), b'') + lambda a: LabelArray( + a.astype(unicode).astype(object), + None, + ) ), - make_expected_output=as_labelarray(bytes_dtype, b''), - missing_value=b'', + make_expected_output=as_labelarray(unicode_dtype, u''), + missing_value=None, ), ) ) diff --git a/tests/pipeline/test_classifier.py b/tests/pipeline/test_classifier.py index 522639df..de111e97 100644 --- a/tests/pipeline/test_classifier.py +++ b/tests/pipeline/test_classifier.py @@ -1,3 +1,4 @@ +from functools import reduce from operator import or_ import numpy as np @@ -260,17 +261,25 @@ class ClassifierTestCase(BasePipelineTestCase): @parameter_space( __fail_fast=True, - compval=['a', 'b', 'ab', 'not in the array'], - missing=['a', 'ab', '', 'not in the array'], + compval=[u'a', u'b', u'ab', u'not in the array'], + missing=[u'a', u'ab', u'', u'not in the array'], labelarray_dtype=(categorical_dtype, bytes_dtype, unicode_dtype), ) def test_string_elementwise_predicates(self, compval, missing, labelarray_dtype): + if labelarray_dtype == bytes_dtype: + compval = compval.encode('utf-8') + missing = missing.encode('utf-8') - missing = labelarray_dtype.type(missing) - compval = labelarray_dtype.type(compval) + startswith_re = b'^' + compval + b'.*' + endswith_re = b'.*' + compval + b'$' + substring_re = b'.*' + compval + b'.*' + else: + startswith_re = '^' + compval + '.*' + endswith_re = '.*' + compval + '$' + substring_re = '.*' + compval + '.*' class C(Classifier): dtype = categorical_dtype @@ -298,9 +307,9 @@ class ClassifierTestCase(BasePipelineTestCase): 'endswith': c.endswith(compval), 'has_substring': c.has_substring(compval), # Equivalent filters using regex matching. - 'startswith_re': c.matches('^' + compval + '.*'), - 'endswith_re': c.matches('.*' + compval + '$'), - 'has_substring_re': c.matches('.*' + compval + '.*'), + 'startswith_re': c.matches(startswith_re), + 'endswith_re': c.matches(endswith_re), + 'has_substring_re': c.matches(substring_re), } expected = { diff --git a/tests/test_labelarray.py b/tests/test_labelarray.py index fa1d2130..f3c98322 100644 --- a/tests/test_labelarray.py +++ b/tests/test_labelarray.py @@ -4,6 +4,7 @@ import numpy as np from zipline.lib.labelarray import LabelArray from zipline.testing import check_arrays, parameter_space, ZiplineTestCase +from zipline.utils.compat import unicode def rotN(l, N): @@ -67,10 +68,15 @@ class LabelArrayTestCase(ZiplineTestCase): # using the ufunc. notmissing = np.not_equal(strs, missing_value) else: + if not isinstance(missing_value, array_astype): + missing_value = array_astype(missing_value, 'utf-8') notmissing = (strs != missing_value) arr = LabelArray(strs, missing_value=missing_value) + if not isinstance(compval, array_astype): + compval = array_astype(compval, 'utf-8') + # arr.missing_value should behave like NaN. check_arrays( arr == compval, diff --git a/zipline/lib/_factorize.pyx b/zipline/lib/_factorize.pyx index d3a98d18..31cd0ff2 100644 --- a/zipline/lib/_factorize.pyx +++ b/zipline/lib/_factorize.pyx @@ -2,7 +2,7 @@ Factorization algorithms. """ from numpy cimport ndarray, int64_t, PyArray_Check, import_array -from numpy import arange, asarray, empty, int64, isnan, ndarray +from numpy import arange, asarray, empty, int64, isnan, ndarray, zeros import_array() @@ -18,7 +18,7 @@ cpdef factorize_strings_known_categories(ndarray[object] values, `missing_value`. """ if missing_value not in categories: - categories.append(missing_value) + categories.insert(0, missing_value) if sort: categories = sorted(categories) @@ -46,6 +46,7 @@ cpdef factorize_strings_known_categories(ndarray[object] values, return codes, asarray(categories, dtype=object), reverse_categories + cpdef factorize_strings(ndarray[object] values, object missing_value, int sort): @@ -94,10 +95,15 @@ cpdef factorize_strings(ndarray[object] values, cdef ndarray[int64_t, ndim=1] reverse_indexer cdef int ncategories cdef ndarray[object] categories_array = asarray(categories, dtype=object) + if sort: - # This is all taken from pandas.core.algorithms.factorize. + # This is all adapted from pandas.core.algorithms.factorize. ncategories = len(categories_array) - sorter = categories_array.argsort() + sorter = zeros(ncategories, dtype=int64) + + # Don't include missing_value in the argsort, because None is + # unorderable with bytes/str in py3. Always just sort it to 0. + sorter[1:] = categories_array[1:].argsort() + 1 reverse_indexer = empty(ncategories, dtype=int64) reverse_indexer.put(sorter, arange(ncategories))