wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-07 10:38:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

BUG: Fix bytes/unicode issues in py3.

Browse Source
This commit is contained in:
Scott Sanderson
2016-05-05 01:30:57 -04:00
parent 2ceeac1237
commit e0aeda4c3e
4 changed files with 54 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/pipeline/test_adjusted_array.py
+22 -18
View File
@@ -27,6 +27,7 @@ from zipline.lib.adjustment import (
from zipline.lib.adjusted_array import AdjustedArray, NOMASK
from zipline.lib.labelarray import LabelArray
from zipline.testing import check_arrays, parameter_space
from zipline.utils.compat import unicode
from zipline.utils.numpy_utils import (
coerce_to_dtype,
datetime64ns_dtype,
@@ -84,7 +85,7 @@ def as_labelarray(initial_dtype, missing_value, array):
"""
return LabelArray(
array.astype(initial_dtype),
missing_value=initial_dtype.type(''),
missing_value=initial_dtype.type(missing_value),
)
@@ -367,9 +368,9 @@ class AdjustedArrayTestCase(TestCase):
),
_gen_unadjusted_cases(
'object_ndarray',
make_input=lambda a: a.astype(str).astype(object),
make_expected_output=as_labelarray(bytes_dtype, b''),
missing_value=b'',
make_input=lambda a: a.astype(unicode).astype(object),
make_expected_output=as_labelarray(unicode_dtype, u''),
missing_value='',
),
# Test passing a LabelArray directly to AdjustedArray.
_gen_unadjusted_cases(
@@ -380,17 +381,17 @@ class AdjustedArrayTestCase(TestCase):
),
_gen_unadjusted_cases(
'unicode_labelarray',
make_input=as_labelarray(unicode_dtype, u''),
make_expected_output=as_labelarray(bytes_dtype, u''),
make_input=as_labelarray(unicode_dtype, None),
make_expected_output=as_labelarray(unicode_dtype, None),
missing_value=u'',
),
_gen_unadjusted_cases(
'object_labelarray',
make_input=(
lambda a: LabelArray(a.astype(str).astype(object), b'')
lambda a: LabelArray(a.astype(unicode).astype(object), u'')
),
make_expected_output=as_labelarray(bytes_dtype, b''),
missing_value=b'',
make_expected_output=as_labelarray(unicode_dtype, ''),
missing_value='',
),
)
)
@@ -442,8 +443,8 @@ class AdjustedArrayTestCase(TestCase):
),
),
# There are six cases here:
# Using np.bytes/np.unicode/python string arrays as inputs.
# Passing np.bytes/np.unicode/python string arrays to LabelArray,
# Using np.bytes/np.unicode/object arrays as inputs.
# Passing np.bytes/np.unicode/object arrays to LabelArray,
# and using those as input.
#
# The outputs should always be LabelArrays.
@@ -461,9 +462,9 @@ class AdjustedArrayTestCase(TestCase):
),
_gen_unadjusted_cases(
'object_ndarray',
make_input=lambda a: a.astype(str).astype(object),
make_expected_output=as_labelarray(bytes_dtype, b''),
missing_value=b'',
make_input=lambda a: a.astype(unicode).astype(object),
make_expected_output=as_labelarray(unicode_dtype, u''),
missing_value=u'',
),
_gen_unadjusted_cases(
'bytes_labelarray',
@@ -474,16 +475,19 @@ class AdjustedArrayTestCase(TestCase):
_gen_unadjusted_cases(
'unicode_labelarray',
make_input=as_labelarray(unicode_dtype, u''),
make_expected_output=as_labelarray(bytes_dtype, u''),
make_expected_output=as_labelarray(unicode_dtype, u''),
missing_value=u'',
),
_gen_unadjusted_cases(
'object_labelarray',
make_input=(
lambda a: LabelArray(a.astype(str).astype(object), b'')
lambda a: LabelArray(
a.astype(unicode).astype(object),
None,
)
),
make_expected_output=as_labelarray(bytes_dtype, b''),
missing_value=b'',
make_expected_output=as_labelarray(unicode_dtype, u''),
missing_value=None,
),
)
)
tests/pipeline/test_classifier.py
+16 -7
View File
@@ -1,3 +1,4 @@
from functools import reduce
from operator import or_
import numpy as np
@@ -260,17 +261,25 @@ class ClassifierTestCase(BasePipelineTestCase):
@parameter_space(
__fail_fast=True,
compval=['a', 'b', 'ab', 'not in the array'],
missing=['a', 'ab', '', 'not in the array'],
compval=[u'a', u'b', u'ab', u'not in the array'],
missing=[u'a', u'ab', u'', u'not in the array'],
labelarray_dtype=(categorical_dtype, bytes_dtype, unicode_dtype),
)
def test_string_elementwise_predicates(self,
compval,
missing,
labelarray_dtype):
if labelarray_dtype == bytes_dtype:
compval = compval.encode('utf-8')
missing = missing.encode('utf-8')
missing = labelarray_dtype.type(missing)
compval = labelarray_dtype.type(compval)
startswith_re = b'^' + compval + b'.*'
endswith_re = b'.*' + compval + b'$'
substring_re = b'.*' + compval + b'.*'
else:
startswith_re = '^' + compval + '.*'
endswith_re = '.*' + compval + '$'
substring_re = '.*' + compval + '.*'
class C(Classifier):
dtype = categorical_dtype
@@ -298,9 +307,9 @@ class ClassifierTestCase(BasePipelineTestCase):
'endswith': c.endswith(compval),
'has_substring': c.has_substring(compval),
# Equivalent filters using regex matching.
'startswith_re': c.matches('^' + compval + '.*'),
'endswith_re': c.matches('.*' + compval + '$'),
'has_substring_re': c.matches('.*' + compval + '.*'),
'startswith_re': c.matches(startswith_re),
'endswith_re': c.matches(endswith_re),
'has_substring_re': c.matches(substring_re),
}
expected = {
tests/test_labelarray.py
+6
View File
@@ -4,6 +4,7 @@ import numpy as np
from zipline.lib.labelarray import LabelArray
from zipline.testing import check_arrays, parameter_space, ZiplineTestCase
from zipline.utils.compat import unicode
def rotN(l, N):
@@ -67,10 +68,15 @@ class LabelArrayTestCase(ZiplineTestCase):
# using the ufunc.
notmissing = np.not_equal(strs, missing_value)
else:
if not isinstance(missing_value, array_astype):
missing_value = array_astype(missing_value, 'utf-8')
notmissing = (strs != missing_value)
arr = LabelArray(strs, missing_value=missing_value)
if not isinstance(compval, array_astype):
compval = array_astype(compval, 'utf-8')
# arr.missing_value should behave like NaN.
check_arrays(
arr == compval,
zipline/lib/_factorize.pyx
+10 -4
View File
@@ -2,7 +2,7 @@
Factorization algorithms.
"""
from numpy cimport ndarray, int64_t, PyArray_Check, import_array
from numpy import arange, asarray, empty, int64, isnan, ndarray
from numpy import arange, asarray, empty, int64, isnan, ndarray, zeros
import_array()
@@ -18,7 +18,7 @@ cpdef factorize_strings_known_categories(ndarray[object] values,
`missing_value`.
"""
if missing_value not in categories:
categories.append(missing_value)
categories.insert(0, missing_value)
if sort:
categories = sorted(categories)
@@ -46,6 +46,7 @@ cpdef factorize_strings_known_categories(ndarray[object] values,
return codes, asarray(categories, dtype=object), reverse_categories
cpdef factorize_strings(ndarray[object] values,
object missing_value,
int sort):
@@ -94,10 +95,15 @@ cpdef factorize_strings(ndarray[object] values,
cdef ndarray[int64_t, ndim=1] reverse_indexer
cdef int ncategories
cdef ndarray[object] categories_array = asarray(categories, dtype=object)
if sort:
# This is all taken from pandas.core.algorithms.factorize.
# This is all adapted from pandas.core.algorithms.factorize.
ncategories = len(categories_array)
sorter = categories_array.argsort()
sorter = zeros(ncategories, dtype=int64)
# Don't include missing_value in the argsort, because None is
# unorderable with bytes/str in py3. Always just sort it to 0.
sorter[1:] = categories_array[1:].argsort() + 1
reverse_indexer = empty(ncategories, dtype=int64)
reverse_indexer.put(sorter, arange(ncategories))