diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index 105ed219..a0faf6be 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -30,7 +30,6 @@ from zipline.pipeline.loaders.blaze import ( NoDeltasWarning, ) from zipline.pipeline.loaders.blaze.core import ( - NonNumpyField, NonPipelineField, no_deltas_rules, ) @@ -277,11 +276,11 @@ class BlazeToPipelineTestCase(TestCase): ) def test_cols_with_missing_vals(self): - dates = (pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-03')) * 3 + dates = (pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-03')) df = pd.DataFrame({ - 'sid': self.sids * 2, - 'value': (np.NaN, 0., 1., 2., 3., 2.,), - 'str_value': (None, "b", "c", "d", "e", "f"), + 'sid': self.sids[:-1], + 'value': (0., 1.,), + 'str_value': ("a", "b",), 'asof_date': dates, 'timestamp': dates, }) @@ -299,7 +298,21 @@ class BlazeToPipelineTestCase(TestCase): fields = OrderedDict(expr.dshape.measure.fields) with tmp_asset_finder() as finder: - expected = pd.DataFrame() + expected = pd.DataFrame( + np.array([["a", 0], + [None, np.NaN], + [None, np.NaN], + ["a", 0], + [None, np.NaN], + [None, np.NaN], + ["a", 0], + ["b", 1], + [None, np.NaN]]), + columns=['str_value', 'value'], + index=pd.MultiIndex.from_product( + (self.dates, finder.retrieve_all(self.sids)) + ) + ) self._test_id( df, var * Record(fields), @@ -369,7 +382,6 @@ class BlazeToPipelineTestCase(TestCase): dates, finder, ).run_pipeline(p, dates[0], dates[-1]) - import pdb; pdb.set_trace() assert_frame_equal( result, _utc_localize_index_level_0(expected), diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index fd265691..53504fe7 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -177,7 +177,7 @@ from zipline.utils.input_validation import ( ensure_timezone, optionally, ) -from zipline.utils.numpy_utils import repeat_last_axis, categorical_dtype +from zipline.utils.numpy_utils import categorical_dtype, repeat_last_axis from zipline.utils.pandas_utils import sort_values from zipline.utils.preprocess import preprocess @@ -314,7 +314,7 @@ def new_dataset(expr, deltas, missing_values): if isinstance(type_, Option): type_ = type_.ty type_ = type_.to_numpy_dtype() - if not isinstance(type_, String) and not can_represent_dtype(type_): + if not can_represent_dtype(type_): raise NotPipelineCompatible() col = Column( type_, @@ -1006,15 +1006,15 @@ class BlazeLoader(dict): ) else: last_in_group = last_in_group.reindex(dates) - import pdb; pdb.set_trace() - # str_cols = df.columns[df.dtypes == categorical_dtype] - # - # # Unstack will fill all missing values with NaN; we need to fix - # # this for strings. - # for col in str_cols: - # last_in_group.iloc[ - # :, last_in_group.columns.get_level_values(0) == col - # ].fillna('None') + # Unstack will fill all missing values with NaN; we need to fix + # this for strings. + if not df.empty: + str_cols = df.columns[df.dtypes == categorical_dtype] + + for col in str_cols: + last_in_group[col] = last_in_group[col].where(pd.notnull( + last_in_group[col]), None) + return last_in_group sparse_deltas = last_in_date_group(non_novel_deltas, reindex=False)