diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index fdfe495a..105ed219 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -128,7 +128,7 @@ class BlazeToPipelineTestCase(TestCase): with self.assertRaises(AttributeError) as e: getattr(ds, field) self.assertIn("'%s'" % field, str(e.exception)) - self.assertIn("'datetime'", str(e.exception)) + self.assertIn("'datetime64[us]'", str(e.exception)) # test memoization self.assertIs( @@ -254,34 +254,12 @@ class BlazeToPipelineTestCase(TestCase): ) self.assertIn(str(expr), str(e.exception)) - def test_non_numpy_field(self): - expr = bz.data( - [], - dshape=""" - var * { - a: datetime, - asof_date: datetime, - timestamp: datetime, - }""", - ) - ds = from_blaze( - expr, - loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, - ) - with self.assertRaises(AttributeError): - ds.a - self.assertIsInstance(object.__getattribute__(ds, 'a'), NonNumpyField) - def test_non_pipeline_field(self): - # NOTE: This test will fail if we ever allow string types in - # the Pipeline API. If this happens, change the dtype of the `a` field - # of expr to another type we don't allow. expr = bz.data( [], dshape=""" var * { - a: string, + a: complex, asof_date: datetime, timestamp: datetime, }""", @@ -298,6 +276,38 @@ class BlazeToPipelineTestCase(TestCase): NonPipelineField, ) + def test_cols_with_missing_vals(self): + dates = (pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-03')) * 3 + df = pd.DataFrame({ + 'sid': self.sids * 2, + 'value': (np.NaN, 0., 1., 2., 3., 2.,), + 'str_value': (None, "b", "c", "d", "e", "f"), + 'asof_date': dates, + 'timestamp': dates, + }) + expr = bz.data( + df, + dshape=""" + var * { + sid: int64, + value: float64, + str_value: string, + asof_date: datetime, + timestamp: datetime, + }""", + ) + fields = OrderedDict(expr.dshape.measure.fields) + + with tmp_asset_finder() as finder: + expected = pd.DataFrame() + self._test_id( + df, + var * Record(fields), + expected, + finder, + ('value', 'str_value'), + ) + def test_complex_expr(self): expr = bz.data(self.df, dshape=self.dshape) # put an Add in the table @@ -359,7 +369,7 @@ class BlazeToPipelineTestCase(TestCase): dates, finder, ).run_pipeline(p, dates[0], dates[-1]) - + import pdb; pdb.set_trace() assert_frame_equal( result, _utc_localize_index_level_0(expected), diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index f54e0a80..fd265691 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -177,7 +177,7 @@ from zipline.utils.input_validation import ( ensure_timezone, optionally, ) -from zipline.utils.numpy_utils import repeat_last_axis +from zipline.utils.numpy_utils import repeat_last_axis, categorical_dtype from zipline.utils.pandas_utils import sort_values from zipline.utils.preprocess import preprocess @@ -1006,7 +1006,15 @@ class BlazeLoader(dict): ) else: last_in_group = last_in_group.reindex(dates) - + import pdb; pdb.set_trace() + # str_cols = df.columns[df.dtypes == categorical_dtype] + # + # # Unstack will fill all missing values with NaN; we need to fix + # # this for strings. + # for col in str_cols: + # last_in_group.iloc[ + # :, last_in_group.columns.get_level_values(0) == col + # ].fillna('None') return last_in_group sparse_deltas = last_in_date_group(non_novel_deltas, reindex=False)