From c8cf5a6761aac013a164fe475e7823f08615321a Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Thu, 26 May 2016 18:36:18 -0400 Subject: [PATCH 1/7] ENH: add ffill checkpointing to blaze core loader --- tests/pipeline/test_blaze.py | 166 ++++++++++++++--- zipline/pipeline/loaders/blaze/__init__.py | 4 +- zipline/pipeline/loaders/blaze/core.py | 207 ++++++++++++++------- 3 files changed, 282 insertions(+), 95 deletions(-) diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index 742a6b70..cf5cb1c8 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -26,12 +26,12 @@ from zipline.pipeline.engine import SimplePipelineEngine from zipline.pipeline.loaders.blaze import ( from_blaze, BlazeLoader, - NoDeltasWarning, + NoMetaDataWarning, ) from zipline.pipeline.loaders.blaze.core import ( NonPipelineField, - no_deltas_rules, ) +from zipline.testing import parameter_space from zipline.testing.fixtures import WithAssetFinder from zipline.utils.numpy_utils import ( float64_dtype, @@ -112,7 +112,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ds = from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ) self.assertEqual(ds.__name__, name) @@ -129,7 +130,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ), ds, @@ -141,7 +143,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): value = from_blaze( expr.value, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ) self.assertEqual(value.name, 'value') @@ -153,7 +156,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr.value, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ), value, @@ -162,7 +166,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ).value, value, @@ -173,7 +178,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ), value.dataset, @@ -196,32 +202,49 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', ) self.assertIn("'asof_date'", str(e.exception)) self.assertIn(repr(str(expr.dshape.measure)), str(e.exception)) - def test_auto_deltas(self): + @parameter_space(deltas={True, False}, checkpoints={True, False}) + def test_auto_metadata(self, deltas, checkpoints): + select_level = op.getitem(('ignore', 'raise')) + m = {'ds': self.df} + if deltas: + m['ds_deltas'] = pd.DataFrame(columns=self.df.columns), + if checkpoints: + m['ds_checkpoints'] = pd.DataFrame(columns=self.df.columns), expr = bz.data( - {'ds': self.df, - 'ds_deltas': pd.DataFrame(columns=self.df.columns)}, - dshape=var * Record(( - ('ds', self.dshape.measure), - ('ds_deltas', self.dshape.measure), - )), + m, + dshape=var * Record((k, self.dshape.measure) for k in m), ) loader = BlazeLoader() ds = from_blaze( expr.ds, loader=loader, missing_values=self.missing_values, + no_deltas_rule=select_level(deltas), + no_checkpoints_rule=select_level(checkpoints), ) self.assertEqual(len(loader), 1) exprdata = loader[ds] self.assertTrue(exprdata.expr.isidentical(expr.ds)) - self.assertTrue(exprdata.deltas.isidentical(expr.ds_deltas)) + if deltas: + self.assertTrue(exprdata.deltas.isidentical(expr.ds_deltas)) + else: + self.assertIsNone(exprdata.deltas) + if checkpoints: + self.assertTrue( + exprdata.checkpoints.isidentical(expr.ds_checkpoints), + ) + else: + self.assertIsNone(exprdata.checkpoints) - def test_auto_deltas_fail_warn(self): + @parameter_space(deltas={True, False}, checkpoints={True, False}) + def test_auto_metadata_fail_warn(self, deltas, checkpoints): + select_level = op.getitem(('ignore', 'warn')) with warnings.catch_warnings(record=True) as ws: warnings.simplefilter('always') loader = BlazeLoader() @@ -229,22 +252,31 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): from_blaze( expr, loader=loader, - no_deltas_rule=no_deltas_rules.warn, + no_deltas_rule=select_level(deltas), + no_checkpoints_rule=select_level(checkpoints), missing_values=self.missing_values, ) - self.assertEqual(len(ws), 1) - w = ws[0].message - self.assertIsInstance(w, NoDeltasWarning) - self.assertIn(str(expr), str(w)) + self.assertEqual(len(ws), deltas + checkpoints) - def test_auto_deltas_fail_raise(self): + for w in ws: + w = w.message + self.assertIsInstance(w, NoMetaDataWarning) + self.assertIn(str(expr), str(w)) + + @parameter_space(deltas={True, False}, checkpoints={True, False}) + def test_auto_metadata_fail_raise(self, deltas, checkpoints): + if not (deltas or checkpoints): + # not a real case + return + select_level = op.getitem(('ignore', 'raise')) loader = BlazeLoader() expr = bz.data(self.df, dshape=self.dshape) with self.assertRaises(ValueError) as e: from_blaze( expr, loader=loader, - no_deltas_rule=no_deltas_rules.raise_, + no_deltas_rule=select_level(deltas), + no_checkpoints_rule=select_level(checkpoints), ) self.assertIn(str(expr), str(e.exception)) @@ -261,7 +293,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ds = from_blaze( expr, loader=self.garbage_loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', ) with self.assertRaises(AttributeError): ds.a @@ -550,6 +583,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): deltas=None, loader=self.garbage_loader, missing_values=self.missing_values, + no_checkpoints_rule='ignore', ) with self.assertRaises(TypeError): @@ -558,6 +592,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): deltas=None, loader=self.garbage_loader, missing_values=self.missing_values, + no_checkpoints_rule='ignore', ) deltas = bz.data( @@ -570,6 +605,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): deltas=deltas, loader=self.garbage_loader, missing_values=self.missing_values, + no_checkpoints_rule='ignore', ) with self.assertRaises(TypeError): @@ -578,6 +614,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): deltas=deltas, loader=self.garbage_loader, missing_values=self.missing_values, + no_checkpoints_rule='ignore', ) def _test_id(self, df, dshape, expected, finder, add): @@ -586,7 +623,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ds = from_blaze( expr, loader=loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ) p = Pipeline() @@ -617,7 +655,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ds = from_blaze( expr, loader=loader, - no_deltas_rule=no_deltas_rules.ignore, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ) p = Pipeline() @@ -1044,6 +1083,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): def _run_pipeline(self, expr, deltas, + checkpoints, expected_views, expected_output, finder, @@ -1056,8 +1096,10 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ds = from_blaze( expr, deltas, + checkpoints, loader=loader, - no_deltas_rule=no_deltas_rules.raise_, + no_deltas_rule='raise', + no_checkpoints_rule='ignore', missing_values=self.missing_values, ) p = Pipeline() @@ -1070,7 +1112,11 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): window_length = window_length_ def compute(self, today, assets, out, data): - assert_array_almost_equal(data, expected_views[today]) + assert_array_almost_equal( + data, + expected_views[today], + err_msg=str(today), + ) out[:] = compute_fn(data) p.add(TestFactor(), 'value') @@ -1142,6 +1188,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self._run_pipeline( expr, deltas, + None, expected_views, expected_output, finder, @@ -1194,6 +1241,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self._run_pipeline( expr, deltas, + None, expected_views, expected_output, finder, @@ -1237,6 +1285,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self._run_pipeline( expr, deltas, + None, expected_views, expected_output, finder, @@ -1311,6 +1360,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self._run_pipeline( expr, deltas, + None, expected_views, expected_output, finder, @@ -1371,6 +1421,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self._run_pipeline( expr, deltas, + None, expected_views, expected_output, finder, @@ -1380,3 +1431,58 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): window_length=3, compute_fn=op.itemgetter(-1), ) + + def test_checkpoints(self): + dates = pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-04') + baseline = pd.DataFrame({ + 'value': [-1.0, 1.0], + 'asof_date': dates, + 'timestamp': dates, + }) + checkpoints_ts = pd.Timestamp('2014-01-02') + checkpoints = pd.DataFrame({ + 'value': [0.0], + 'asof_date': checkpoints_ts, + 'timestamp': checkpoints_ts, + }) + + asset_info = asset_infos[0][0] + nassets = len(asset_info) + expected_views = keymap(pd.Timestamp, { + '2014-01-03': repeat_last_axis( + np.array([0.0]), + nassets, + ), + '2014-01-04': repeat_last_axis( + np.array([1.0]), + nassets, + ), + }) + + with tmp_asset_finder(equities=asset_info) as finder: + expected_output = pd.DataFrame( + list(concatv([0.0] * nassets, [1.0] * nassets)), + index=pd.MultiIndex.from_product(( + sorted(expected_views.keys()), + finder.retrieve_all(asset_info.index), + )), + columns=('value',), + ) + + self._run_pipeline( + bz.data(baseline, name='expr', dshape=self.macro_dshape), + None, + bz.data( + checkpoints, + name='expr_checkpoints', + dshape=self.macro_dshape, + ), + expected_views, + expected_output, + finder, + calendar=pd.date_range('2014-01-01', '2014-01-04'), + start=checkpoints_ts + pd.Timedelta('1 days'), + end=dates[-1], + window_length=1, + compute_fn=op.itemgetter(-1), + ) diff --git a/zipline/pipeline/loaders/blaze/__init__.py b/zipline/pipeline/loaders/blaze/__init__.py index ec88396e..5f283b8f 100644 --- a/zipline/pipeline/loaders/blaze/__init__.py +++ b/zipline/pipeline/loaders/blaze/__init__.py @@ -1,6 +1,6 @@ from .core import ( BlazeLoader, - NoDeltasWarning, + NoMetaDataWarning, from_blaze, global_loader, ) @@ -9,5 +9,5 @@ __all__ = ( 'BlazeLoader', 'from_blaze', 'global_loader', - 'NoDeltasWarning', + 'NoMetaDataWarning', ) diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index 6fbe3a73..4b4d555b 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -127,6 +127,7 @@ from __future__ import division, absolute_import from abc import ABCMeta, abstractproperty from collections import namedtuple, defaultdict from copy import copy +import datetime from functools import partial from itertools import count import warnings @@ -169,7 +170,6 @@ from zipline.pipeline.loaders.utils import ( from zipline.pipeline.term import NotSpecified from zipline.lib.adjusted_array import AdjustedArray, can_represent_dtype from zipline.lib.adjustment import Float64Overwrite -from zipline.utils.enum import enum from zipline.utils.input_validation import ( expect_element, ensure_timezone, @@ -196,7 +196,7 @@ is_invalid_deltas_node = complement(flip(isinstance, valid_deltas_node_types)) get__name__ = op.attrgetter('__name__') -class ExprData(namedtuple('ExprData', 'expr deltas odo_kwargs')): +class ExprData(namedtuple('ExprData', 'expr deltas checkpoints odo_kwargs')): """A pair of expressions and data resources. The expresions will be computed using the resources as the starting scope. @@ -206,14 +206,17 @@ class ExprData(namedtuple('ExprData', 'expr deltas odo_kwargs')): The baseline values. deltas : Expr, optional The deltas for the data. + checkpoints : Expr, optional + The forward fill checkpoints for the data. odo_kwargs : dict, optional The keyword arguments to forward to the odo calls internally. """ - def __new__(cls, expr, deltas=None, odo_kwargs=None): + def __new__(cls, expr, deltas=None, checkpoints=None, odo_kwargs=None): return super(ExprData, cls).__new__( cls, expr, deltas, + checkpoints, odo_kwargs or {}, ) @@ -224,6 +227,7 @@ class ExprData(namedtuple('ExprData', 'expr deltas odo_kwargs')): return super(ExprData, cls).__repr__(cls( str(self.expr), str(self.deltas), + str(self.checkpoint), self.odo_kwargs, )) @@ -411,58 +415,66 @@ def _check_datetime_field(name, measure): ) -class NoDeltasWarning(UserWarning): - """Warning used to signal that no deltas could be found and none - were provided. +class NoMetaDataWarning(UserWarning): + """Warning used to signal that no deltas or checkpoints could be found and + none were provided. Parameters ---------- expr : Expr The expression that was searched. + field : {'deltas', 'checkpoints'} + The field that was looked up. """ - def __init__(self, expr): + def __init__(self, expr, field): self._expr = expr + self._field = field def __str__(self): - return 'No deltas could be inferred from expr: %s' % self._expr + return 'No %s could be inferred from expr: %s' % ( + self._field, + self._expr, + ) -no_deltas_rules = enum('warn', 'raise_', 'ignore') +no_metadata_rules = frozenset({'warn', 'raise', 'ignore'}) -def get_deltas(expr, deltas, no_deltas_rule): - """Find the correct deltas for the expression. +def _get_metadata(field, expr, metadata_expr, no_metadata_rule): + """Find the correct metadata expression for the expression. Parameters ---------- + field : {'deltas', 'checkpoints'} + The kind of metadata expr to lookup. expr : Expr The baseline expression. - deltas : Expr, 'auto', or None - The deltas argument. If this is 'auto', then the deltas table will + metadata_expr : Expr, 'auto', or None + The metadata argument. If this is 'auto', then the metadata table will be searched for by walking up the expression tree. If this cannot be reflected, then an action will be taken based on the - ``no_deltas_rule``. - no_deltas_rule : no_deltas_rule - How to handle the case where deltas='auto' but no deltas could be - found. + ``no_metadata_rule``. + no_metadata_rule : {'warn', 'raise', 'ignore'} + How to handle the case where the metadata_expr='auto' but no expr + could be found. Returns ------- - deltas : Expr or None + metadata : Expr or None The deltas table to use. """ - if isinstance(deltas, bz.Expr) or deltas != 'auto': - return deltas + if isinstance(metadata_expr, bz.Expr) or metadata_expr != 'auto': + return metadata_expr try: - return expr._child[(expr._name or '') + '_deltas'] + return expr._child['_'.join(((expr._name or ''), field))] except (ValueError, AttributeError): - if no_deltas_rule == no_deltas_rules.raise_: + if no_metadata_rule == 'raise': raise ValueError( - "no deltas table could be reflected for %s" % expr + "no %s table could be reflected for %s" % (field, expr) ) - elif no_deltas_rule == no_deltas_rules.warn: - warnings.warn(NoDeltasWarning(expr)) + elif no_metadata_rule == 'warn': + warnings.warn(NoMetaDataWarning(expr, field), stacklevel=4) return None @@ -502,26 +514,37 @@ def _ensure_timestamp_field(dataset_expr, deltas): return dataset_expr, deltas -@expect_element(no_deltas_rule=no_deltas_rules) +@expect_element( + no_deltas_rule=no_metadata_rules, + no_checkpoints_rule=no_metadata_rules, +) def from_blaze(expr, deltas='auto', + checkpoints='auto', loader=None, resources=None, odo_kwargs=None, missing_values=None, - no_deltas_rule=no_deltas_rules.warn): + no_deltas_rule='warn', + no_checkpoints_rule='warn'): """Create a Pipeline API object from a blaze expression. Parameters ---------- expr : Expr The blaze expression to use. - deltas : Expr or 'auto', optional + deltas : Expr, 'auto' or None, optional The expression to use for the point in time adjustments. If the string 'auto' is passed, a deltas expr will be looked up by stepping up the expression tree and looking for another field - with the name of ``expr`` + '_deltas'. If None is passed, no deltas - will be used. + with the name of ``expr._name`` + '_deltas'. If None is passed, no + deltas will be used. + deltas : Expr, 'auto' or None, optional + The expression to use for the forward fill checkpoints. + If the string 'auto' is passed, a checkpoints expr will be looked up + by stepping up the expression tree and looking for another field + with the name of ``expr._name`` + '_checkpoints'. If None is passed, + no checkpoints will be used. loader : BlazeLoader, optional The blaze loader to attach this pipeline dataset to. If None is passed, the global blaze loader is used. @@ -533,11 +556,16 @@ def from_blaze(expr, missing_values : dict[str -> any], optional A dict mapping column names to missing values for those columns. Missing values are required for integral columns. - no_deltas_rule : no_deltas_rule + no_deltas_rule : {'warn', 'raise', 'ignore'}, optional What should happen if ``deltas='auto'`` but no deltas can be found. 'warn' says to raise a warning but continue. 'raise' says to raise an exception if no deltas can be found. 'ignore' says take no action and proceed with no deltas. + no_checkpoints_rule : {'warn', 'raise', 'ignore'}, optional + What should happen if ``checkpoints='auto'`` but no checkpoints can be + found. 'warn' says to raise a warning but continue. + 'raise' says to raise an exception if no deltas can be found. + 'ignore' says take no action and proceed with no deltas. Returns ------- @@ -548,13 +576,28 @@ def from_blaze(expr, is passed, a ``BoundColumn`` on the dataset that would be constructed from passing the parent is returned. """ - deltas = get_deltas(expr, deltas, no_deltas_rule) - if deltas is not None: + deltas = _get_metadata( + 'deltas', + expr, + deltas, + no_deltas_rule, + ) + checkpoints = _get_metadata( + 'checkpoints', + expr, + checkpoints, + no_checkpoints_rule, + ) + if 'auto' in {deltas, checkpoints}: invalid_nodes = tuple(filter(is_invalid_deltas_node, expr._subterms())) if invalid_nodes: raise TypeError( - 'expression with deltas may only contain (%s) nodes,' + 'expression with %s may only contain (%s) nodes,' " found: %s" % ( + ' or '.join( + ['deltas'] if deltas is not None else [] + + ['checkpoints'] if checkpoints is not None else [], + ), ', '.join(map(get__name__, valid_deltas_node_types)), ', '.join( set(map(compose(get__name__, type), invalid_nodes)), @@ -632,6 +675,9 @@ def from_blaze(expr, bind_expression_to_resources(deltas, resources) if deltas is not None else None, + bind_expression_to_resources(checkpoints, resources) + if checkpoints is not None else + None, odo_kwargs=odo_kwargs, ) if single_column is not None: @@ -669,10 +715,12 @@ def overwrite_novel_deltas(baseline, deltas, dates): ) <= 1 novel_deltas = deltas.loc[novel_idx] non_novel_deltas = deltas.loc[~novel_idx] - return sort_values(pd.concat( + cat = pd.concat( (baseline, novel_deltas), ignore_index=True, - ), TS_FIELD_NAME), non_novel_deltas + ) + sort_values(cat, TS_FIELD_NAME, inplace=True) + return cat, non_novel_deltas def overwrite_from_dates(asof, dense_dates, sparse_dates, asset_idx, value): @@ -822,6 +870,31 @@ def adjustments_from_deltas_with_sids(dense_dates, return dict(adjustments) # no subclasses of dict +def _checkpoint_ts(lower_dt): + """Given a lower time bound for a query, get the date in the checkpoint + table to query for. + + Parameters + ---------- + lower_dt : datetime + The lower time bound for the query. + + Returns + ------- + checkpoint_ts : pd.Timestamp + The date in the checkpoint table to query for. + """ + date = lower_dt.date() + return pd.Timestamp.combine( + date.replace( + day=1, + month=(date.month - 2) % 12 + 1, + year=date.year - 1 if date.month == 1 else date.year, + ), + datetime.time(0), + ).tz_localize('US/Eastern') + + class BlazeLoader(dict): """A PipelineLoader for datasets constructed with ``from_blaze``. @@ -873,13 +946,14 @@ class BlazeLoader(dict): except ValueError: raise AssertionError('all columns must come from the same dataset') - expr, deltas, odo_kwargs = self[dataset] + expr, deltas, checkpoints, odo_kwargs = self[dataset] have_sids = SID_FIELD_NAME in expr.fields asset_idx = pd.Series(index=assets, data=np.arange(len(assets))) assets = list(map(int, assets)) # coerce from numpy.int64 added_query_fields = [AD_FIELD_NAME, TS_FIELD_NAME] + ( [SID_FIELD_NAME] if have_sids else [] ) + colnames = added_query_fields + list(map(getname, columns)) data_query_time = self._data_query_time data_query_tz = self._data_query_tz @@ -890,30 +964,15 @@ class BlazeLoader(dict): data_query_tz, ) - def where(e): - """Create the query to run against the resources. - - Parameters - ---------- - e : Expr - The baseline or deltas expression. - - Returns - ------- - q : Expr - The query to run. - """ - return e[ - (e[TS_FIELD_NAME] <= upper_dt) - ][added_query_fields + list(map(getname, columns))] - - def collect_expr(e): + def collect_expr(e, lower): """Materialize the expression as a dataframe. Parameters ---------- e : Expr The baseline or deltas expression. + lower : datetime + The lower time bound to query. Returns ------- @@ -925,17 +984,39 @@ class BlazeLoader(dict): This can return more data than needed. The in memory reindex will handle this. """ - df = odo(where(e), pd.DataFrame, **odo_kwargs) - df.sort(TS_FIELD_NAME, inplace=True) # sort for the groupby later - return df + predicate = e[TS_FIELD_NAME] <= upper_dt + if lower is not None: + predicate &= e[TS_FIELD_NAME] >= lower - materialized_expr = collect_expr(expr) + return odo(e[predicate][colnames], pd.DataFrame, **odo_kwargs) + + if checkpoints is not None: + ts = checkpoints[TS_FIELD_NAME] + checkpoints_ts = odo(ts[ts <= lower_dt].max(), pd.Timestamp) + if pd.isnull(checkpoints_ts): + materialized_checkpoints = pd.DataFrame(columns=colnames) + lower = lower_dt + else: + materialized_checkpoints = odo( + checkpoints[ts == checkpoints_ts][colnames], + pd.DataFrame, + **odo_kwargs + ) + lower = checkpoints_ts + else: + materialized_checkpoints = pd.DataFrame(columns=colnames) + lower = None + + materialized_expr = collect_expr(expr, lower) + if materialized_checkpoints is not None: + materialized_expr = pd.concat(( + materialized_checkpoints, + materialized_expr, + )) materialized_deltas = ( - collect_expr(deltas) + collect_expr(deltas, lower) if deltas is not None else - pd.DataFrame( - columns=added_query_fields + list(map(getname, columns)), - ) + pd.DataFrame(columns=colnames) ) # It's not guaranteed that assets returned by the engine will contain From 1c3ad7043183327f18f25953b4a792caca024f00 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Mon, 13 Jun 2016 18:26:08 -0400 Subject: [PATCH 2/7] DOC: docs for checkpoints --- docs/source/whatsnew/1.0.2.txt | 23 +++++++++++++++++++++++ zipline/pipeline/loaders/blaze/core.py | 15 ++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 docs/source/whatsnew/1.0.2.txt diff --git a/docs/source/whatsnew/1.0.2.txt b/docs/source/whatsnew/1.0.2.txt new file mode 100644 index 00000000..348dc076 --- /dev/null +++ b/docs/source/whatsnew/1.0.2.txt @@ -0,0 +1,23 @@ +Release 1.0.2 +------------- + +:Release: 1.0.2 +:Date: TBD + +Enhancements +~~~~~~~~~~~~ + +- Adds forward fill checkpoint tables for the blaze core loader. This allow the + loader to more efficiently forward fill the data by capping the lower date it + must search for when querying data. The checkpoints should have novel deltas + applied (:issue:`1276`). + +Bug Fixes +~~~~~~~~~ + +None + +Documentation +~~~~~~~~~~~~~ + +None diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index 4b4d555b..a23d5b70 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -1,4 +1,5 @@ -"""Blaze integration with the Pipeline API. +""" +Blaze integration with the Pipeline API. For an overview of the blaze project, see blaze.pydata.org @@ -81,6 +82,18 @@ actually 3. By pulling our data into these two tables and not silently updating our original table we can run our pipelines using the information we would have had on that day, and we can prevent lookahead bias in the pipelines. + +Another optional expression that may be provided is ``checkpoints``. The +``checkpoints`` expression is used when doing a forward fill query to cap the +lower date that must be searched. This expression has the same shape as the +``baseline`` and ``deltas`` expressions but should be downsampled with novel +deltas applied. For example, imagine we had one data point per asset per day +for some dataset. We could dramatically speed up our queries by pre populating +a downsampled version which has the most recently known value at the start of +each month. Then, when we query, we only must look back at most one month +before the start of the pipeline query to provide enough data to forward fill +correctly. + Conversion from Blaze to the Pipeline API ----------------------------------------- From cb67ee425ebf9b33d47133491c6ba9b8dd51d2aa Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Wed, 15 Jun 2016 13:53:43 -0400 Subject: [PATCH 3/7] TST: coverage --- tests/pipeline/test_blaze.py | 153 ++++++++++++++++++++++++- zipline/pipeline/loaders/blaze/core.py | 49 ++------ zipline/testing/predicates.py | 6 + 3 files changed, 167 insertions(+), 41 deletions(-) diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index cf5cb1c8..29885a30 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -21,7 +21,7 @@ from toolz.curried import operator as op from zipline.assets.synthetic import make_simple_equity_info from zipline.pipeline import Pipeline, CustomFactor -from zipline.pipeline.data import DataSet, BoundColumn +from zipline.pipeline.data import DataSet, BoundColumn, Column from zipline.pipeline.engine import SimplePipelineEngine from zipline.pipeline.loaders.blaze import ( from_blaze, @@ -29,16 +29,21 @@ from zipline.pipeline.loaders.blaze import ( NoMetaDataWarning, ) from zipline.pipeline.loaders.blaze.core import ( + ExprData, NonPipelineField, ) -from zipline.testing import parameter_space +from zipline.testing import ( + ZiplineTestCase, + parameter_space, + tmp_asset_finder, +) from zipline.testing.fixtures import WithAssetFinder from zipline.utils.numpy_utils import ( float64_dtype, int64_dtype, repeat_last_axis, ) -from zipline.testing import tmp_asset_finder, ZiplineTestCase +from zipline.testing.predicates import assert_equal, assert_isidentical nameof = op.attrgetter('name') dtypeof = op.attrgetter('dtype') @@ -208,6 +213,108 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): self.assertIn("'asof_date'", str(e.exception)) self.assertIn(repr(str(expr.dshape.measure)), str(e.exception)) + def test_missing_timestamp(self): + expr = bz.data( + self.df.loc[:, ['sid', 'value', 'asof_date']], + name='expr', + dshape=""" + var * { + sid: ?int64, + value: float64, + asof_date: datetime, + }""", + ) + + loader = BlazeLoader() + + from_blaze( + expr, + loader=loader, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', + ) + + self.assertEqual(len(loader), 1) + exprdata, = loader.values() + + assert_isidentical( + exprdata.expr, + bz.transform(expr, timestamp=expr.asof_date), + ) + + def test_from_blaze_no_resources_dataset_expr(self): + expr = bz.symbol('expr', self.dshape) + + with self.assertRaises(ValueError) as e: + from_blaze( + expr, + loader=self.garbage_loader, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', + missing_values=self.missing_values, + ) + assert_equal( + str(e.exception), + 'no resources provided to compute expr', + ) + + @parameter_space(metadata={'deltas', 'checkpoints'}) + def test_from_blaze_no_resources_metadata_expr(self, metadata): + expr = bz.data(self.df, name='expr', dshape=self.dshape) + metadata_expr = bz.symbol('metadata', self.dshape) + + with self.assertRaises(ValueError) as e: + from_blaze( + expr, + loader=self.garbage_loader, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', + missing_values=self.missing_values, + **{metadata: metadata_expr} + ) + assert_equal( + str(e.exception), + 'no resources provided to compute %s' % metadata, + ) + + def test_from_blaze_mixed_resources_dataset_expr(self): + expr = bz.data(self.df, name='expr', dshape=self.dshape) + + with self.assertRaises(ValueError) as e: + from_blaze( + expr, + resources={expr: self.df}, + loader=self.garbage_loader, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', + missing_values=self.missing_values, + ) + assert_equal( + str(e.exception), + 'explicit and implicit resources provided to compute expr', + ) + + @parameter_space(metadata={'deltas', 'checkpoints'}) + def test_from_blaze_mixed_resources_metadata_expr(self, metadata): + expr = bz.symbol('expr', self.dshape) + metadata_expr = bz.data(self.df, name=metadata, dshape=self.dshape) + + with self.assertRaises(ValueError) as e: + from_blaze( + expr, + resources={metadata_expr: self.df}, + loader=self.garbage_loader, + no_deltas_rule='ignore', + no_checkpoints_rule='ignore', + missing_values=self.missing_values, + **{metadata: metadata_expr} + ) + assert_equal( + str(e.exception), + 'explicit and implicit resources provided to compute %s' % + metadata, + ) + @parameter_space(deltas={True, False}, checkpoints={True, False}) def test_auto_metadata(self, deltas, checkpoints): select_level = op.getitem(('ignore', 'raise')) @@ -1486,3 +1593,43 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): window_length=1, compute_fn=op.itemgetter(-1), ) + + +class MiscTestCase(ZiplineTestCase): + def test_exprdata_repr(self): + strd = set() + + class BadRepr(object): + """A class which cannot be repr'd. + """ + def __init__(self, name): + self._name = name + + def __repr__(self): # pragma: no cover + raise AssertionError('ayy') + + def __str__(self): + strd.add(self) + return self._name + + assert_equal( + repr(ExprData( + expr=BadRepr('expr'), + deltas=BadRepr('deltas'), + checkpoints=BadRepr('checkpoints'), + odo_kwargs={'a': 'b'}, + )), + "ExprData(expr='expr', deltas='deltas'," + " checkpoints='checkpoints', odo_kwargs={'a': 'b'})", + ) + + def test_blaze_loader_repr(self): + assert_equal(repr(BlazeLoader()), '') + + def test_blaze_loader_lookup_failure(self): + class D(DataSet): + c = Column(dtype='float64') + + with self.assertRaises(KeyError) as e: + BlazeLoader()(D.c) + assert_equal(str(e.exception), 'D.c::float64') diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index a23d5b70..47cf5967 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -140,7 +140,6 @@ from __future__ import division, absolute_import from abc import ABCMeta, abstractproperty from collections import namedtuple, defaultdict from copy import copy -import datetime from functools import partial from itertools import count import warnings @@ -240,7 +239,7 @@ class ExprData(namedtuple('ExprData', 'expr deltas checkpoints odo_kwargs')): return super(ExprData, cls).__repr__(cls( str(self.expr), str(self.deltas), - str(self.checkpoint), + str(self.checkpoints), self.odo_kwargs, )) @@ -257,7 +256,7 @@ class InvalidField(with_metaclass(ABCMeta)): The shape of the field. """ @abstractproperty - def error_format(self): + def error_format(self): # pragma: no cover raise NotImplementedError('error_format') def __init__(self, field, type_): @@ -282,14 +281,6 @@ class NonPipelineField(InvalidField): ) -class NotPipelineCompatible(TypeError): - """Exception used to indicate that a dshape is not Pipeline API - compatible. - """ - def __str__(self): - return "'%s' is a non Pipeline API compatible type'" % self.args - - _new_names = ('BlazeDataSet_%d' % n for n in count()) @@ -369,7 +360,7 @@ def new_dataset(expr, deltas, missing_values): # unicode is a name error in py3 but the branch is only hit # when we are in python 2. - if PY2 and isinstance(name, unicode): # noqa + if PY2 and isinstance(name, unicode): # pragma: no cover # noqa name = name.encode('utf-8') return type(name, (DataSet,), columns) @@ -674,8 +665,9 @@ def from_blaze(expr, ) # Ensure that we have a data resource to execute the query against. - _check_resources('dataset_expr', dataset_expr, resources) + _check_resources('expr', dataset_expr, resources) _check_resources('deltas', deltas, resources) + _check_resources('checkpoints', checkpoints, resources) # Create or retrieve the Pipeline API dataset. if missing_values is None: @@ -883,31 +875,6 @@ def adjustments_from_deltas_with_sids(dense_dates, return dict(adjustments) # no subclasses of dict -def _checkpoint_ts(lower_dt): - """Given a lower time bound for a query, get the date in the checkpoint - table to query for. - - Parameters - ---------- - lower_dt : datetime - The lower time bound for the query. - - Returns - ------- - checkpoint_ts : pd.Timestamp - The date in the checkpoint table to query for. - """ - date = lower_dt.date() - return pd.Timestamp.combine( - date.replace( - day=1, - month=(date.month - 2) % 12 + 1, - year=date.year - 1 if date.month == 1 else date.year, - ), - datetime.time(0), - ).tz_localize('US/Eastern') - - class BlazeLoader(dict): """A PipelineLoader for datasets constructed with ``from_blaze``. @@ -945,6 +912,12 @@ class BlazeLoader(dict): return self raise KeyError(column) + def __repr__(self): + return '<%s: %s>' % ( + type(self).__name__, + super(BlazeLoader, self).__repr__(), + ) + def load_adjusted_array(self, columns, dates, assets, mask): return dict( concat(map( diff --git a/zipline/testing/predicates.py b/zipline/testing/predicates.py index 3bc35346..ed1c472c 100644 --- a/zipline/testing/predicates.py +++ b/zipline/testing/predicates.py @@ -380,6 +380,12 @@ def assert_timestamp_and_datetime_equal(result, ) +def assert_isidentical(result, expected, msg=''): + assert result.isidentical(expected), ( + '%s%s is not identical to %s' % (_fmt_msg(msg), result, expected) + ) + + try: # pull the dshape cases in from datashape.util.testing import assert_dshape_equal From caebdf7cfcd5ba3f7c89c70d330647e7e605fca4 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Mon, 20 Jun 2016 13:35:07 -0400 Subject: [PATCH 4/7] MAINT: shuffle the complex expression checks --- tests/pipeline/test_blaze.py | 48 +++++++++++----- zipline/pipeline/loaders/blaze/core.py | 80 ++++++++++++++++++-------- 2 files changed, 91 insertions(+), 37 deletions(-) diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index 29885a30..d54f9476 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -680,49 +680,71 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ) def test_complex_expr(self): - expr = bz.data(self.df, dshape=self.dshape) + expr = bz.data(self.df, dshape=self.dshape, name='expr') # put an Add in the table expr_with_add = bz.transform(expr, value=expr.value + 1) - # Test that we can have complex expressions with no deltas + # test that we can have complex expressions with no metadata from_blaze( expr_with_add, deltas=None, + checkpoints=None, loader=self.garbage_loader, missing_values=self.missing_values, no_checkpoints_rule='ignore', ) - with self.assertRaises(TypeError): + with self.assertRaises(TypeError) as e: + # test that we cannot create a single column from a non field from_blaze( expr.value + 1, # put an Add in the column deltas=None, + checkpoints=None, loader=self.garbage_loader, missing_values=self.missing_values, no_checkpoints_rule='ignore', ) + assert_equal( + str(e.exception), + "expression 'expr.value + 1' was array-like but not a simple field" + " of some larger table", + ) deltas = bz.data( pd.DataFrame(columns=self.df.columns), dshape=self.dshape, + name='deltas', + ) + checkpoints = bz.data( + pd.DataFrame(columns=self.df.columns), + dshape=self.dshape, + name='checkpoints', ) - with self.assertRaises(TypeError): - from_blaze( - expr_with_add, - deltas=deltas, - loader=self.garbage_loader, - missing_values=self.missing_values, - no_checkpoints_rule='ignore', - ) - with self.assertRaises(TypeError): + # test that we can have complex expressions with explicit metadata + from_blaze( + expr_with_add, + deltas=deltas, + checkpoints=checkpoints, + loader=self.garbage_loader, + missing_values=self.missing_values, + ) + + with self.assertRaises(TypeError) as e: + # test that we cannot create a single column from a non field + # even with explicit metadata from_blaze( expr.value + 1, deltas=deltas, + checkpoints=checkpoints, loader=self.garbage_loader, missing_values=self.missing_values, - no_checkpoints_rule='ignore', ) + assert_equal( + str(e.exception), + "expression 'expr.value + 1' was array-like but not a simple field" + " of some larger table", + ) def _test_id(self, df, dshape, expected, finder, add): expr = bz.data(df, name='expr', dshape=dshape) diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index 47cf5967..6a9ec792 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -482,7 +482,27 @@ def _get_metadata(field, expr, metadata_expr, no_metadata_rule): return None -def _ensure_timestamp_field(dataset_expr, deltas): +def _ad_as_ts(expr): + """Duplicate the asof_date column as the timestamp column. + + Parameters + ---------- + expr : Expr or None + The expression to change the columns of. + + Returns + ------- + transformed : Expr or None + The transformed expression or None if ``expr`` is None. + """ + return ( + None + if expr is None else + bz.transform(expr, **{TS_FIELD_NAME: expr[AD_FIELD_NAME]}) + ) + + +def _ensure_timestamp_field(dataset_expr, deltas, checkpoints): """Verify that the baseline and deltas expressions have a timestamp field. If there is not a ``TS_FIELD_NAME`` on either of the expressions, it will @@ -495,6 +515,8 @@ def _ensure_timestamp_field(dataset_expr, deltas): The baseline expression. deltas : Expr or None The deltas expression if any was provided. + checkpoints : Expr or None + The checkpoints expression if any was provided. Returns ------- @@ -507,15 +529,12 @@ def _ensure_timestamp_field(dataset_expr, deltas): dataset_expr, **{TS_FIELD_NAME: dataset_expr[AD_FIELD_NAME]} ) - if deltas is not None: - deltas = bz.transform( - deltas, - **{TS_FIELD_NAME: deltas[AD_FIELD_NAME]} - ) + deltas = _ad_as_ts(deltas) + checkpoints = _ad_as_ts(checkpoints) else: _check_datetime_field(TS_FIELD_NAME, measure) - return dataset_expr, deltas + return dataset_expr, deltas, checkpoints @expect_element( @@ -580,6 +599,22 @@ def from_blaze(expr, is passed, a ``BoundColumn`` on the dataset that would be constructed from passing the parent is returned. """ + if 'auto' in {deltas, checkpoints}: + invalid_nodes = tuple(filter(is_invalid_deltas_node, expr._subterms())) + if invalid_nodes: + raise TypeError( + 'expression with auto %s may only contain (%s) nodes,' + " found: %s" % ( + ' or '.join( + ['deltas'] if deltas is not None else [] + + ['checkpoints'] if checkpoints is not None else [], + ), + ', '.join(map(get__name__, valid_deltas_node_types)), + ', '.join( + set(map(compose(get__name__, type), invalid_nodes)), + ), + ), + ) deltas = _get_metadata( 'deltas', expr, @@ -592,22 +627,6 @@ def from_blaze(expr, checkpoints, no_checkpoints_rule, ) - if 'auto' in {deltas, checkpoints}: - invalid_nodes = tuple(filter(is_invalid_deltas_node, expr._subterms())) - if invalid_nodes: - raise TypeError( - 'expression with %s may only contain (%s) nodes,' - " found: %s" % ( - ' or '.join( - ['deltas'] if deltas is not None else [] + - ['checkpoints'] if checkpoints is not None else [], - ), - ', '.join(map(get__name__, valid_deltas_node_types)), - ', '.join( - set(map(compose(get__name__, type), invalid_nodes)), - ), - ), - ) # Check if this is a single column out of a dataset. if bz.ndim(expr) != 1: @@ -653,7 +672,11 @@ def from_blaze(expr, ), ) _check_datetime_field(AD_FIELD_NAME, measure) - dataset_expr, deltas = _ensure_timestamp_field(dataset_expr, deltas) + dataset_expr, deltas, checkpoints = _ensure_timestamp_field( + dataset_expr, + deltas, + checkpoints, + ) if deltas is not None and (sorted(deltas.dshape.measure.fields) != sorted(measure.fields)): @@ -663,6 +686,15 @@ def from_blaze(expr, deltas.dshape.measure, ), ) + if (checkpoints is not None and + (sorted(checkpoints.dshape.measure.fields) != + sorted(measure.fields))): + raise TypeError( + 'baseline measure != checkpoints measure:\n%s != %s' % ( + measure, + deltas.dshape.measure, + ), + ) # Ensure that we have a data resource to execute the query against. _check_resources('expr', dataset_expr, resources) From c2723f3487657c0b82e2102dbcdc4e73031609f2 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Tue, 21 Jun 2016 13:20:43 -0400 Subject: [PATCH 5/7] BUG: query everything if deltas are empty --- zipline/pipeline/loaders/blaze/core.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index 6a9ec792..a843d232 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -465,9 +465,9 @@ def _get_metadata(field, expr, metadata_expr, no_metadata_rule): Returns ------- metadata : Expr or None - The deltas table to use. + The the deltas or metadata table to use. """ - if isinstance(metadata_expr, bz.Expr) or metadata_expr != 'auto': + if isinstance(metadata_expr, bz.Expr) or metadata_expr is None: return metadata_expr try: @@ -692,7 +692,7 @@ def from_blaze(expr, raise TypeError( 'baseline measure != checkpoints measure:\n%s != %s' % ( measure, - deltas.dshape.measure, + checkpoints.dshape.measure, ), ) @@ -755,6 +755,7 @@ def overwrite_novel_deltas(baseline, deltas, dates): cat = pd.concat( (baseline, novel_deltas), ignore_index=True, + copy=False, ) sort_values(cat, TS_FIELD_NAME, inplace=True) return cat, non_novel_deltas @@ -1013,7 +1014,7 @@ class BlazeLoader(dict): checkpoints_ts = odo(ts[ts <= lower_dt].max(), pd.Timestamp) if pd.isnull(checkpoints_ts): materialized_checkpoints = pd.DataFrame(columns=colnames) - lower = lower_dt + lower = None else: materialized_checkpoints = odo( checkpoints[ts == checkpoints_ts][colnames], @@ -1027,10 +1028,14 @@ class BlazeLoader(dict): materialized_expr = collect_expr(expr, lower) if materialized_checkpoints is not None: - materialized_expr = pd.concat(( - materialized_checkpoints, - materialized_expr, - )) + materialized_expr = pd.concat( + ( + materialized_checkpoints, + materialized_expr, + ), + ignore_index=True, + copy=False, + ) materialized_deltas = ( collect_expr(deltas, lower) if deltas is not None else From cb266b983ab6f3fdc772537b38bd385f8d1f0a83 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Tue, 21 Jun 2016 13:20:52 -0400 Subject: [PATCH 6/7] TST: more test for checkpoints --- tests/pipeline/test_blaze.py | 214 +++++++++++++++++++++++++++++------ 1 file changed, 180 insertions(+), 34 deletions(-) diff --git a/tests/pipeline/test_blaze.py b/tests/pipeline/test_blaze.py index d54f9476..3a26cb7e 100644 --- a/tests/pipeline/test_blaze.py +++ b/tests/pipeline/test_blaze.py @@ -59,6 +59,7 @@ asset_infos = ( pd.Timestamp('2015'), ),), ) +simple_asset_info = asset_infos[0][0] with_extra_sid = parameterized.expand(asset_infos) with_ignore_sid = parameterized.expand( product(chain.from_iterable(asset_infos), [True, False]) @@ -111,6 +112,13 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): cls.garbage_loader = BlazeLoader() cls.missing_values = {'int_value': 0} + cls.value_dshape = dshape("""var * { + sid: ?int64, + value: float64, + asof_date: datetime, + timestamp: datetime, + }""") + def test_tabular(self): name = 'expr' expr = bz.data(self.df, name=name, dshape=self.dshape) @@ -195,9 +203,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): expr = bz.data( self.df.loc[:, ['sid', 'value', 'timestamp']], name='expr', - dshape=""" - var * { - sid: ?int64, + dshape="""var * { + sid: int64, value: float64, timestamp: datetime, }""", @@ -217,9 +224,8 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): expr = bz.data( self.df.loc[:, ['sid', 'value', 'asof_date']], name='expr', - dshape=""" - var * { - sid: ?int64, + dshape="""var * { + sid: int64, value: float64, asof_date: datetime, }""", @@ -977,13 +983,12 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): Equity(66 [B]) 2 Equity(67 [C]) 2 """ - asset_info = asset_infos[0][0] - nassets = len(asset_info) + nassets = len(simple_asset_info) expected = pd.DataFrame( list(concatv([0] * nassets, [1] * nassets, [2] * nassets)), index=pd.MultiIndex.from_product(( self.macro_df.timestamp, - self.asset_finder.retrieve_all(asset_info.index), + self.asset_finder.retrieve_all(simple_asset_info.index), )), columns=('value',), ) @@ -1075,15 +1080,14 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): fields = OrderedDict(self.macro_dshape.measure.fields) fields['other'] = fields['value'] - asset_info = asset_infos[0][0] - with tmp_asset_finder(equities=asset_info) as finder: + with tmp_asset_finder(equities=simple_asset_info) as finder: expected = pd.DataFrame( np.array([[0, 1], [1, 2], [2, 3]]).repeat(3, axis=0), index=pd.MultiIndex.from_product(( df.timestamp, - finder.retrieve_all(asset_info.index), + finder.retrieve_all(simple_asset_info.index), )), columns=('value', 'other'), ).sort_index(axis=1) @@ -1382,7 +1386,6 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ) def test_deltas_macro(self): - asset_info = asset_infos[0][0] expr = bz.data(self.macro_df, name='expr', dshape=self.macro_dshape) deltas = bz.data( self.macro_df.iloc[:-1], @@ -1395,18 +1398,18 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): timestamp=deltas.timestamp + timedelta(days=1), ) - nassets = len(asset_info) + nassets = len(simple_asset_info) expected_views = keymap(pd.Timestamp, { '2014-01-02': repeat_last_axis(np.array([10.0, 1.0]), nassets), '2014-01-03': repeat_last_axis(np.array([11.0, 2.0]), nassets), }) - with tmp_asset_finder(equities=asset_info) as finder: + with tmp_asset_finder(equities=simple_asset_info) as finder: expected_output = pd.DataFrame( list(concatv([10] * nassets, [11] * nassets)), index=pd.MultiIndex.from_product(( sorted(expected_views.keys()), - finder.retrieve_all(asset_info.index), + finder.retrieve_all(simple_asset_info.index), )), columns=('value',), ) @@ -1501,7 +1504,6 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ) def test_novel_deltas_macro(self): - asset_info = asset_infos[0][0] base_dates = pd.DatetimeIndex([ pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-04') @@ -1519,7 +1521,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): timestamp=deltas.timestamp + timedelta(days=1), ) - nassets = len(asset_info) + nassets = len(simple_asset_info) expected_views = keymap(pd.Timestamp, { '2014-01-03': repeat_last_axis( np.array([10.0, 10.0, 10.0]), @@ -1538,12 +1540,12 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): # omitting the 4th and 5th to simulate a weekend pd.Timestamp('2014-01-06'), ]) - with tmp_asset_finder(equities=asset_info) as finder: + with tmp_asset_finder(equities=simple_asset_info) as finder: expected_output = pd.DataFrame( list(concatv([10] * nassets, [11] * nassets)), index=pd.MultiIndex.from_product(( sorted(expected_views.keys()), - finder.retrieve_all(asset_info.index), + finder.retrieve_all(simple_asset_info.index), )), columns=('value',), ) @@ -1561,25 +1563,31 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): compute_fn=op.itemgetter(-1), ) - def test_checkpoints(self): + def _test_checkpoints_macro(self, checkpoints, ffilled_value=-1.0): + """Simple checkpoints test that accepts a checkpoints dataframe and + the expected value for 2014-01-03 for macro datasets. + + The underlying data has value -1.0 on 2014-01-01 and 1.0 on 2014-01-04. + + Parameters + ---------- + checkpoints : pd.DataFrame + The checkpoints data. + ffilled_value : float, optional + The value to be read on the third, if not provided, it will be the + value in the base data that will be naturally ffilled there. + """ dates = pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-04') baseline = pd.DataFrame({ 'value': [-1.0, 1.0], 'asof_date': dates, 'timestamp': dates, }) - checkpoints_ts = pd.Timestamp('2014-01-02') - checkpoints = pd.DataFrame({ - 'value': [0.0], - 'asof_date': checkpoints_ts, - 'timestamp': checkpoints_ts, - }) - asset_info = asset_infos[0][0] - nassets = len(asset_info) + nassets = len(simple_asset_info) expected_views = keymap(pd.Timestamp, { '2014-01-03': repeat_last_axis( - np.array([0.0]), + np.array([ffilled_value]), nassets, ), '2014-01-04': repeat_last_axis( @@ -1588,12 +1596,12 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): ), }) - with tmp_asset_finder(equities=asset_info) as finder: + with tmp_asset_finder(equities=simple_asset_info) as finder: expected_output = pd.DataFrame( - list(concatv([0.0] * nassets, [1.0] * nassets)), + list(concatv([ffilled_value] * nassets, [1.0] * nassets)), index=pd.MultiIndex.from_product(( sorted(expected_views.keys()), - finder.retrieve_all(asset_info.index), + finder.retrieve_all(simple_asset_info.index), )), columns=('value',), ) @@ -1610,12 +1618,150 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase): expected_output, finder, calendar=pd.date_range('2014-01-01', '2014-01-04'), - start=checkpoints_ts + pd.Timedelta('1 days'), + start=pd.Timestamp('2014-01-03'), end=dates[-1], window_length=1, compute_fn=op.itemgetter(-1), ) + def test_checkpoints_macro(self): + ffilled_value = 0.0 + + checkpoints_ts = pd.Timestamp('2014-01-02') + checkpoints = pd.DataFrame({ + 'value': [ffilled_value], + 'asof_date': checkpoints_ts, + 'timestamp': checkpoints_ts, + }) + + self._test_checkpoints_macro(checkpoints, ffilled_value) + + def test_empty_checkpoints_macro(self): + empty_checkpoints = pd.DataFrame({ + 'value': [], + 'asof_date': [], + 'timestamp': [], + }) + + self._test_checkpoints_macro(empty_checkpoints) + + def test_checkpoints_out_of_bounds_macro(self): + # provide two checkpoints, one before the data in the base table + # and one after, these should not affect the value on the third + dates = pd.to_datetime(['2013-12-31', '2014-01-05']) + checkpoints = pd.DataFrame({ + 'value': [-2, 2], + 'asof_date': dates, + 'timestamp': dates, + }) + + self._test_checkpoints_macro(checkpoints) + + def _test_checkpoints(self, checkpoints, ffilled_values=None): + """Simple checkpoints test that accepts a checkpoints dataframe and + the expected value for 2014-01-03. + + The underlying data has value -1.0 on 2014-01-01 and 1.0 on 2014-01-04. + + Parameters + ---------- + checkpoints : pd.DataFrame + The checkpoints data. + ffilled_value : float, optional + The value to be read on the third, if not provided, it will be the + value in the base data that will be naturally ffilled there. + """ + nassets = len(simple_asset_info) + + dates = pd.to_datetime(['2014-01-01', '2014-01-04']) + dates_repeated = np.tile(dates, nassets) + values = np.arange(nassets) + 1 + values = np.hstack((values[::-1], values)) + baseline = pd.DataFrame({ + 'sid': np.tile(simple_asset_info.index, 2), + 'value': values, + 'asof_date': dates_repeated, + 'timestamp': dates_repeated, + }) + + if ffilled_values is None: + ffilled_values = baseline.value.iloc[:nassets] + + updated_values = baseline.value.iloc[nassets:] + + expected_views = keymap(pd.Timestamp, { + '2014-01-03': [ffilled_values], + '2014-01-04': [updated_values], + }) + + with tmp_asset_finder(equities=simple_asset_info) as finder: + expected_output = pd.DataFrame( + list(concatv(ffilled_values, updated_values)), + index=pd.MultiIndex.from_product(( + sorted(expected_views.keys()), + finder.retrieve_all(simple_asset_info.index), + )), + columns=('value',), + ) + + self._run_pipeline( + bz.data(baseline, name='expr', dshape=self.value_dshape), + None, + bz.data( + checkpoints, + name='expr_checkpoints', + dshape=self.value_dshape, + ), + expected_views, + expected_output, + finder, + calendar=pd.date_range('2014-01-01', '2014-01-04'), + start=pd.Timestamp('2014-01-03'), + end=dates[-1], + window_length=1, + compute_fn=op.itemgetter(-1), + ) + + def test_checkpoints(self): + nassets = len(simple_asset_info) + ffilled_values = (np.arange(nassets, dtype=np.float64) + 1) * 10 + dates = [pd.Timestamp('2014-01-02')] * nassets + checkpoints = pd.DataFrame({ + 'sid': simple_asset_info.index, + 'value': ffilled_values, + 'asof_date': dates, + 'timestamp': dates, + }) + + self._test_checkpoints(checkpoints, ffilled_values) + + def test_empty_checkpoints(self): + checkpoints = pd.DataFrame({ + 'sid': [], + 'value': [], + 'asof_date': [], + 'timestamp': [], + }) + + self._test_checkpoints(checkpoints) + + def test_checkpoints_out_of_bounds(self): + nassets = len(simple_asset_info) + # provide two sets of checkpoints, one before the data in the base + # table and one after, these should not affect the value on the third + dates = pd.to_datetime(['2013-12-31', '2014-01-05']) + dates_repeated = np.tile(dates, nassets) + ffilled_values = (np.arange(nassets) + 2) * 10 + ffilled_values = np.hstack((ffilled_values[::-1], ffilled_values)) + checkpoints = pd.DataFrame({ + 'sid': np.tile(simple_asset_info.index, 2), + 'value': ffilled_values, + 'asof_date': dates_repeated, + 'timestamp': dates_repeated, + }) + + self._test_checkpoints(checkpoints) + class MiscTestCase(ZiplineTestCase): def test_exprdata_repr(self): From b210acb1217f136081b7f08360d1352107bb19c3 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Tue, 21 Jun 2016 14:00:40 -0400 Subject: [PATCH 7/7] DOC: typo --- zipline/pipeline/loaders/blaze/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zipline/pipeline/loaders/blaze/core.py b/zipline/pipeline/loaders/blaze/core.py index a843d232..12832616 100644 --- a/zipline/pipeline/loaders/blaze/core.py +++ b/zipline/pipeline/loaders/blaze/core.py @@ -465,7 +465,7 @@ def _get_metadata(field, expr, metadata_expr, no_metadata_rule): Returns ------- metadata : Expr or None - The the deltas or metadata table to use. + The deltas or metadata table to use. """ if isinstance(metadata_expr, bz.Expr) or metadata_expr is None: return metadata_expr