From a3df1e3cef0389cd6c0d26c96c92df56814991a7 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Thu, 3 Nov 2016 12:20:26 -0400 Subject: [PATCH] ENH: Allow configurable history prefetch length. To support using a `DataPortal` and `HistoryLoader` in a notebook, allow the prefetch length to be configurable, so that it can be set to 0. Unlike backtesting where the prefetch is useful for repeated history windows viewed from datetimes which are monotonically increasing by a small amount, the notebook usage of history windows needs only to retrieve the exact data needed for the window specified. This patch also fixes some boundary conditions related to rolls and adjustments which were uncovered by querying for the adjustments with an end date near the end of the window. --- tests/test_continuous_futures.py | 5 +++++ tests/test_history.py | 10 ++++++++++ zipline/assets/roll_finder.py | 6 ++++-- zipline/data/data_portal.py | 12 +++++++++++- zipline/data/history_loader.py | 23 +++++++++-------------- zipline/testing/fixtures.py | 13 ++++++++++++- 6 files changed, 51 insertions(+), 18 deletions(-) diff --git a/tests/test_continuous_futures.py b/tests/test_continuous_futures.py index b3c30d6e..4a2c46f2 100644 --- a/tests/test_continuous_futures.py +++ b/tests/test_continuous_futures.py @@ -972,3 +972,8 @@ class OrderedContractsTestCase(ZiplineTestCase): chain = oc.active_chain(4, pd.Timestamp('2015-01-04', tz='UTC').value) self.assertEquals([4], list(chain), "[4] should be active beginning at its start date.") + + +class NoPrefetchContinuousFuturesTestCase(ContinuousFuturesTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 diff --git a/tests/test_history.py b/tests/test_history.py index 233d879d..8745d661 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1351,6 +1351,11 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): format(field, minute)) +class NoPrefetchMinuteEquityHistoryTestCase(MinuteEquityHistoryTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 + + class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): CREATE_BARDATA_DATA_FREQUENCY = 'daily' @@ -1755,3 +1760,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): window_2[self.ASSET1].values) np.testing.assert_almost_equal(window_1[self.ASSET2].values, window_2[self.ASSET2].values) + + +class NoPrefetchDailyEquityHistoryTestCase(DailyEquityHistoryTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 diff --git a/zipline/assets/roll_finder.py b/zipline/assets/roll_finder.py index fe6ec5e5..c4633b72 100644 --- a/zipline/assets/roll_finder.py +++ b/zipline/assets/roll_finder.py @@ -88,8 +88,10 @@ class RollFinder(with_metaclass(ABCMeta, object)): for i, sid in enumerate(oc.contract_sids): if sid == first: break - rolls = [(first, None)] - sessions = self.trading_calendar.sessions_in_range(start, end) + rolls = [(first + offset, None)] + tc = self.trading_calendar + sessions = tc.sessions_in_range(tc.minute_to_session_label(start), + tc.minute_to_session_label(end)) if first == front: i -= 1 else: diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 5b89a2af..566b97fa 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -86,6 +86,12 @@ OHLCVP_FIELDS = frozenset([ HISTORY_FREQUENCIES = set(["1m", "1d"]) +DEFAULT_MINUTE_HISTORY_PREFETCH = 1560 +DEFAULT_DAILY_HISTORY_PREFETCH = 40 + +_DEF_M_HIST_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH +_DEF_D_HIST_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH + class DataPortal(object): """Interface to all of the data that a zipline simulation needs. @@ -138,7 +144,9 @@ class DataPortal(object): future_minute_reader=None, adjustment_reader=None, last_available_session=None, - last_available_minute=None): + last_available_minute=None, + minute_history_prefetch_length=_DEF_M_HIST_PREFETCH, + daily_history_prefetch_length=_DEF_D_HIST_PREFETCH): self.trading_calendar = trading_calendar self.asset_finder = asset_finder @@ -241,6 +249,7 @@ class DataPortal(object): self._adjustment_reader, self.asset_finder, self._roll_finders, + prefetch_length=daily_history_prefetch_length, ) self._minute_history_loader = MinuteHistoryLoader( self.trading_calendar, @@ -248,6 +257,7 @@ class DataPortal(object): self._adjustment_reader, self.asset_finder, self._roll_finders, + prefetch_length=minute_history_prefetch_length, ) self._first_trading_day = first_trading_day diff --git a/zipline/data/history_loader.py b/zipline/data/history_loader.py index 61816376..40efde29 100644 --- a/zipline/data/history_loader.py +++ b/zipline/data/history_loader.py @@ -307,7 +307,8 @@ class HistoryLoader(with_metaclass(ABCMeta)): def __init__(self, trading_calendar, reader, equity_adjustment_reader, asset_finder, roll_finders=None, - sid_cache_size=1000): + sid_cache_size=1000, + prefetch_length=0): self.trading_calendar = trading_calendar self._asset_finder = asset_finder self._reader = reader @@ -327,15 +328,12 @@ class HistoryLoader(with_metaclass(ABCMeta)): field: ExpiringCache(LRU(sid_cache_size)) for field in self.FIELDS } + self._prefetch_length = prefetch_length @abstractproperty def _frequency(self): pass - @abstractproperty - def _prefetch_length(self): - pass - @abstractproperty def _calendar(self): pass @@ -406,6 +404,11 @@ class HistoryLoader(with_metaclass(ABCMeta)): prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1) prefetch_end = cal[prefetch_end_ix] prefetch_dts = cal[start_ix:prefetch_end_ix + 1] + if is_perspective_after: + adj_end_ix = min(prefetch_end_ix + 1, len(cal) - 1) + adj_dts = cal[start_ix:adj_end_ix + 1] + else: + adj_dts = prefetch_dts prefetch_len = len(prefetch_dts) array = self._array(prefetch_dts, needed_assets, field) @@ -426,7 +429,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): adj_reader = None if adj_reader is not None: adjs = adj_reader.load_adjustments( - [field], prefetch_dts, [asset])[0] + [field], adj_dts, [asset])[0] else: adjs = {} window = window_type( @@ -539,10 +542,6 @@ class DailyHistoryLoader(HistoryLoader): def _frequency(self): return 'daily' - @property - def _prefetch_length(self): - return 40 - @property def _calendar(self): return self._reader.sessions @@ -562,10 +561,6 @@ class MinuteHistoryLoader(HistoryLoader): def _frequency(self): return 'minute' - @property - def _prefetch_length(self): - return 1560 - @lazyval def _calendar(self): mm = self.trading_calendar.all_minutes diff --git a/zipline/testing/fixtures.py b/zipline/testing/fixtures.py index b7435bbe..f983c8d6 100644 --- a/zipline/testing/fixtures.py +++ b/zipline/testing/fixtures.py @@ -13,7 +13,11 @@ from .core import ( create_minute_bar_data, tmp_dir, ) -from ..data.data_portal import DataPortal +from ..data.data_portal import ( + DataPortal, + DEFAULT_MINUTE_HISTORY_PREFETCH, + DEFAULT_DAILY_HISTORY_PREFETCH, +) from ..data.resample import ( minute_frame_to_session_frame, MinuteResampleSessionBarReader @@ -1272,6 +1276,9 @@ class WithDataPortal(WithAdjustmentReader, DATA_PORTAL_LAST_AVAILABLE_SESSION = None DATA_PORTAL_LAST_AVAILABLE_MINUTE = None + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH + DATA_PORTAL_DAILY_HISTORY_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH + def make_data_portal(self): if self.DATA_PORTAL_FIRST_TRADING_DAY is None: if self.DATA_PORTAL_USE_MINUTE_DATA: @@ -1315,6 +1322,10 @@ class WithDataPortal(WithAdjustmentReader, ), last_available_session=self.DATA_PORTAL_LAST_AVAILABLE_SESSION, last_available_minute=self.DATA_PORTAL_LAST_AVAILABLE_MINUTE, + minute_history_prefetch_length=self. + DATA_PORTAL_MINUTE_HISTORY_PREFETCH, + daily_history_prefetch_length=self. + DATA_PORTAL_DAILY_HISTORY_PREFETCH, ) def init_instance_fixtures(self):