diff --git a/tests/test_continuous_futures.py b/tests/test_continuous_futures.py index b3c30d6e..4a2c46f2 100644 --- a/tests/test_continuous_futures.py +++ b/tests/test_continuous_futures.py @@ -972,3 +972,8 @@ class OrderedContractsTestCase(ZiplineTestCase): chain = oc.active_chain(4, pd.Timestamp('2015-01-04', tz='UTC').value) self.assertEquals([4], list(chain), "[4] should be active beginning at its start date.") + + +class NoPrefetchContinuousFuturesTestCase(ContinuousFuturesTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 diff --git a/tests/test_history.py b/tests/test_history.py index 233d879d..8745d661 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1351,6 +1351,11 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): format(field, minute)) +class NoPrefetchMinuteEquityHistoryTestCase(MinuteEquityHistoryTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 + + class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): CREATE_BARDATA_DATA_FREQUENCY = 'daily' @@ -1755,3 +1760,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): window_2[self.ASSET1].values) np.testing.assert_almost_equal(window_1[self.ASSET2].values, window_2[self.ASSET2].values) + + +class NoPrefetchDailyEquityHistoryTestCase(DailyEquityHistoryTestCase): + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0 + DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0 diff --git a/zipline/assets/roll_finder.py b/zipline/assets/roll_finder.py index fe6ec5e5..c4633b72 100644 --- a/zipline/assets/roll_finder.py +++ b/zipline/assets/roll_finder.py @@ -88,8 +88,10 @@ class RollFinder(with_metaclass(ABCMeta, object)): for i, sid in enumerate(oc.contract_sids): if sid == first: break - rolls = [(first, None)] - sessions = self.trading_calendar.sessions_in_range(start, end) + rolls = [(first + offset, None)] + tc = self.trading_calendar + sessions = tc.sessions_in_range(tc.minute_to_session_label(start), + tc.minute_to_session_label(end)) if first == front: i -= 1 else: diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 5b89a2af..566b97fa 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -86,6 +86,12 @@ OHLCVP_FIELDS = frozenset([ HISTORY_FREQUENCIES = set(["1m", "1d"]) +DEFAULT_MINUTE_HISTORY_PREFETCH = 1560 +DEFAULT_DAILY_HISTORY_PREFETCH = 40 + +_DEF_M_HIST_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH +_DEF_D_HIST_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH + class DataPortal(object): """Interface to all of the data that a zipline simulation needs. @@ -138,7 +144,9 @@ class DataPortal(object): future_minute_reader=None, adjustment_reader=None, last_available_session=None, - last_available_minute=None): + last_available_minute=None, + minute_history_prefetch_length=_DEF_M_HIST_PREFETCH, + daily_history_prefetch_length=_DEF_D_HIST_PREFETCH): self.trading_calendar = trading_calendar self.asset_finder = asset_finder @@ -241,6 +249,7 @@ class DataPortal(object): self._adjustment_reader, self.asset_finder, self._roll_finders, + prefetch_length=daily_history_prefetch_length, ) self._minute_history_loader = MinuteHistoryLoader( self.trading_calendar, @@ -248,6 +257,7 @@ class DataPortal(object): self._adjustment_reader, self.asset_finder, self._roll_finders, + prefetch_length=minute_history_prefetch_length, ) self._first_trading_day = first_trading_day diff --git a/zipline/data/history_loader.py b/zipline/data/history_loader.py index 61816376..40efde29 100644 --- a/zipline/data/history_loader.py +++ b/zipline/data/history_loader.py @@ -307,7 +307,8 @@ class HistoryLoader(with_metaclass(ABCMeta)): def __init__(self, trading_calendar, reader, equity_adjustment_reader, asset_finder, roll_finders=None, - sid_cache_size=1000): + sid_cache_size=1000, + prefetch_length=0): self.trading_calendar = trading_calendar self._asset_finder = asset_finder self._reader = reader @@ -327,15 +328,12 @@ class HistoryLoader(with_metaclass(ABCMeta)): field: ExpiringCache(LRU(sid_cache_size)) for field in self.FIELDS } + self._prefetch_length = prefetch_length @abstractproperty def _frequency(self): pass - @abstractproperty - def _prefetch_length(self): - pass - @abstractproperty def _calendar(self): pass @@ -406,6 +404,11 @@ class HistoryLoader(with_metaclass(ABCMeta)): prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1) prefetch_end = cal[prefetch_end_ix] prefetch_dts = cal[start_ix:prefetch_end_ix + 1] + if is_perspective_after: + adj_end_ix = min(prefetch_end_ix + 1, len(cal) - 1) + adj_dts = cal[start_ix:adj_end_ix + 1] + else: + adj_dts = prefetch_dts prefetch_len = len(prefetch_dts) array = self._array(prefetch_dts, needed_assets, field) @@ -426,7 +429,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): adj_reader = None if adj_reader is not None: adjs = adj_reader.load_adjustments( - [field], prefetch_dts, [asset])[0] + [field], adj_dts, [asset])[0] else: adjs = {} window = window_type( @@ -539,10 +542,6 @@ class DailyHistoryLoader(HistoryLoader): def _frequency(self): return 'daily' - @property - def _prefetch_length(self): - return 40 - @property def _calendar(self): return self._reader.sessions @@ -562,10 +561,6 @@ class MinuteHistoryLoader(HistoryLoader): def _frequency(self): return 'minute' - @property - def _prefetch_length(self): - return 1560 - @lazyval def _calendar(self): mm = self.trading_calendar.all_minutes diff --git a/zipline/testing/fixtures.py b/zipline/testing/fixtures.py index b7435bbe..f983c8d6 100644 --- a/zipline/testing/fixtures.py +++ b/zipline/testing/fixtures.py @@ -13,7 +13,11 @@ from .core import ( create_minute_bar_data, tmp_dir, ) -from ..data.data_portal import DataPortal +from ..data.data_portal import ( + DataPortal, + DEFAULT_MINUTE_HISTORY_PREFETCH, + DEFAULT_DAILY_HISTORY_PREFETCH, +) from ..data.resample import ( minute_frame_to_session_frame, MinuteResampleSessionBarReader @@ -1272,6 +1276,9 @@ class WithDataPortal(WithAdjustmentReader, DATA_PORTAL_LAST_AVAILABLE_SESSION = None DATA_PORTAL_LAST_AVAILABLE_MINUTE = None + DATA_PORTAL_MINUTE_HISTORY_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH + DATA_PORTAL_DAILY_HISTORY_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH + def make_data_portal(self): if self.DATA_PORTAL_FIRST_TRADING_DAY is None: if self.DATA_PORTAL_USE_MINUTE_DATA: @@ -1315,6 +1322,10 @@ class WithDataPortal(WithAdjustmentReader, ), last_available_session=self.DATA_PORTAL_LAST_AVAILABLE_SESSION, last_available_minute=self.DATA_PORTAL_LAST_AVAILABLE_MINUTE, + minute_history_prefetch_length=self. + DATA_PORTAL_MINUTE_HISTORY_PREFETCH, + daily_history_prefetch_length=self. + DATA_PORTAL_DAILY_HISTORY_PREFETCH, ) def init_instance_fixtures(self):