ENH: Allow configurable history prefetch length.

To support using a `DataPortal` and `HistoryLoader` in a notebook, allow
the prefetch length to be configurable, so that it can be set to 0.
Unlike backtesting where the prefetch is useful for repeated history
windows viewed from datetimes which are monotonically increasing by a
small amount, the notebook usage of history windows needs only to
retrieve the exact data needed for the window specified.

This patch also fixes some boundary conditions related to rolls and
adjustments which were uncovered by querying for the adjustments with an
end date near the end of the window.

tests/test_continuous_futures.py

@@ -972,3 +972,8 @@ class OrderedContractsTestCase(ZiplineTestCase):
         chain = oc.active_chain(4, pd.Timestamp('2015-01-04', tz='UTC').value)
         self.assertEquals([4], list(chain),
                           "[4] should be active beginning at its start date.")
+
+
+class NoPrefetchContinuousFuturesTestCase(ContinuousFuturesTestCase):
+    DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
+    DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0

tests/test_history.py

@@ -1351,6 +1351,11 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                                            format(field, minute))
 
 
+class NoPrefetchMinuteEquityHistoryTestCase(MinuteEquityHistoryTestCase):
+    DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
+    DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0
+
+
 class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
     CREATE_BARDATA_DATA_FREQUENCY = 'daily'
 
@@ -1755,3 +1760,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                                        window_2[self.ASSET1].values)
         np.testing.assert_almost_equal(window_1[self.ASSET2].values,
                                        window_2[self.ASSET2].values)
+
+
+class NoPrefetchDailyEquityHistoryTestCase(DailyEquityHistoryTestCase):
+    DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
+    DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0

zipline/assets/roll_finder.py

@@ -88,8 +88,10 @@ class RollFinder(with_metaclass(ABCMeta, object)):
         for i, sid in enumerate(oc.contract_sids):
             if sid == first:
                 break
-        rolls = [(first, None)]
-        sessions = self.trading_calendar.sessions_in_range(start, end)
+        rolls = [(first + offset, None)]
+        tc = self.trading_calendar
+        sessions = tc.sessions_in_range(tc.minute_to_session_label(start),
+                                        tc.minute_to_session_label(end))
         if first == front:
             i -= 1
         else:

zipline/data/data_portal.py

@@ -86,6 +86,12 @@ OHLCVP_FIELDS = frozenset([
 
 HISTORY_FREQUENCIES = set(["1m", "1d"])
 
+DEFAULT_MINUTE_HISTORY_PREFETCH = 1560
+DEFAULT_DAILY_HISTORY_PREFETCH = 40
+
+_DEF_M_HIST_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH
+_DEF_D_HIST_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH
+
 
 class DataPortal(object):
     """Interface to all of the data that a zipline simulation needs.
@@ -138,7 +144,9 @@ class DataPortal(object):
                  future_minute_reader=None,
                  adjustment_reader=None,
                  last_available_session=None,
-                 last_available_minute=None):
+                 last_available_minute=None,
+                 minute_history_prefetch_length=_DEF_M_HIST_PREFETCH,
+                 daily_history_prefetch_length=_DEF_D_HIST_PREFETCH):
 
         self.trading_calendar = trading_calendar
         self.asset_finder = asset_finder
@@ -241,6 +249,7 @@ class DataPortal(object):
             self._adjustment_reader,
             self.asset_finder,
             self._roll_finders,
+            prefetch_length=daily_history_prefetch_length,
         )
         self._minute_history_loader = MinuteHistoryLoader(
             self.trading_calendar,
@@ -248,6 +257,7 @@ class DataPortal(object):
             self._adjustment_reader,
             self.asset_finder,
             self._roll_finders,
+            prefetch_length=minute_history_prefetch_length,
         )
 
         self._first_trading_day = first_trading_day

zipline/data/history_loader.py

@@ -307,7 +307,8 @@ class HistoryLoader(with_metaclass(ABCMeta)):
     def __init__(self, trading_calendar, reader, equity_adjustment_reader,
                  asset_finder,
                  roll_finders=None,
-                 sid_cache_size=1000):
+                 sid_cache_size=1000,
+                 prefetch_length=0):
         self.trading_calendar = trading_calendar
         self._asset_finder = asset_finder
         self._reader = reader
@@ -327,15 +328,12 @@ class HistoryLoader(with_metaclass(ABCMeta)):
             field: ExpiringCache(LRU(sid_cache_size))
             for field in self.FIELDS
         }
+        self._prefetch_length = prefetch_length
 
     @abstractproperty
     def _frequency(self):
         pass
 
-    @abstractproperty
-    def _prefetch_length(self):
-        pass
-
     @abstractproperty
     def _calendar(self):
         pass
@@ -406,6 +404,11 @@ class HistoryLoader(with_metaclass(ABCMeta)):
             prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1)
             prefetch_end = cal[prefetch_end_ix]
             prefetch_dts = cal[start_ix:prefetch_end_ix + 1]
+            if is_perspective_after:
+                adj_end_ix = min(prefetch_end_ix + 1, len(cal) - 1)
+                adj_dts = cal[start_ix:adj_end_ix + 1]
+            else:
+                adj_dts = prefetch_dts
             prefetch_len = len(prefetch_dts)
             array = self._array(prefetch_dts, needed_assets, field)
 
@@ -426,7 +429,7 @@ class HistoryLoader(with_metaclass(ABCMeta)):
                     adj_reader = None
                 if adj_reader is not None:
                     adjs = adj_reader.load_adjustments(
-                        [field], prefetch_dts, [asset])[0]
+                        [field], adj_dts, [asset])[0]
                 else:
                     adjs = {}
                 window = window_type(
@@ -539,10 +542,6 @@ class DailyHistoryLoader(HistoryLoader):
     def _frequency(self):
         return 'daily'
 
-    @property
-    def _prefetch_length(self):
-        return 40
-
     @property
     def _calendar(self):
         return self._reader.sessions
@@ -562,10 +561,6 @@ class MinuteHistoryLoader(HistoryLoader):
     def _frequency(self):
         return 'minute'
 
-    @property
-    def _prefetch_length(self):
-        return 1560
-
     @lazyval
     def _calendar(self):
         mm = self.trading_calendar.all_minutes

zipline/testing/fixtures.py

@@ -13,7 +13,11 @@ from .core import (
     create_minute_bar_data,
     tmp_dir,
 )
-from ..data.data_portal import DataPortal
+from ..data.data_portal import (
+    DataPortal,
+    DEFAULT_MINUTE_HISTORY_PREFETCH,
+    DEFAULT_DAILY_HISTORY_PREFETCH,
+)
 from ..data.resample import (
     minute_frame_to_session_frame,
     MinuteResampleSessionBarReader
@@ -1272,6 +1276,9 @@ class WithDataPortal(WithAdjustmentReader,
     DATA_PORTAL_LAST_AVAILABLE_SESSION = None
     DATA_PORTAL_LAST_AVAILABLE_MINUTE = None
 
+    DATA_PORTAL_MINUTE_HISTORY_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH
+    DATA_PORTAL_DAILY_HISTORY_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH
+
     def make_data_portal(self):
         if self.DATA_PORTAL_FIRST_TRADING_DAY is None:
             if self.DATA_PORTAL_USE_MINUTE_DATA:
@@ -1315,6 +1322,10 @@ class WithDataPortal(WithAdjustmentReader,
             ),
             last_available_session=self.DATA_PORTAL_LAST_AVAILABLE_SESSION,
             last_available_minute=self.DATA_PORTAL_LAST_AVAILABLE_MINUTE,
+            minute_history_prefetch_length=self.
+            DATA_PORTAL_MINUTE_HISTORY_PREFETCH,
+            daily_history_prefetch_length=self.
+            DATA_PORTAL_DAILY_HISTORY_PREFETCH,
         )
 
     def init_instance_fixtures(self):