wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-08 02:55:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1580 from quantopian/research-compatible-history-loader

Browse Source 
ENH: Allow configurable history prefetch length.
This commit is contained in:
Eddie Hebert
2016-11-04 14:07:33 -04:00
committed by GitHub
parent a0e36d492d a3df1e3cef
commit 676fb9cb89
6 changed files with 51 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_continuous_futures.py
+5
View File
@@ -972,3 +972,8 @@ class OrderedContractsTestCase(ZiplineTestCase):
chain = oc.active_chain(4, pd.Timestamp('2015-01-04', tz='UTC').value)
self.assertEquals([4], list(chain),
"[4] should be active beginning at its start date.")
class NoPrefetchContinuousFuturesTestCase(ContinuousFuturesTestCase):
DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0
tests/test_history.py
+10
View File
@@ -1351,6 +1351,11 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
format(field, minute))
class NoPrefetchMinuteEquityHistoryTestCase(MinuteEquityHistoryTestCase):
DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0
class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
CREATE_BARDATA_DATA_FREQUENCY = 'daily'
@@ -1755,3 +1760,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
window_2[self.ASSET1].values)
np.testing.assert_almost_equal(window_1[self.ASSET2].values,
window_2[self.ASSET2].values)
class NoPrefetchDailyEquityHistoryTestCase(DailyEquityHistoryTestCase):
DATA_PORTAL_MINUTE_HISTORY_PREFETCH = 0
DATA_PORTAL_DAILY_HISTORY_PREFETCH = 0
zipline/assets/roll_finder.py
+4 -2
View File
@@ -88,8 +88,10 @@ class RollFinder(with_metaclass(ABCMeta, object)):
for i, sid in enumerate(oc.contract_sids):
if sid == first:
break
rolls = [(first, None)]
sessions = self.trading_calendar.sessions_in_range(start, end)
rolls = [(first + offset, None)]
tc = self.trading_calendar
sessions = tc.sessions_in_range(tc.minute_to_session_label(start),
tc.minute_to_session_label(end))
if first == front:
i -= 1
else:
zipline/data/data_portal.py
+11 -1
View File
@@ -86,6 +86,12 @@ OHLCVP_FIELDS = frozenset([
HISTORY_FREQUENCIES = set(["1m", "1d"])
DEFAULT_MINUTE_HISTORY_PREFETCH = 1560
DEFAULT_DAILY_HISTORY_PREFETCH = 40
_DEF_M_HIST_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH
_DEF_D_HIST_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH
class DataPortal(object):
"""Interface to all of the data that a zipline simulation needs.
@@ -138,7 +144,9 @@ class DataPortal(object):
future_minute_reader=None,
adjustment_reader=None,
last_available_session=None,
last_available_minute=None):
last_available_minute=None,
minute_history_prefetch_length=_DEF_M_HIST_PREFETCH,
daily_history_prefetch_length=_DEF_D_HIST_PREFETCH):
self.trading_calendar = trading_calendar
self.asset_finder = asset_finder
@@ -241,6 +249,7 @@ class DataPortal(object):
self._adjustment_reader,
self.asset_finder,
self._roll_finders,
prefetch_length=daily_history_prefetch_length,
)
self._minute_history_loader = MinuteHistoryLoader(
self.trading_calendar,
@@ -248,6 +257,7 @@ class DataPortal(object):
self._adjustment_reader,
self.asset_finder,
self._roll_finders,
prefetch_length=minute_history_prefetch_length,
)
self._first_trading_day = first_trading_day
zipline/data/history_loader.py
+9 -14
View File
@@ -307,7 +307,8 @@ class HistoryLoader(with_metaclass(ABCMeta)):
def __init__(self, trading_calendar, reader, equity_adjustment_reader,
asset_finder,
roll_finders=None,
sid_cache_size=1000):
sid_cache_size=1000,
prefetch_length=0):
self.trading_calendar = trading_calendar
self._asset_finder = asset_finder
self._reader = reader
@@ -327,15 +328,12 @@ class HistoryLoader(with_metaclass(ABCMeta)):
field: ExpiringCache(LRU(sid_cache_size))
for field in self.FIELDS
}
self._prefetch_length = prefetch_length
@abstractproperty
def _frequency(self):
pass
@abstractproperty
def _prefetch_length(self):
pass
@abstractproperty
def _calendar(self):
pass
@@ -406,6 +404,11 @@ class HistoryLoader(with_metaclass(ABCMeta)):
prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1)
prefetch_end = cal[prefetch_end_ix]
prefetch_dts = cal[start_ix:prefetch_end_ix + 1]
if is_perspective_after:
adj_end_ix = min(prefetch_end_ix + 1, len(cal) - 1)
adj_dts = cal[start_ix:adj_end_ix + 1]
else:
adj_dts = prefetch_dts
prefetch_len = len(prefetch_dts)
array = self._array(prefetch_dts, needed_assets, field)
@@ -426,7 +429,7 @@ class HistoryLoader(with_metaclass(ABCMeta)):
adj_reader = None
if adj_reader is not None:
adjs = adj_reader.load_adjustments(
[field], prefetch_dts, [asset])[0]
[field], adj_dts, [asset])[0]
else:
adjs = {}
window = window_type(
@@ -539,10 +542,6 @@ class DailyHistoryLoader(HistoryLoader):
def _frequency(self):
return 'daily'
@property
def _prefetch_length(self):
return 40
@property
def _calendar(self):
return self._reader.sessions
@@ -562,10 +561,6 @@ class MinuteHistoryLoader(HistoryLoader):
def _frequency(self):
return 'minute'
@property
def _prefetch_length(self):
return 1560
@lazyval
def _calendar(self):
mm = self.trading_calendar.all_minutes
zipline/testing/fixtures.py
+12 -1
View File
@@ -13,7 +13,11 @@ from .core import (
create_minute_bar_data,
tmp_dir,
)
from ..data.data_portal import DataPortal
from ..data.data_portal import (
DataPortal,
DEFAULT_MINUTE_HISTORY_PREFETCH,
DEFAULT_DAILY_HISTORY_PREFETCH,
)
from ..data.resample import (
minute_frame_to_session_frame,
MinuteResampleSessionBarReader
@@ -1272,6 +1276,9 @@ class WithDataPortal(WithAdjustmentReader,
DATA_PORTAL_LAST_AVAILABLE_SESSION = None
DATA_PORTAL_LAST_AVAILABLE_MINUTE = None
DATA_PORTAL_MINUTE_HISTORY_PREFETCH = DEFAULT_MINUTE_HISTORY_PREFETCH
DATA_PORTAL_DAILY_HISTORY_PREFETCH = DEFAULT_DAILY_HISTORY_PREFETCH
def make_data_portal(self):
if self.DATA_PORTAL_FIRST_TRADING_DAY is None:
if self.DATA_PORTAL_USE_MINUTE_DATA:
@@ -1315,6 +1322,10 @@ class WithDataPortal(WithAdjustmentReader,
),
last_available_session=self.DATA_PORTAL_LAST_AVAILABLE_SESSION,
last_available_minute=self.DATA_PORTAL_LAST_AVAILABLE_MINUTE,
minute_history_prefetch_length=self.
DATA_PORTAL_MINUTE_HISTORY_PREFETCH,
daily_history_prefetch_length=self.
DATA_PORTAL_DAILY_HISTORY_PREFETCH,
)
def init_instance_fixtures(self):