wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-29 11:18:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1243 from quantopian/source-daily-from-minute

Browse Source 
TST:  Enable sourcing daily data from minute data.
This commit is contained in:
Eddie Hebert
2016-06-02 13:10:28 -04:00
parent 8f0b3f41a1 2f80e94203
commit 38d6107faf
2 changed files with 130 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_history.py
+69 -22
View File
@@ -39,7 +39,7 @@ ALL_FIELDS = OHLCP + ['volume']
class WithHistory(WithDataPortal):
TRADING_START_DT = TRADING_ENV_MIN_DATE = START_DATE = pd.Timestamp(
'2014-02-03',
'2014-01-03',
tz='UTC',
)
TRADING_END_DT = END_DATE = pd.Timestamp('2016-01-29', tz='UTC')
@@ -445,10 +445,13 @@ MINUTE_FIELD_INFO = {
class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
BCOLZ_DAILY_BAR_SOURCE_FROM_MINUTE = True
@classmethod
def make_minute_bar_data(cls):
data = {}
sids = {2, 4, 5, 6, cls.SHORT_ASSET_SID, cls.HALF_DAY_TEST_ASSET_SID}
sids = {2, 5, cls.SHORT_ASSET_SID, cls.HALF_DAY_TEST_ASSET_SID}
for sid in sids:
asset = cls.asset_finder.retrieve_asset(sid)
data[sid] = create_minute_df_for_asset(
@@ -465,6 +468,27 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
start_val=2,
)
# Start values are crafted so that the thousands place are equal when
# adjustments are applied correctly.
# The splits and mergers are defined as 2:1 splits, so the prices
# approximate that adjustment by halving the thousands place each day.
data[cls.MERGER_ASSET_SID] = data[cls.SPLIT_ASSET_SID] = pd.concat((
create_minute_df_for_asset(
cls.env,
pd.Timestamp('2015-01-05', tz='UTC'),
pd.Timestamp('2015-01-05', tz='UTC'),
start_val=4000),
create_minute_df_for_asset(
cls.env,
pd.Timestamp('2015-01-06', tz='UTC'),
pd.Timestamp('2015-01-06', tz='UTC'),
start_val=2000),
create_minute_df_for_asset(
cls.env,
pd.Timestamp('2015-01-07', tz='UTC'),
pd.Timestamp('2015-01-07', tz='UTC'),
start_val=1000)
))
asset3 = cls.asset_finder.retrieve_asset(3)
data[3] = create_minute_df_for_asset(
cls.env,
@@ -691,7 +715,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
'close'
)[asset]
np.testing.assert_array_equal(np.array(range(382, 392)), window1)
np.testing.assert_array_equal(
np.array(range(4380, 4390)), window1)
# straddling the first event
window2 = self.data_portal.get_history_window(
@@ -704,7 +729,18 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
# five minutes from 1/5 should be halved
np.testing.assert_array_equal(
[193.5, 194, 194.5, 195, 195.5, 392, 393, 394, 395, 396],
[2192.5,
2193,
2193.5,
2194,
2194.5,
# Split occurs. The value of the thousands place should
# match.
2000,
2001,
2002,
2003,
2004],
window2
)
@@ -717,20 +753,20 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
'close'
)[asset]
# first five minutes should be 387-391, but quartered
# first five minutes should be 4385-4390, but quartered
np.testing.assert_array_equal(
[96.75, 97, 97.25, 97.5, 97.75],
[1096.25, 1096.5, 1096.75, 1097, 1097.25],
window3[0:5]
)
# next 390 minutes should be 392-781, but halved
# next 390 minutes should be 2000-2390, but halved
np.testing.assert_array_equal(
np.array(range(392, 782), dtype='float64') / 2,
np.array(range(2000, 2390), dtype='float64') / 2,
window3[5:395]
)
# final 5 minutes should be 782-787
np.testing.assert_array_equal(range(782, 787), window3[395:])
# final 5 minutes should be 1000-1004
np.testing.assert_array_equal(range(1000, 1005), window3[395:])
# after last event
window4 = self.data_portal.get_history_window(
@@ -741,8 +777,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
'close'
)[asset]
# should not be adjusted, should be 787 to 791
np.testing.assert_array_equal(range(787, 792), window4)
# should not be adjusted, should be 1005 to 1009
np.testing.assert_array_equal(range(1005, 1010), window4)
def test_minute_dividends(self):
# self.DIVIDEND_ASSET had dividends on 1/6 and 1/7
@@ -823,6 +859,15 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
current_dt = pd.Timestamp('2015-01-06 8:45', tz='US/Eastern')
bar_data = BarData(self.data_portal, lambda: current_dt, 'minute')
adj_expected = {
'open': np.arange(4381, 4391) / 2.0,
'high': np.arange(4382, 4392) / 2.0,
'low': np.arange(4379, 4389) / 2.0,
'close': np.arange(4380, 4390) / 2.0,
'volume': np.arange(4380, 4390) * 100 * 2.0,
'price': np.arange(4380, 4390) / 2.0,
}
expected = {
'open': np.arange(383, 393) / 2.0,
'high': np.arange(384, 394) / 2.0,
@@ -836,23 +881,25 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
# Single field, single asset
for field in ALL_FIELDS:
values = bar_data.history(self.SPLIT_ASSET, field, 10, '1m')
np.testing.assert_array_equal(values.values, expected[field])
np.testing.assert_array_equal(values.values,
adj_expected[field],
err_msg=field)
# Multi field, single asset
values = bar_data.history(
self.SPLIT_ASSET, ['open', 'volume'], 10, '1m'
)
np.testing.assert_array_equal(values.open.values,
expected['open'])
adj_expected['open'])
np.testing.assert_array_equal(values.volume.values,
expected['volume'])
adj_expected['volume'])
# Single field, multi asset
values = bar_data.history(
[self.SPLIT_ASSET, self.ASSET2], 'open', 10, '1m'
)
np.testing.assert_array_equal(values[self.SPLIT_ASSET].values,
expected['open'])
adj_expected['open'])
np.testing.assert_array_equal(values[self.ASSET2].values,
expected['open'] * 2)
@@ -862,11 +909,11 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
)
np.testing.assert_array_equal(
values.open[self.SPLIT_ASSET].values,
expected['open']
adj_expected['open']
)
np.testing.assert_array_equal(
values.volume[self.SPLIT_ASSET].values,
expected['volume']
adj_expected['volume']
)
np.testing.assert_array_equal(
values.open[self.ASSET2].values,
@@ -946,8 +993,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
self.TRADING_START_DT
)
exp_msg = (
'History window extends before 2014-02-03. To use this history '
'window, start the backtest on or after 2014-02-04.'
'History window extends before 2014-01-03. To use this history '
'window, start the backtest on or after 2014-01-06.'
)
for field in OHLCP:
with self.assertRaisesRegexp(
@@ -1454,8 +1501,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
second_day = self.env.next_trading_day(self.TRADING_START_DT)
exp_msg = (
'History window extends before 2014-02-03. To use this history '
'window, start the backtest on or after 2014-02-07.'
'History window extends before 2014-01-03. To use this history '
'window, start the backtest on or after 2014-01-09.'
)
with self.assertRaisesRegexp(HistoryWindowStartsBeforeData, exp_msg):
zipline/testing/fixtures.py
+61 -5
View File
@@ -585,6 +585,9 @@ class WithBcolzDailyBarReader(WithTradingEnvironment, WithTmpDir):
BCOLZ_DAILY_BAR_READ_ALL_THRESHOLD : int
If this flag is set, use the value as the `read_all_threshold`
parameter to BcolzDailyBarReader, otherwise use the default value.
BCOLZ_DAILY_BAR_SOURCE_FROM_MINUTE : bool
If this flag is set, `make_daily_bar_data` will read data from the
minute bar reader defined by a `WithBcolzMinuteBarReader`.
Methods
-------
@@ -607,16 +610,67 @@ class WithBcolzDailyBarReader(WithTradingEnvironment, WithTmpDir):
BCOLZ_DAILY_BAR_START_DATE = alias('START_DATE')
BCOLZ_DAILY_BAR_END_DATE = alias('END_DATE')
BCOLZ_DAILY_BAR_READ_ALL_THRESHOLD = None
BCOLZ_DAILY_BAR_SOURCE_FROM_MINUTE = False
# allows WithBcolzDailyBarReaderFromCSVs to call the `write_csvs` method
# without needing to reimplement `init_class_fixtures`
_write_method_name = 'write'
@classmethod
def _make_daily_bar_from_minute(cls):
assets = cls.asset_finder.retrieve_all(cls.asset_finder.sids)
ohclv_how = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
# TODO: Change test data so that large minute volumes are not used,
# so that 'sum' can be used without going over the uint limit.
# When that data is changed, this function can and should be moved
# to the `data` module so that loaders and tests can use the same
# source from minute logic.
'volume': 'last'
}
mm = cls.env.market_minutes
m_opens = cls.env.open_and_closes.market_open
m_closes = cls.env.open_and_closes.market_close
for asset in assets:
first_minute = m_opens.loc[asset.start_date]
last_minute = m_closes.loc[asset.end_date]
window = cls.bcolz_minute_bar_reader.load_raw_arrays(
fields=['open', 'high', 'low', 'close', 'volume'],
start_dt=first_minute,
end_dt=last_minute,
sids=[asset.sid],
)
opens, highs, lows, closes, volumes = [c.reshape(-1)
for c in window]
minutes = mm[mm.slice_indexer(start=first_minute,
end=last_minute)]
df = pd.DataFrame(
{
'open': opens,
'high': highs,
'low': lows,
'close': closes,
'volume': volumes,
},
index=minutes
)
yield asset.sid, df.resample('1d', how=ohclv_how).dropna()
@classmethod
def make_daily_bar_data(cls):
return create_daily_bar_data(
cls.bcolz_daily_bar_days,
cls.asset_finder.sids,
)
# Requires a minute bar reader to come before in the MRO.
# Resample that data so that daily and minute bar data are aligned.
if cls.BCOLZ_DAILY_BAR_SOURCE_FROM_MINUTE:
return cls._make_daily_bar_from_minute()
else:
return create_daily_bar_data(
cls.bcolz_daily_bar_days,
cls.asset_finder.sids,
)
@classmethod
def init_class_fixtures(cls):
@@ -1069,7 +1123,9 @@ class WithSeededRandomPipelineEngine(WithNYSETradingDays, WithAssetFinder):
)
class WithDataPortal(WithBcolzMinuteBarReader, WithAdjustmentReader):
class WithDataPortal(WithAdjustmentReader,
# Ordered so that bcolz minute reader is used first.
WithBcolzMinuteBarReader):
"""
ZiplineTestCase mixin providing self.data_portal as an instance level
fixture.