ENH: Add a reader writer pair for HDF5 minute bar updates.

This format is intended for storing data for all sids of an asset type,
e.g. equities or futures for a session. bcolz is not used to avoid the overhead
of creating the directories and files for each asset (which numbers around ~8000
for active equities) can be removed since the update is meant to be read at
once, instead of supporting the random access pattern needed by the simulation.

This patch only adds the reader/writer pair, with the management of finding the
paths to delta files and the application of the updates to the bcolz write left
to internal loader code.

Also, the update reader interface is intentionally constrained to the data for
an entire session to allow for an implementation that allows for mid-session updates.
This commit is contained in:
Eddie Hebert
2017-01-04 12:09:10 -05:00
committed by Eddie Hebert
parent 3095f8c573
commit e913519734
3 changed files with 150 additions and 1 deletions
+55 -1
View File
@@ -42,7 +42,9 @@ from zipline.data.minute_bars import (
BcolzMinuteBarReader,
BcolzMinuteOverlappingData,
US_EQUITIES_MINUTES_PER_DAY,
BcolzMinuteWriterColumnMismatch
BcolzMinuteWriterColumnMismatch,
H5MinuteBarUpdateWriter,
H5MinuteBarUpdateReader,
)
from zipline.testing.fixtures import (
@@ -1146,3 +1148,55 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
"The last traded dt should be before the early "
"close, even when data is written between the early "
"close and the next open.")
def test_minute_updates(self):
"""
Test minute updates.
"""
start_minute = self.market_opens[TEST_CALENDAR_START]
minutes = [start_minute,
start_minute + Timedelta('1 min'),
start_minute + Timedelta('2 min')]
sids = [1, 2]
data_1 = DataFrame(
data={
'open': [15.0, nan, 15.1],
'high': [17.0, nan, 17.1],
'low': [11.0, nan, 11.1],
'close': [14.0, nan, 14.1],
'volume': [1000, 0, 1001]
},
index=minutes)
data_2 = DataFrame(
data={
'open': [25.0, nan, 25.1],
'high': [27.0, nan, 27.1],
'low': [21.0, nan, 21.1],
'close': [24.0, nan, 24.1],
'volume': [2000, 0, 2001]
},
index=minutes)
frames = {1: data_1, 2: data_2}
update_path = self.instance_tmpdir.getpath('updates.h5')
update_writer = H5MinuteBarUpdateWriter(update_path)
update_writer.write(frames)
update_reader = H5MinuteBarUpdateReader(update_path)
self.writer.write(update_reader.read(minutes, sids))
# Refresh the reader since truncate update the metadata.
reader = BcolzMinuteBarReader(self.dest)
columns = ['open', 'high', 'low', 'close', 'volume']
sids = [sids[0], sids[1]]
arrays = list(map(transpose, reader.load_raw_arrays(
columns, minutes[0], minutes[-1], sids,
)))
data = {sids[0]: data_1, sids[1]: data_2}
for i, col in enumerate(columns):
for j, sid in enumerate(sids):
assert_almost_equal(data[sid][col], arrays[i][j])