MAINT: Handle gaps in input to daily bars writer (#1778)

Previously, a dataframe passed into BcolzDailyBarWriter.write that was
missing an expected session between its first and last sessions would be
written incorrectly. Upon converting the dataframe to a ctable, the
values for all days following the gap would be shifted backwards, and
nans would be shifted in at the end.

This commit handles the issue by asserting that the number of rows in
the input table matches the number of sessions in the calendar between
the table's first and last sessions.

Also fixes a test that was mistakenly using minutes_in_range where it
should have been using sessions_in_range (uncovered by this change).
This commit is contained in:
Andrew Daniels
2017-05-03 20:49:22 -04:00
committed by GitHub
parent f0e100bcf3
commit 52667b4a90
3 changed files with 84 additions and 23 deletions
+43
View File
@@ -31,6 +31,7 @@ from pandas.util.testing import assert_index_equal
from zipline.data.us_equity_pricing import (
BcolzDailyBarReader,
BcolzDailyBarWriter,
NoDataBeforeDate,
NoDataAfterDate,
)
@@ -44,7 +45,10 @@ from zipline.pipeline.loaders.synthetic import (
)
from zipline.testing import seconds_to_timestamp
from zipline.testing.fixtures import (
WithAssetFinder,
WithBcolzEquityDailyBarReader,
WithTmpDir,
WithTradingCalendars,
ZiplineTestCase,
)
from zipline.utils.calendars import get_calendar
@@ -362,3 +366,42 @@ class BcolzDailyBarNeverReadAllTestCase(BcolzDailyBarTestCase):
`load_raw_array`.
"""
BCOLZ_DAILY_BAR_READ_ALL_THRESHOLD = maxsize
class BcolzDailyBarWriterMissingDataTestCase(WithAssetFinder,
WithTmpDir,
WithTradingCalendars,
ZiplineTestCase):
# Sid 3 is active from 2015-06-02 to 2015-06-30.
MISSING_DATA_SID = 3
# Leave out data for a day in the middle of the query range.
MISSING_DATA_DAY = Timestamp('2015-06-15', tz='UTC')
@classmethod
def make_equity_info(cls):
return EQUITY_INFO.loc[EQUITY_INFO.index == cls.MISSING_DATA_SID]
def test_missing_values_assertion(self):
sessions = self.trading_calendar.sessions_in_range(
TEST_CALENDAR_START,
TEST_CALENDAR_STOP,
)
sessions_with_gap = sessions[sessions != self.MISSING_DATA_DAY]
bar_data = make_bar_data(self.make_equity_info(), sessions_with_gap)
writer = BcolzDailyBarWriter(
self.tmpdir.path,
self.trading_calendar,
sessions[0],
sessions[-1],
)
# There are 21 sessions between the start and end date for this
# asset, and we excluded one.
expected_msg = (
'Got 20 rows for daily bars table with first day=2015-06-02, last '
'day=2015-06-30, expected 21 rows.'
)
with self.assertRaisesRegexp(AssertionError, expected_msg):
writer.write(bar_data)