From 066a7776d2a2581a95c477d2c4f897f0a8a0368a Mon Sep 17 00:00:00 2001 From: dmichalowicz Date: Tue, 28 Mar 2017 15:56:53 -0400 Subject: [PATCH] BUG: Open and close resampling code could hit index errors --- tests/data/test_resample.py | 18 +++++++++++++++--- zipline/data/_resample.pyx | 20 +++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/data/test_resample.py b/tests/data/test_resample.py index e160153d..d9639323 100644 --- a/tests/data/test_resample.py +++ b/tests/data/test_resample.py @@ -143,6 +143,8 @@ _FUTURE_CASES = ( ('none_missing', 'day_0_back'))), (1003, (('missing_last', 'day_0_back'), ('missing_first', 'day_1_front'))), + (1004, (('all_missing', 'day_0_back'), + ('none_missing', 'day_1_front'))), ) FUTURE_CASES = OrderedDict() @@ -207,7 +209,6 @@ EXPECTED_AGGREGATION = { 'close': [nan, 103.3, 102.3, 101.3, 103.3, 102.3], 'volume': [0, 1003, 2005, 3006, 4009, 5011], }, columns=OHLCV), - # Equity 3 straddles two days. 1003: DataFrame({ 'open': [107.5, 107.5, 107.5, nan, 103.5, 103.5], 'high': [107.9, 108.9, 108.9, nan, 103.9, 103.9], @@ -215,6 +216,13 @@ EXPECTED_AGGREGATION = { 'close': [107.3, 108.3, 108.3, nan, 103.3, 102.3], 'volume': [1007, 2015, 2015, 0, 1003, 2005], }, columns=OHLCV), + 1004: DataFrame({ + 'open': [nan, nan, nan, 101.5, 101.5, 101.5], + 'high': [nan, nan, nan, 101.9, 103.9, 103.9], + 'low': [nan, nan, nan, 101.1, 101.1, 101.1], + 'close': [nan, nan, nan, 101.3, 103.3, 102.3], + 'volume': [0, 0, 0, 1001, 2004, 3006], + }, columns=OHLCV), } EXPECTED_SESSIONS = { @@ -236,7 +244,11 @@ EXPECTED_SESSIONS = { 1003: DataFrame(EXPECTED_AGGREGATION[1003].iloc[[2, 5]].values, columns=OHLCV, index=pd.to_datetime(['2016-03-16', '2016-03-17'], - utc=True)) + utc=True)), + 1004: DataFrame(EXPECTED_AGGREGATION[1004].iloc[[2, 5]].values, + columns=OHLCV, + index=pd.to_datetime(['2016-03-16', '2016-03-17'], + utc=True)), } @@ -513,7 +525,7 @@ class TestResampleSessionBars(WithBcolzFutureMinuteBarReader, TRADING_CALENDAR_STRS = ('us_futures',) TRADING_CALENDAR_PRIMARY_CAL = 'us_futures' - ASSET_FINDER_FUTURE_SIDS = 1001, 1002, 1003 + ASSET_FINDER_FUTURE_SIDS = 1001, 1002, 1003, 1004 START_DATE = pd.Timestamp('2016-03-16', tz='UTC') END_DATE = pd.Timestamp('2016-03-17', tz='UTC') diff --git a/zipline/data/_resample.pyx b/zipline/data/_resample.pyx index e6d2da8f..22bd9f8d 100644 --- a/zipline/data/_resample.pyx +++ b/zipline/data/_resample.pyx @@ -23,10 +23,15 @@ cpdef void _minute_to_session_open(intp_t[:] close_locs, cdef intp_t i, close_loc, loc = 0 cdef float64_t val for i, close_loc in enumerate(close_locs): - val = data[loc] + val = nan + # Start by getting the price value at the opening minute of each day. + # If the value is NaN, continue looking forward until we either find a + # valid value or reach the closing minute, at which point the value is + # just kept as a NaN. We increment 'loc' after obtaining the value to + # ensure we do not reach an out of bounds index. while isnan(val) and loc <= close_loc: - loc += 1 val = data[loc] + loc += 1 out[i] = val loc = close_loc + 1 @@ -75,17 +80,22 @@ cpdef void _minute_to_session_close(intp_t[:] close_locs, float64_t[:] out): cdef intp_t i, prev_close_loc, loc = 0 cdef float64_t val - num_out = len(close_locs) + num_out = len(out) for i in range(num_out - 1, -1, -1): if i > 0: prev_close_loc = close_locs[i - 1] else: prev_close_loc = -1 loc = close_locs[i] - val = data[loc] + val = nan + # Start by getting the price value at the closing minute of each day. + # If the value is NaN, continue looking back until we either find a + # valid value or reach the closing minute of the previous day, at which + # point the value is just kept as a NaN. We decrement 'loc' after + # obtaining the value to ensure we do not reach a negative index. while isnan(val) and loc > prev_close_loc: - loc -= 1 val = data[loc] + loc -= 1 out[i] = val