diff --git a/tests/test_continuous_futures.py b/tests/test_continuous_futures.py index 387e22ae..cc3bd451 100644 --- a/tests/test_continuous_futures.py +++ b/tests/test_continuous_futures.py @@ -62,37 +62,42 @@ class ContinuousFuturesTestCase(WithCreateBarData, @classmethod def make_futures_info(self): return DataFrame({ - 'symbol': ['FOF16', 'FOG16', 'FOH16', 'FOJ16', 'FOF22'], - 'root_symbol': ['FO', 'FO', 'FO', 'FO', 'FO'], - 'asset_name': ['Foo'] * 5, + 'symbol': ['FOF16', 'FOG16', 'FOH16', 'FOJ16', 'FOK16', 'FOF22'], + 'root_symbol': ['FO'] * 6, + 'asset_name': ['Foo'] * 6, 'start_date': [Timestamp('2015-01-05', tz='UTC'), Timestamp('2015-02-05', tz='UTC'), Timestamp('2015-03-05', tz='UTC'), Timestamp('2015-04-05', tz='UTC'), + Timestamp('2015-05-05', tz='UTC'), Timestamp('2021-01-05', tz='UTC')], 'end_date': [Timestamp('2016-08-19', tz='UTC'), Timestamp('2016-09-19', tz='UTC'), Timestamp('2016-10-19', tz='UTC'), Timestamp('2016-11-19', tz='UTC'), + Timestamp('2016-12-19', tz='UTC'), Timestamp('2022-08-19', tz='UTC')], 'notice_date': [Timestamp('2016-01-27', tz='UTC'), Timestamp('2016-02-26', tz='UTC'), Timestamp('2016-03-24', tz='UTC'), Timestamp('2016-04-26', tz='UTC'), + Timestamp('2016-05-26', tz='UTC'), Timestamp('2022-01-26', tz='UTC')], 'expiration_date': [Timestamp('2016-01-27', tz='UTC'), Timestamp('2016-02-26', tz='UTC'), Timestamp('2016-03-24', tz='UTC'), Timestamp('2016-04-26', tz='UTC'), + Timestamp('2016-05-26', tz='UTC'), Timestamp('2022-01-26', tz='UTC')], 'auto_close_date': [Timestamp('2016-01-27', tz='UTC'), Timestamp('2016-02-26', tz='UTC'), Timestamp('2016-03-24', tz='UTC'), Timestamp('2016-04-26', tz='UTC'), + Timestamp('2016-05-26', tz='UTC'), Timestamp('2022-01-26', tz='UTC')], - 'tick_size': [0.001] * 5, - 'multiplier': [1000.0] * 5, - 'exchange': ['CME'] * 5, + 'tick_size': [0.001] * 6, + 'multiplier': [1000.0] * 6, + 'exchange': ['CME'] * 6, }) @classmethod @@ -126,7 +131,6 @@ class ContinuousFuturesTestCase(WithCreateBarData, arange(r, r * FUTURES_MINUTES_PER_DAY + r, r, dtype=int64), len(sessions)) vol_markers = vol_day_markers + vol_min_markers - base_df = pd.DataFrame( { 'open': full(len(dts), 102000.0) + markers, @@ -138,8 +142,39 @@ class ContinuousFuturesTestCase(WithCreateBarData, index=dts) # Add the sid to the ones place of the prices, so that the ones # place can be used to eyeball the source contract. - for i in range(5): - yield i, base_df + i * 10000 + + # For volume roll tests end sid volume early. + # FOF16 cuts out day before autoclose of 01-26 + # FOG16 cuts out on autoclose + # FOH16 cuts out 4 days before autoclose + # FOJ16 cuts out 3 days before autoclose + + sid_to_vol_stop_session = { + 0: Timestamp('2016-01-26', tz='UTC'), + 1: Timestamp('2016-02-26', tz='UTC'), + 2: Timestamp('2016-03-18', tz='UTC'), + 3: Timestamp('2016-04-20', tz='UTC'), + } + for i in range(6): + df = base_df.copy() + df += i * 10000 + if i in sid_to_vol_stop_session: + vol_stop_session = sid_to_vol_stop_session[i] + m_open = tc.open_and_close_for_session(vol_stop_session)[0] + loc = dts.searchsorted(m_open) + # Add a little bit of noise to roll. So that checks for exacly + # 0 do not work, since there may be stragglers after a roll. + df.volume.values[loc] = 1000 + df.volume.values[loc + 1:] = 0 + j = i - 1 + if j in sid_to_vol_stop_session: + non_primary_end = sid_to_vol_stop_session[j] - sessions.freq + m_close = tc.open_and_close_for_session(non_primary_end)[1] + loc = dts.searchsorted(m_close) + # Add some volume before a roll, since a contracted may be + # entered earlier than when it is the primary. + df.volume.values[:loc] = 2000 + yield i, df def test_create_continuous_future(self): cf_primary = self.asset_finder.create_continuous_future( @@ -190,6 +225,29 @@ class ContinuousFuturesTestCase(WithCreateBarData, contract = bar_data.current(cf_primary, 'contract') self.assertEqual(contract.symbol, 'FOG16') + def test_current_contract_volume_roll(self): + cf_primary = self.asset_finder.create_continuous_future( + 'FO', 0, 'volume') + bar_data = self.create_bardata( + lambda: pd.Timestamp('2016-01-26', tz='UTC')) + contract = bar_data.current(cf_primary, 'contract') + + self.assertEqual(contract.symbol, 'FOG16') + + bar_data = self.create_bardata( + lambda: pd.Timestamp('2016-01-26', tz='UTC')) + contract = bar_data.current(cf_primary, 'contract') + + self.assertEqual(contract.symbol, 'FOG16', + 'Auto close at beginning of session. FOG16 remains ' + 'the current contract.') + + bar_data = self.create_bardata( + lambda: pd.Timestamp('2016-02-26', tz='UTC')) + contract = bar_data.current(cf_primary, 'contract') + self.assertEqual(contract.symbol, 'FOH16', + 'Volume switch to FOH16, should have triggered roll.') + def test_current_contract_in_algo(self): code = dedent(""" from zipline.api import ( @@ -276,15 +334,15 @@ def record_current_contract(algo, data): result = results.iloc[0] self.assertEqual(result.primary_len, - 4, - 'There should be only 4 contracts in the chain for ' - 'the primary, there are 5 contracts defined in the ' + 5, + 'There should be only 5 contracts in the chain for ' + 'the primary, there are 6 contracts defined in the ' 'fixture, but one has a start after the simulation ' 'date.') self.assertEqual(result.secondary_len, - 3, - 'There should be only 3 contracts in the chain for ' - 'the primary, there are 5 contracts defined in the ' + 4, + 'There should be only 4 contracts in the chain for ' + 'the primary, there are 6 contracts defined in the ' 'fixture, but one has a start after the simulation ' 'date. And the first is not included because it is ' 'the primary on that date.') @@ -299,27 +357,27 @@ def record_current_contract(algo, data): 'session.') self.assertEqual(result.primary_last, - 'FOJ16', - 'End of primary chain should be FOJ16 on first ' + 'FOK16', + 'End of primary chain should be FOK16 on first ' 'session.') self.assertEqual(result.secondary_last, - 'FOJ16', - 'End of secondary chain should be FOJ16 on first ' + 'FOK16', + 'End of secondary chain should be FOK16 on first ' 'session.') # Second day, primary should switch to FOG result = results.iloc[1] self.assertEqual(result.primary_len, - 3, - 'There should be only 3 contracts in the chain for ' - 'the primary, there are 5 contracts defined in the ' + 4, + 'There should be only 4 contracts in the chain for ' + 'the primary, there are 6 contracts defined in the ' 'fixture, but one has a start after the simulation ' 'date. The first is not included because of roll.') self.assertEqual(result.secondary_len, - 2, - 'There should be only 2 contracts in the chain for ' - 'the primary, there are 5 contracts defined in the ' + 3, + 'There should be only 3 contracts in the chain for ' + 'the primary, there are 6 contracts defined in the ' 'fixture, but one has a start after the simulation ' 'date. The first is not included because of roll, ' 'the second is the primary on that date.') @@ -336,12 +394,12 @@ def record_current_contract(algo, data): # These values remain FOJ16 because fixture data is not exhaustive # enough to move the end of the chain. self.assertEqual(result.primary_last, - 'FOJ16', - 'End of primary chain should be FOJ16 on second ' + 'FOK16', + 'End of primary chain should be FOK16 on second ' 'session.') self.assertEqual(result.secondary_last, - 'FOJ16', - 'End of secondary chain should be FOJ16 on second ' + 'FOK16', + 'End of secondary chain should be FOK16 on second ' 'session.') def test_history_sid_session(self): @@ -398,6 +456,70 @@ def record_current_contract(algo, data): 3, "Should be FOJ16 on session after roll.") + def test_history_sid_session_volume_roll(self): + cf = self.data_portal.asset_finder.create_continuous_future( + 'FO', 0, 'volume') + window = self.data_portal.get_history_window( + [cf], + Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'), + 30, '1d', 'sid') + + # Volume cuts out for FOF16 on 2016-01-25 + self.assertEqual(window.loc['2016-01-26', cf], + 1, + "Should be FOG16 at beginning of window.") + + self.assertEqual(window.loc['2016-01-27', cf], + 1, + "Should have remained FOG16.") + + self.assertEqual(window.loc['2016-02-25', cf], + 1, + "Should be FOG16 on session before roll.") + + self.assertEqual(window.loc['2016-02-26', cf], + 2, + "Should be FOH16 on session with roll.") + + self.assertEqual(window.loc['2016-02-29', cf], + 2, + "Should be FOH16 on session after roll.") + + # Advance the window a month. + window = self.data_portal.get_history_window( + [cf], + Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'), + 30, '1d', 'sid') + + self.assertEqual(window.loc['2016-02-25', cf], + 1, + "Should be FOG16 at beginning of window.") + + self.assertEqual(window.loc['2016-02-26', cf], + 2, + "Should be FOH16 on session with roll.") + + self.assertEqual(window.loc['2016-02-29', cf], + 2, + "Should be FOH16 on session after roll.") + + self.assertEqual(window.loc['2016-03-17', cf], + 2, + "Should be FOH16 on session before volume cuts out.") + + self.assertEqual(window.loc['2016-03-18', cf], + 3, + "Should be FOJ16 on session where the volume of " + "FOH16 cuts out.") + + self.assertEqual(window.loc['2016-03-24', cf], + 3, + "Should have remained FOJ16.") + + self.assertEqual(window.loc['2016-03-28', cf], + 3, + "Should have remained FOJ16.") + def test_history_sid_minute(self): cf = self.data_portal.asset_finder.create_continuous_future( 'FO', 0, 'calendar') @@ -550,39 +672,39 @@ def record_current_contract(algo, data): # difference: 7008.56 assert_almost_equal( window.loc['2016-02-24', cf_mul], - 129059.581, + 129090.459, err_msg="At beginning of window, should be FOG16's 22nd value, " "with two adjustments.") assert_almost_equal( window.loc['2016-02-24', cf_add], - 129238.561, + 129268.561, err_msg="At beginning of window, should be FOG16's 22nd value, " "with two adjustments") # Unadjusted: 125241.44 assert_almost_equal( window.loc['2016-02-26', cf_mul], - 132239.942, + 132271.58, err_msg="On session with roll, should be FOH16's 24th value, " "with one adjustment.") assert_almost_equal( window.loc['2016-02-26', cf_add], - 132250.0, + 132280.0, err_msg="On session with roll, should be FOH16's 24th value, " "with one adjustment.") # Unadjusted: 125251.44 assert_almost_equal( window.loc['2016-02-29', cf_mul], - 132250.500, + 132282.141, err_msg="On session after roll, should be FOH16's 25th value, " "with one adjustment.") assert_almost_equal( window.loc['2016-02-29', cf_add], - 132260.000, + 132290.00, err_msg="On session after roll, should be FOH16's 25th value, " "unadjusted.") diff --git a/zipline/assets/assets.py b/zipline/assets/assets.py index 9dd928bd..d92b5b6a 100644 --- a/zipline/assets/assets.py +++ b/zipline/assets/assets.py @@ -129,7 +129,8 @@ SID_TYPE_IDS = { } CONTINUOUS_FUTURE_ROLL_STYLE_IDS = { - 'calendar': 0 + 'calendar': 0, + 'volume': 1, } CONTINUOUS_FUTURE_ADJUSTMENT_STYLE_IDS = { diff --git a/zipline/assets/continuous_futures.pyx b/zipline/assets/continuous_futures.pyx index 528d1e27..572c3754 100644 --- a/zipline/assets/continuous_futures.pyx +++ b/zipline/assets/continuous_futures.pyx @@ -297,7 +297,7 @@ cdef class OrderedContracts(object): break return self.contract_sids[i] - cpdef long_t contract_at_offset(self, long_t sid, Py_ssize_t offset): + cpdef contract_at_offset(self, long_t sid, Py_ssize_t offset, int64_t start_cap): """ Get the sid which is the given sid plus the offset distance. An offset of 0 should be reflexive. @@ -305,9 +305,13 @@ cdef class OrderedContracts(object): cdef Py_ssize_t i cdef long_t[:] sids sids = self.contract_sids + start_dates = self.start_dates for i in range(self._size): if sid == sids[i]: - return sids[i + offset] + if start_dates[i + offset] < start_cap: + return sids[i + offset] + else: + return None cpdef long_t[:] active_chain(self, long_t starting_sid, long_t dt_value): cdef Py_ssize_t left, right, i, j diff --git a/zipline/assets/roll_finder.py b/zipline/assets/roll_finder.py index 47862900..3df6c94c 100644 --- a/zipline/assets/roll_finder.py +++ b/zipline/assets/roll_finder.py @@ -23,8 +23,10 @@ class RollFinder(with_metaclass(ABCMeta, object)): Abstract base class for calculating when futures contracts are the active contract. """ - @abstractmethod + def _active_contract(self, oc, front, back, dt): + raise NotImplementedError + def get_contract_center(self, root_symbol, dt, offset): """ Parameters @@ -42,9 +44,16 @@ class RollFinder(with_metaclass(ABCMeta, object)): Future The active future contract at the given dt. """ - raise NotImplemented + oc = self.asset_finder.get_ordered_contracts(root_symbol) + session = self.trading_calendar.minute_to_session_label(dt) + front = oc.contract_before_auto_close(session.value) + back = oc.contract_at_offset(front, 1, dt.value) + if back is None: + return front + session = self.trading_calendar.minute_to_session_label(dt) + primary = self._active_contract(oc, front, back, session) + return oc.contract_at_offset(primary, offset, session.value) - @abstractmethod def get_rolls(self, root_symbol, start, end, offset): """ Get the rolls, i.e. the session at which to hop from contract to @@ -69,7 +78,39 @@ class RollFinder(with_metaclass(ABCMeta, object)): The last pair in the chain has a value of `None` since the roll is after the range. """ - raise NotImplemented + oc = self.asset_finder.get_ordered_contracts(root_symbol) + front = self.get_contract_center(root_symbol, end, offset) + back = oc.contract_at_offset(front, 1, end.value) + if back is not None: + first = self._active_contract(oc, front, back, end) + else: + first = front + for i, sid in enumerate(oc.contract_sids): + if sid == first: + break + rolls = [(first, None)] + sessions = self.trading_calendar.sessions_in_range(start, end) + if first == front: + i -= 1 + else: + i -= 2 + auto_close_date = Timestamp(oc.auto_close_dates[i], tz='UTC') + while auto_close_date > start and i > -1: + session_loc = sessions.searchsorted(auto_close_date) + front = oc.contract_sids[i] + back = oc.contract_sids[i + 1] + while session_loc > -1: + session = sessions[session_loc] + if back != self._active_contract(oc, front, back, session): + break + session_loc -= 1 + roll_session = sessions[session_loc + 1] + if roll_session > start: + rolls.insert(0, (front, roll_session)) + i -= 1 + auto_close_date = Timestamp(oc.auto_close_dates[i], + tz='UTC') + return rolls class CalendarRollFinder(RollFinder): @@ -82,31 +123,30 @@ class CalendarRollFinder(RollFinder): self.trading_calendar = trading_calendar self.asset_finder = asset_finder - def get_contract_center(self, root_symbol, dt, offset): - oc = self.asset_finder.get_ordered_contracts(root_symbol) - session = self.trading_calendar.minute_to_session_label(dt) - primary_candidate = oc.contract_before_auto_close(session.value) - - # Here is where a volume check would be. - primary = primary_candidate - return oc.contract_at_offset(primary, offset) - - def get_rolls(self, root_symbol, start, end, offset): - oc = self.asset_finder.get_ordered_contracts(root_symbol) - primary_at_end = self.get_contract_center(root_symbol, end, 0) + def _active_contract(self, oc, front, back, dt): for i, sid in enumerate(oc.contract_sids): - if sid == primary_at_end: + if sid == front: break - i += offset - first = oc.contract_sids[i] - rolls = [(first, None)] - i -= 1 - auto_close_date = Timestamp(oc.auto_close_dates[i - offset], tz='UTC') - while auto_close_date > start and i > -1: - rolls.insert(0, (oc.contract_sids[i - offset], - auto_close_date)) - i -= 1 - auto_close_date = Timestamp(oc.auto_close_dates[i - offset], - tz='UTC') + auto_close_date = Timestamp(oc.auto_close_dates[i], tz='UTC') + before_auto_close = dt < auto_close_date + return front if before_auto_close else back - return rolls + +class VolumeRollFinder(RollFinder): + """ + The CalendarRollFinder calculates contract rolls based on when + volume activity transfers from one contract to another. + """ + + THRESHOLD = 0.10 + + def __init__(self, trading_calendar, asset_finder, session_reader): + self.trading_calendar = trading_calendar + self.asset_finder = asset_finder + self.session_reader = session_reader + + def _active_contract(self, oc, front, back, dt): + # FIXME: Possible vector for look ahead bias. + front_vol = self.session_reader.get_value(front, dt, 'volume') + back_vol = self.session_reader.get_value(back, dt, 'volume') + return back if back_vol > front_vol else front diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index c021b11f..4946cf1d 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -30,7 +30,10 @@ from zipline.data.continuous_future_reader import ( ContinuousFutureSessionBarReader, ContinuousFutureMinuteBarReader ) -from zipline.assets.roll_finder import CalendarRollFinder +from zipline.assets.roll_finder import ( + CalendarRollFinder, + VolumeRollFinder +) from zipline.data.dispatch_bar_reader import ( AssetDispatchMinuteBarReader, AssetDispatchSessionBarReader @@ -165,11 +168,6 @@ class DataPortal(object): else: self._last_trading_session = None - self._roll_finders = { - 'calendar': CalendarRollFinder(self.trading_calendar, - self.asset_finder) - } - aligned_equity_minute_reader = self._ensure_reader_aligned( equity_minute_reader) aligned_equity_session_reader = self._ensure_reader_aligned( @@ -179,6 +177,11 @@ class DataPortal(object): aligned_future_session_reader = self._ensure_reader_aligned( future_daily_reader) + self._roll_finders = { + 'calendar': CalendarRollFinder(self.trading_calendar, + self.asset_finder), + } + aligned_minute_readers = {} aligned_session_readers = {} @@ -197,6 +200,11 @@ class DataPortal(object): if aligned_future_session_reader is not None: aligned_session_readers[Future] = aligned_future_session_reader + self._roll_finders['volume'] = VolumeRollFinder( + self.trading_calendar, + self.asset_finder, + aligned_future_session_reader, + ) aligned_session_readers[ContinuousFuture] = \ ContinuousFutureSessionBarReader( aligned_future_session_reader,