From fbd3774278cf64a5f3b9f913f2c742598a4b2a38 Mon Sep 17 00:00:00 2001 From: Jean Bredeche Date: Tue, 16 Aug 2016 15:08:32 -0400 Subject: [PATCH] ENH: Update can_trade to check exchange time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BarData now takes the trading calendar as a parameter. can_trade now checks if the asset’s exchange is open at the current or next market minute (defined by the given trading calendar). --- tests/finance/test_slippage.py | 63 +++++--- tests/test_api_shim.py | 4 +- tests/test_bar_data.py | 266 +++++++++++++++++++++----------- tests/test_blotter.py | 2 + tests/test_finance.py | 3 +- tests/test_history.py | 39 +++-- zipline/_protocol.pyx | 54 +++++-- zipline/gens/tradesimulation.py | 1 + 8 files changed, 289 insertions(+), 143 deletions(-) diff --git a/tests/finance/test_slippage.py b/tests/finance/test_slippage.py index 4d96e335..2c6ae008 100644 --- a/tests/finance/test_slippage.py +++ b/tests/finance/test_slippage.py @@ -112,7 +112,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(data_portal, lambda: self.minutes[0], - 'minute') + 'minute', + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -153,7 +154,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): # Volume share slippage should not execute when there is no trade. bar_data = BarData(data_portal, lambda: self.minutes[1], - 'minute') + 'minute', + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -179,7 +181,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -201,7 +204,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -223,7 +227,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -262,7 +267,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[0], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -284,7 +290,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[0], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -306,7 +313,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[1], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -495,7 +503,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): dt = pd.Timestamp('2006-01-05 14:31', tz='UTC') bar_data = BarData(data_portal, lambda: dt, - 'minute') + 'minute', + self.trading_calendar) _, txn = next(slippage_model.simulate( bar_data, self.ASSET133, @@ -529,7 +538,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[2], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -541,7 +551,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -564,7 +575,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[2], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -576,7 +588,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -599,7 +612,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[2], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -611,7 +625,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[3], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -647,7 +662,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[0], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -659,7 +675,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[1], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -682,7 +699,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[0], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -694,7 +712,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[1], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -717,7 +736,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[0], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, @@ -729,7 +749,8 @@ class SlippageTestCase(WithSimParams, WithDataPortal, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: self.minutes[1], - self.sim_params.data_frequency) + self.sim_params.data_frequency, + self.trading_calendar) orders_txns = list(slippage_model.simulate( bar_data, diff --git a/tests/test_api_shim.py b/tests/test_api_shim.py index 634aa2f9..ee73a1bf 100644 --- a/tests/test_api_shim.py +++ b/tests/test_api_shim.py @@ -188,7 +188,9 @@ class TestAPIShim(WithDataPortal, WithSimParams, ZiplineTestCase): )[-1] bar_data = BarData( self.data_portal, - lambda: test_end_minute, "minute" + lambda: test_end_minute, + "minute", + self.trading_calendar ) ohlcvp_fields = [ "open", diff --git a/tests/test_bar_data.py b/tests/test_bar_data.py index 795a45b5..cafcbc4d 100644 --- a/tests/test_bar_data.py +++ b/tests/test_bar_data.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from datetime import timedelta +from itertools import chain + from nose_parameterized import parameterized import numpy as np from numpy import nan @@ -31,6 +33,7 @@ from zipline.testing.fixtures import ( WithDataPortal, ZiplineTestCase, ) +from zipline.utils.calendars import get_calendar OHLC = ["open", "high", "low", "close"] OHLCP = OHLC + ["price"] @@ -200,7 +203,8 @@ class TestMinuteBarData(WithBarDataChecks, # this entire day is before either asset has started trading for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) @@ -242,7 +246,8 @@ class TestMinuteBarData(WithBarDataChecks, # this test covers the "IPO morning" case, because asset2 only # has data starting on the 10th minute. - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.check_internal_consistency(bar_data) asset2_has_data = (((idx + 1) % 10) == 0) @@ -321,7 +326,8 @@ class TestMinuteBarData(WithBarDataChecks, # this is the last day the assets exist for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.assertTrue(bar_data.can_trade(self.ASSET1)) self.assertTrue(bar_data.can_trade(self.ASSET2)) @@ -339,7 +345,8 @@ class TestMinuteBarData(WithBarDataChecks, # this entire day is after both assets have stopped trading for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.assertFalse(bar_data.can_trade(self.ASSET1)) self.assertFalse(bar_data.can_trade(self.ASSET2)) @@ -381,7 +388,8 @@ class TestMinuteBarData(WithBarDataChecks, ) for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.assertEqual( idx + 1, bar_data.current(self.SPLIT_ASSET, "price") @@ -398,14 +406,16 @@ class TestMinuteBarData(WithBarDataChecks, ) for idx, minute in enumerate(day0_minutes[-10:-1]): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) self.assertEqual( 380, bar_data.current(self.ILLIQUID_SPLIT_ASSET, "price") ) bar_data = BarData( - self.data_portal, lambda: day0_minutes[-1], "minute" + self.data_portal, lambda: day0_minutes[-1], "minute", + self.trading_calendar ) self.assertEqual( @@ -414,7 +424,8 @@ class TestMinuteBarData(WithBarDataChecks, ) for idx, minute in enumerate(day1_minutes[0:9]): - bar_data = BarData(self.data_portal, lambda: minute, "minute") + bar_data = BarData(self.data_portal, lambda: minute, "minute", + self.trading_calendar) # should be half of 390, due to the split self.assertEqual( @@ -433,10 +444,12 @@ class TestMinuteBarData(WithBarDataChecks, tz='US/Eastern' ) - bar_data = BarData(self.data_portal, lambda: day, "minute") + bar_data = BarData(self.data_portal, lambda: day, "minute", + self.trading_calendar) bar_data2 = BarData(self.data_portal, lambda: eight_fortyfive_am_eastern, - "minute") + "minute", + self.trading_calendar) with handle_non_market_minutes(bar_data), \ handle_non_market_minutes(bar_data2): @@ -464,91 +477,152 @@ class TestMinuteBarData(WithBarDataChecks, bd.current(self.HILARIOUSLY_ILLIQUID_ASSET, "volume") ) - # FIXME temporarily commenting out until we restore the new can_trade - # functionality that checks exchange status - # def test_can_trade_during_non_market_hours(self): - # # make sure that if we use `can_trade` at midnight, we don't pretend - # # we're in the previous day's last minute - # the_day_after = self.trading_calendar.next_session_label( - # self.equity_minute_bar_days[-1] - # ) - # - # bar_data = BarData(self.data_portal, lambda: the_day_after, "minute") - # - # for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: - # self.assertFalse(bar_data.can_trade(asset)) - # - # with handle_non_market_minutes(bar_data): - # self.assertFalse(bar_data.can_trade(asset)) - # - # # NYSE is closed at midnight, so even if the asset is alive, - # # can_trade should return False - # bar_data2 = BarData( - # self.data_portal, - # lambda: self.equity_minute_bar_days[1], - # "minute", - # ) - # for asset in [self.ASSET1, self.HILARIOUSLY_ILLIQUID_ASSET]: - # self.assertFalse(bar_data2.can_trade(asset)) - # - # with handle_non_market_minutes(bar_data2): - # self.assertFalse(bar_data2.can_trade(asset)) + def test_can_trade_equity_same_cal_outside_lifetime(self): + cal = get_calendar(self.ASSET1.exchange) - # FIXME temporarily commenting out until we restore the new can_trade - # functionality that checks exchange status - # def test_can_trade_exchange_closed(self): - # nyse_asset = self.asset_finder.retrieve_asset(1) - # ice_asset = self.asset_finder.retrieve_asset(6) - # - # # minutes we're going to check (to verify that that the same bardata - # # can check multiple exchange calendars, all times Eastern): - # # 2016-01-05: - # # 20:00 (minute before ICE opens) - # # 20:01 (first minute of ICE session) - # # 20:02 (second minute of ICE session) - # # 00:00 (Cinderella's ride becomes a pumpkin) - # # 2016-01-06: - # # 9:30 (minute before NYSE opens) - # # 9:31 (first minute of NYSE session) - # # 9:32 (second minute of NYSE session) - # # 15:59 (second-to-last minute of NYSE session) - # # 16:00 (last minute of NYSE session) - # # 16:01 (minute after NYSE closed) - # # 17:59 (second-to-last minute of ICE session) - # # 18:00 (last minute of ICE session) - # # 18:01 (minute after ICE closed) - # - # # each row is dt, whether-nyse-is-open, whether-ice-is-open - # minutes_to_check = [ - # (pd.Timestamp("2016-01-05 20:00", tz="US/Eastern"), False, - # False), - # (pd.Timestamp("2016-01-05 20:01", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-05 20:02", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 00:00", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 9:30", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 9:31", tz="US/Eastern"), True, True), - # (pd.Timestamp("2016-01-06 9:32", tz="US/Eastern"), True, True), - # (pd.Timestamp("2016-01-06 15:59", tz="US/Eastern"), True, True), - # (pd.Timestamp("2016-01-06 16:00", tz="US/Eastern"), True, True), - # (pd.Timestamp("2016-01-06 16:01", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 17:59", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 18:00", tz="US/Eastern"), False, True), - # (pd.Timestamp("2016-01-06 18:01", tz="US/Eastern"), False, - # False), - # ] - # - # for info in minutes_to_check: - # bar_data = BarData(self.data_portal, lambda: info[0], "minute") - # series = bar_data.can_trade([nyse_asset, ice_asset]) - # - # self.assertEqual(info[1], series.loc[nyse_asset]) - # self.assertEqual(info[2], series.loc[ice_asset]) + # verify that can_trade returns False for the session before the + # asset's first session + session_before_asset1_start = cal.previous_session_label( + self.ASSET1.start_date + ) + minutes_for_session = cal.minutes_for_session( + session_before_asset1_start + ) + + # for good measure, check the minute before the session too + minutes_to_check = chain( + [minutes_for_session[0] - pd.Timedelta(minutes=1)], + minutes_for_session + ) + + for minute in minutes_to_check: + bar_data = BarData( + self.data_portal, lambda: minute, "minute", cal + ) + + self.assertFalse(bar_data.can_trade(self.ASSET1)) + + # after asset lifetime + session_after_asset1_end = cal.next_session_label( + self.ASSET1.end_date + ) + bts_after_asset1_end = session_after_asset1_end.replace( + hour=8, minute=45 + ).tz_convert(None).tz_localize("US/Eastern") + + minutes_to_check = chain( + cal.minutes_for_session(session_after_asset1_end), + [bts_after_asset1_end] + ) + + for minute in minutes_to_check: + bar_data = BarData( + self.data_portal, lambda: minute, "minute", cal + ) + + self.assertFalse(bar_data.can_trade(self.ASSET1)) + + def test_can_trade_equity_same_cal_exchange_closed(self): + cal = get_calendar(self.ASSET1.exchange) + + # verify that can_trade returns true for minutes that are + # outside the asset's calendar (assuming the asset is alive and + # there is a last price), because the asset is alive on the + # next market minute. + minutes = cal.minutes_for_sessions_in_range( + self.ASSET1.start_date, + self.ASSET1.end_date + ) + + for minute in minutes: + bar_data = BarData( + self.data_portal, lambda: minute, "minute", cal + ) + + self.assertTrue(bar_data.can_trade(self.ASSET1)) + + def test_can_trade_equity_same_cal_no_last_price(self): + # self.HILARIOUSLY_ILLIQUID_ASSET's first trade is at + # 2016-01-05 15:20:00+00:00. Make sure that can_trade returns false + # for all minutes in that session before the first trade, and true + # for all minutes afterwards. + cal = get_calendar(self.ASSET1.exchange) + + minutes_in_session = cal.minutes_for_session(self.ASSET1.start_date) + + for minute in minutes_in_session[0:49]: + bar_data = BarData( + self.data_portal, lambda: minute, "minute", cal + ) + + self.assertFalse(bar_data.can_trade( + self.HILARIOUSLY_ILLIQUID_ASSET) + ) + + for minute in minutes_in_session[50:]: + bar_data = BarData( + self.data_portal, lambda: minute, "minute", cal + ) + + self.assertTrue(bar_data.can_trade( + self.HILARIOUSLY_ILLIQUID_ASSET) + ) + + def test_can_trade_multiple_exchange_closed(self): + nyse_asset = self.asset_finder.retrieve_asset(1) + ice_asset = self.asset_finder.retrieve_asset(6) + + # minutes we're going to check (to verify that that the same bardata + # can check multiple exchange calendars, all times Eastern): + # 2016-01-05: + # 20:00 (minute before ICE opens) + # 20:01 (first minute of ICE session) + # 20:02 (second minute of ICE session) + # 00:00 (Cinderella's ride becomes a pumpkin) + # 2016-01-06: + # 9:30 (minute before NYSE opens) + # 9:31 (first minute of NYSE session) + # 9:32 (second minute of NYSE session) + # 15:59 (second-to-last minute of NYSE session) + # 16:00 (last minute of NYSE session) + # 16:01 (minute after NYSE closed) + # 17:59 (second-to-last minute of ICE session) + # 18:00 (last minute of ICE session) + # 18:01 (minute after ICE closed) + + # each row is dt, whether-nyse-is-open, whether-ice-is-open + minutes_to_check = [ + (pd.Timestamp("2016-01-05 20:00", tz="US/Eastern"), False, False), + (pd.Timestamp("2016-01-05 20:01", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-05 20:02", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 00:00", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 9:30", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 9:31", tz="US/Eastern"), True, True), + (pd.Timestamp("2016-01-06 9:32", tz="US/Eastern"), True, True), + (pd.Timestamp("2016-01-06 15:59", tz="US/Eastern"), True, True), + (pd.Timestamp("2016-01-06 16:00", tz="US/Eastern"), True, True), + (pd.Timestamp("2016-01-06 16:01", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 17:59", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 18:00", tz="US/Eastern"), False, True), + (pd.Timestamp("2016-01-06 18:01", tz="US/Eastern"), False, False), + ] + + for info in minutes_to_check: + # use the CME calendar, which covers 24 hours + bar_data = BarData(self.data_portal, lambda: info[0], "minute", + trading_calendar=get_calendar("CME")) + + series = bar_data.can_trade([nyse_asset, ice_asset]) + + self.assertEqual(info[1], series.loc[nyse_asset]) + self.assertEqual(info[2], series.loc[ice_asset]) def test_is_stale_during_non_market_hours(self): bar_data = BarData( self.data_portal, lambda: self.equity_minute_bar_days[1], "minute", + self.trading_calendar ) with handle_non_market_minutes(bar_data): @@ -578,7 +652,8 @@ class TestMinuteBarData(WithBarDataChecks, bar_data = BarData(self.data_portal, lambda: eight_fortyfive_am_eastern, - "minute") + "minute", + self.trading_calendar) expected = { 'open': 391 / 2.0, @@ -743,7 +818,8 @@ class TestDailyBarData(WithBarDataChecks, ) ) - bar_data = BarData(self.data_portal, lambda: minute, "daily") + bar_data = BarData(self.data_portal, lambda: minute, "daily", + self.trading_calendar) self.check_internal_consistency(bar_data) self.assertFalse(bar_data.can_trade(self.ASSET1)) @@ -771,6 +847,7 @@ class TestDailyBarData(WithBarDataChecks, self.equity_daily_bar_days[0] ), "daily", + self.trading_calendar ) self.check_internal_consistency(bar_data) @@ -808,6 +885,7 @@ class TestDailyBarData(WithBarDataChecks, self.equity_daily_bar_days[1] ), "daily", + self.trading_calendar ) self.check_internal_consistency(bar_data) @@ -834,6 +912,7 @@ class TestDailyBarData(WithBarDataChecks, self.equity_daily_bar_days[-1] ), "daily", + self.trading_calendar ) self.check_internal_consistency(bar_data) @@ -862,7 +941,8 @@ class TestDailyBarData(WithBarDataChecks, def test_after_assets_dead(self): session = self.END_DATE - bar_data = BarData(self.data_portal, lambda: session, "daily") + bar_data = BarData(self.data_portal, lambda: session, "daily", + self.trading_calendar) self.check_internal_consistency(bar_data) for asset in self.ASSETS: @@ -916,6 +996,7 @@ class TestDailyBarData(WithBarDataChecks, self.data_portal, lambda: self.equity_daily_bar_days[0], "daily", + self.trading_calendar ) self.assertEqual( liquid_day_0_price, @@ -925,6 +1006,7 @@ class TestDailyBarData(WithBarDataChecks, self.data_portal, lambda: self.equity_daily_bar_days[1], "daily", + self.trading_calendar ) self.assertEqual( liquid_day_1_price, @@ -937,6 +1019,7 @@ class TestDailyBarData(WithBarDataChecks, self.data_portal, lambda: self.equity_daily_bar_days[1], "daily", + self.trading_calendar ) self.assertEqual( illiquid_day_0_price, bar_data.current(illiquid_asset, "price") @@ -946,6 +1029,7 @@ class TestDailyBarData(WithBarDataChecks, self.data_portal, lambda: self.equity_daily_bar_days[2], "daily", + self.trading_calendar ) # 3 (price from previous day) * 0.5 (split ratio) diff --git a/tests/test_blotter.py b/tests/test_blotter.py index eed7ae91..89c251db 100644 --- a/tests/test_blotter.py +++ b/tests/test_blotter.py @@ -223,6 +223,7 @@ class BlotterTestCase(WithLogger, self.data_portal, lambda: self.sim_params.sessions[-1], self.sim_params.data_frequency, + self.trading_calendar ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: @@ -298,6 +299,7 @@ class BlotterTestCase(WithLogger, self.data_portal, lambda: dt, self.sim_params.data_frequency, + self.trading_calendar ) txns, _, _ = blotter.get_transactions(bar_data) for txn in txns: diff --git a/tests/test_finance.py b/tests/test_finance.py index 1de69f74..4b374a35 100644 --- a/tests/test_finance.py +++ b/tests/test_finance.py @@ -319,7 +319,8 @@ class FinanceTestCase(WithLogger, bar_data = BarData( data_portal, lambda: tick, - sim_params.data_frequency + sim_params.data_frequency, + self.trading_calendar ) txns, _, closed_orders = blotter.get_transactions(bar_data) for txn in txns: diff --git a/tests/test_history.py b/tests/test_history.py index d5307cc4..a68b7216 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -251,7 +251,8 @@ class WithHistory(WithDataPortal): fields = fields if fields is not None else ALL_FIELDS assets = assets if assets is not None else [self.ASSET2, self.ASSET3] - bar_data = BarData(self.data_portal, lambda: dt, mode) + bar_data = BarData(self.data_portal, lambda: dt, mode, + self.trading_calendar) check_internal_consistency( bar_data, assets, fields, 10, freq ) @@ -703,7 +704,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): )[0:60] for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, 'minute') + bar_data = BarData(self.data_portal, lambda: minute, 'minute', + self.trading_calendar) check_internal_consistency( bar_data, [self.ASSET2, self.ASSET3], ALL_FIELDS, 10, '1m' ) @@ -765,10 +767,12 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): )[1] midnight_bar_data = \ - BarData(self.data_portal, lambda: midnight, 'minute') + BarData(self.data_portal, lambda: midnight, 'minute', + self.trading_calendar) yesterday_bar_data = \ - BarData(self.data_portal, lambda: last_minute, 'minute') + BarData(self.data_portal, lambda: last_minute, 'minute', + self.trading_calendar) with handle_non_market_minutes(midnight_bar_data): for field in ALL_FIELDS: @@ -785,7 +789,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): )[0:60] for idx, minute in enumerate(minutes): - bar_data = BarData(self.data_portal, lambda: minute, 'minute') + bar_data = BarData(self.data_portal, lambda: minute, 'minute', + self.trading_calendar) check_internal_consistency( bar_data, self.SHORT_ASSET, ALL_FIELDS, 30, '1m' ) @@ -794,7 +799,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): data_portal = self.make_data_portal() # choose a window that contains the last minute of the asset - bar_data = BarData(data_portal, lambda: minutes[15], 'minute') + bar_data = BarData(data_portal, lambda: minutes[15], 'minute', + self.trading_calendar) # close high low open price volume # 2015-01-06 20:47:00+00:00 768 770 767 769 768 76800 @@ -1006,7 +1012,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): def test_passing_iterable_to_history_regular_hours(self): # regular hours current_dt = pd.Timestamp("2015-01-06 9:45", tz='US/Eastern') - bar_data = BarData(self.data_portal, lambda: current_dt, "minute") + bar_data = BarData(self.data_portal, lambda: current_dt, "minute", + self.trading_calendar) bar_data.history(pd.Index([self.ASSET1, self.ASSET2]), "high", 5, "1m") @@ -1014,7 +1021,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): def test_passing_iterable_to_history_bts(self): # before market hours current_dt = pd.Timestamp("2015-01-07 8:45", tz='US/Eastern') - bar_data = BarData(self.data_portal, lambda: current_dt, "minute") + bar_data = BarData(self.data_portal, lambda: current_dt, "minute", + self.trading_calendar) with handle_non_market_minutes(bar_data): bar_data.history(pd.Index([self.ASSET1, self.ASSET2]), @@ -1023,7 +1031,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): def test_overnight_adjustments(self): # Should incorporate adjustments on midnight 01/06 current_dt = pd.Timestamp('2015-01-06 8:45', tz='US/Eastern') - bar_data = BarData(self.data_portal, lambda: current_dt, 'minute') + bar_data = BarData(self.data_portal, lambda: current_dt, 'minute', + self.trading_calendar) adj_expected = { 'open': np.arange(8381, 8391) / 4.0, @@ -1394,7 +1403,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): ) for idx, day in enumerate(days): - bar_data = BarData(self.data_portal, lambda: day, 'daily') + bar_data = BarData(self.data_portal, lambda: day, 'daily', + self.trading_calendar) check_internal_consistency( bar_data, [self.ASSET2, self.ASSET3], ALL_FIELDS, 10, '1d' ) @@ -1437,7 +1447,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: pd.Timestamp('2016-01-06', tz='UTC'), - 'daily') + 'daily', + self.trading_calendar) for field in OHLCP: window = bar_data.history( @@ -1475,7 +1486,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): # days has 1/7, 1/8 for idx, day in enumerate(days): - bar_data = BarData(self.data_portal, lambda: day, 'daily') + bar_data = BarData(self.data_portal, lambda: day, 'daily', + self.trading_calendar) check_internal_consistency( bar_data, self.SHORT_ASSET, ALL_FIELDS, 2, '1d' ) @@ -1629,7 +1641,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): bar_data = BarData(self.data_portal, lambda: pd.Timestamp('2016-01-06 16:00', tz='UTC'), - 'daily') + 'daily', + self.trading_calendar) for field in OHLCP: window = bar_data.history( diff --git a/zipline/_protocol.pyx b/zipline/_protocol.pyx index fe7851ce..a1b68a05 100644 --- a/zipline/_protocol.pyx +++ b/zipline/_protocol.pyx @@ -165,14 +165,12 @@ cdef class BarData: cdef object _last_calculated_universe cdef object _universe_last_updated_at cdef bool _daily_mode + cdef object _trading_calendar cdef bool _adjust_minutes def __init__(self, data_portal, simulation_dt_func, data_frequency, - universe_func=None): - """ - - """ + trading_calendar, universe_func=None): self.data_portal = data_portal self.simulation_dt_func = simulation_dt_func self.data_frequency = data_frequency @@ -186,6 +184,8 @@ cdef class BarData: self._adjust_minutes = False + self._trading_calendar = trading_calendar + cdef _get_equity_price_view(self, asset): """ Returns a DataPortalSidView for the given asset. Used to support the @@ -428,9 +428,25 @@ cdef class BarData: """ For the given asset or iterable of assets, returns true if all of the following are true: - - the asset is alive at the current simulation time - - the asset's exchange is open at the current simulation time - - there is a known last price for the asset. + 1) the asset is alive for the session of the current simulation time + (if current simulation time is not a market minute, we use the next + session) + 2) (if we are in minute mode) the asset's exchange is open at the + current simulation time or at the simulation calendar's next market + minute + 3) there is a known last price for the asset. + + Notes + ----- + The second condition above warrants some further explanation. + - If the asset's exchange calendar is identical to the simulation + calendar, then this condition always returns True. + - If there are market minutes in the simulation calendar outside of + this asset's exchange's trading hours (for example, if the simulation + is running on the CME calendar but the asset is MSFT, which trades on + the NYSE), during those minutes, this condition will return false + (for example, 3:15 am Eastern on a weekday, during which the CME is + open but the NYSE is closed). Parameters ---------- @@ -462,20 +478,26 @@ cdef class BarData: }) cdef bool _can_trade_for_asset(self, asset, dt, adjusted_dt, data_portal): - session_label = normalize_date(dt) # FIXME + cdef object session_label + cdef object dt_to_use_for_exchange_check, + + session_label = self._trading_calendar.minute_to_session_label(dt) + if not asset.is_alive_for_session(session_label): # asset isn't alive return False - # FIXME temporarily commenting out while we sort out some downstream - # dependencies - # if not asset.is_exchange_open(dt): - # # exchange isn't open - # return False + if not self._daily_mode: + # Find the next market minute for this calendar, and check if this + # asset's exchange is open at that minute. + if self._trading_calendar.is_open_on_minute(dt): + dt_to_use_for_exchange_check = dt + else: + dt_to_use_for_exchange_check = \ + self._trading_calendar.next_open(dt) - if isinstance(asset, Future): - # FIXME: this will get removed once we can get prices for futures - return True + if not asset.is_exchange_open(dt_to_use_for_exchange_check): + return False # is there a last price? return not np.isnan( diff --git a/zipline/gens/tradesimulation.py b/zipline/gens/tradesimulation.py index 31264828..2ab816d7 100644 --- a/zipline/gens/tradesimulation.py +++ b/zipline/gens/tradesimulation.py @@ -88,6 +88,7 @@ class AlgorithmSimulator(object): data_portal=self.data_portal, simulation_dt_func=self.get_simulation_dt, data_frequency=self.sim_params.data_frequency, + trading_calendar=self.algo.trading_calendar, universe_func=universe_func )