diff --git a/tests/pipeline/test_pipeline_algo.py b/tests/pipeline/test_pipeline_algo.py index bf176153..436c08f6 100644 --- a/tests/pipeline/test_pipeline_algo.py +++ b/tests/pipeline/test_pipeline_algo.py @@ -26,6 +26,7 @@ from pandas import ( Series, Timestamp, ) +from pandas.tseries.tools import normalize_date from six import iteritems, itervalues from zipline.algorithm import TradingAlgorithm @@ -512,7 +513,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs, attach_pipeline(pipeline, 'test') def handle_data(context, data): - today = get_datetime() + today = normalize_date(get_datetime()) results = pipeline_output('test') expect_over_300 = { AAPL: today < self.AAPL_split_date, diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 5222517e..eacff880 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -3731,12 +3731,18 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): transactions = output['transactions'] initial_fills = transactions.iloc[1] self.assertEqual(len(initial_fills), len(assets)) + + last_minute_of_session = \ + self.trading_calendar.open_and_close_for_session( + self.test_days[1] + )[1] + for sid, txn in zip(sids, initial_fills): self.assertDictContainsSubset( { 'amount': order_size, 'commission': None, - 'dt': self.test_days[1], + 'dt': last_minute_of_session, 'price': initial_fill_prices[sid], 'sid': sid, }, @@ -3803,15 +3809,17 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): context.portfolio.cash == context.portfolio.starting_cash ) - now = context.get_datetime() + today_session = self.trading_calendar.minute_to_session_label( + context.get_datetime() + ) - if now == first_asset_end_date: + if today_session == first_asset_end_date: # Equity 0 will no longer exist tomorrow, so this order will # never be filled. assert len(context.get_open_orders()) == 0 context.order(context.sid(0), 10) assert len(context.get_open_orders()) == 1 - elif now == first_asset_auto_close_date: + elif today_session == first_asset_auto_close_date: assert len(context.get_open_orders()) == 0 algo = TradingAlgorithm( @@ -3829,12 +3837,18 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): original_open_orders = orders_for_date(first_asset_end_date) assert len(original_open_orders) == 1 + + last_close_for_asset = \ + algo.trading_calendar.open_and_close_for_session( + first_asset_end_date + )[1] + self.assertDictContainsSubset( { 'amount': 10, 'commission': 0, - 'created': first_asset_end_date, - 'dt': first_asset_end_date, + 'created': last_close_for_asset, + 'dt': last_close_for_asset, 'sid': assets[0], 'status': ORDER_STATUS.OPEN, 'filled': 0, @@ -3848,7 +3862,7 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): { 'amount': 10, 'commission': 0, - 'created': first_asset_end_date, + 'created': last_close_for_asset, 'dt': first_asset_auto_close_date, 'sid': assets[0], 'status': ORDER_STATUS.CANCELLED, diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index 05760e8b..4b5cc060 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -417,6 +417,7 @@ def handle_data(context, data): algocode = """ from pandas import Timestamp +from pandas.tseries.tools import normalize_date from zipline.api import fetch_csv, record, sid, get_datetime def initialize(context): @@ -432,7 +433,7 @@ def initialize(context): context.bar_count = 0 def handle_data(context, data): - expected = context.expected_sids[get_datetime()] + expected = context.expected_sids[normalize_date(get_datetime())] actual = data.fetcher_assets for stk in expected: if stk not in actual: diff --git a/zipline/_protocol.pyx b/zipline/_protocol.pyx index 122f5f82..33d0df5b 100644 --- a/zipline/_protocol.pyx +++ b/zipline/_protocol.pyx @@ -164,6 +164,7 @@ cdef class BarData: cdef object _universe_func cdef object _last_calculated_universe cdef object _universe_last_updated_at + cdef bool _daily_mode cdef bool _adjust_minutes @@ -177,6 +178,8 @@ cdef class BarData: self.data_frequency = data_frequency self._views = {} + self._daily_mode = (self.data_frequency == "daily") + self._universe_func = universe_func self._last_calculated_universe = None self._universe_last_updated_at = None @@ -220,12 +223,28 @@ cdef class BarData: ) cdef _get_current_minute(self): + """ + Internal utility method to get the current simulation time. + + Possible answers are: + - whatever the algorithm's get_datetime() method returns (this is what + `self.simulation_dt_func()` points to) + - sometimes we're knowingly not in a market minute, like if we're in + before_trading_start. In that case, `self._adjust_minutes` is + True, and we get the previous market minute. + - if we're in daily mode, get the session label for this minute. + """ dt = self.simulation_dt_func() if self._adjust_minutes: dt = \ self.data_portal.trading_calendar.previous_minute(dt) + if self._daily_mode: + # if we're in daily mode, take the given dt (which is the last + # minute of the session) and get the session label for it. + dt = self.data_portal.trading_calendar.minute_to_session_label(dt) + return dt @check_parameters(('assets', 'fields'), ((Asset, str), str)) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 0584d337..c8f68078 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -119,10 +119,7 @@ from zipline.utils.preprocess import preprocess import zipline.protocol from zipline.sources.requests_csv import PandasRequestsCSV -from zipline.gens.sim_engine import ( - MinuteSimulationClock, - DailySimulationClock, -) +from zipline.gens.sim_engine import MinuteSimulationClock from zipline.sources.benchmark_source import BenchmarkSource from zipline.zipline_warnings import ZiplineDeprecationWarning @@ -497,25 +494,31 @@ class TradingAlgorithm(object): """ If the clock property is not set, then create one based on frequency. """ + trading_o_and_c = self.trading_calendar.schedule.ix[ + self.sim_params.sessions] + market_closes = trading_o_and_c['market_close'].values.astype(np.int64) + if self.sim_params.data_frequency == 'minute': - trading_o_and_c = self.trading_calendar.schedule.ix[ - self.sim_params.sessions] market_opens = trading_o_and_c['market_open'].values.astype( - 'datetime64[ns]').astype(np.int64) - market_closes = trading_o_and_c['market_close'].values.astype( - 'datetime64[ns]').astype(np.int64) + np.int64) minutely_emission = self.sim_params.emission_rate == "minute" - clock = MinuteSimulationClock( + return MinuteSimulationClock( self.sim_params.sessions, market_opens, market_closes, minutely_emission ) - return clock else: - return DailySimulationClock(self.sim_params.sessions) + # in daily mode, we want to have one bar per session, timestamped + # as the last minute of the session. + return MinuteSimulationClock( + self.sim_params.sessions, + market_closes, + market_closes, + False + ) def _create_benchmark_source(self): return BenchmarkSource( diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 50850015..dcb3e0b1 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -773,19 +773,20 @@ class DataPortal(object): if field not in BASE_FIELDS: raise KeyError("Invalid column: " + str(field)) + session_label = self.trading_calendar.minute_to_session_label(dt) + if dt < asset.start_date or \ - (data_frequency == "daily" and dt > asset.end_date) or \ + (data_frequency == "daily" and + session_label > asset.end_date) or \ (data_frequency == "minute" and - normalize_date(dt) > asset.end_date): + session_label > asset.end_date): if field == "volume": return 0 elif field != "last_traded": return np.NaN if data_frequency == "daily": - day_to_use = dt - day_to_use = normalize_date(day_to_use) - return self._get_daily_data(asset, field, day_to_use) + return self._get_daily_data(asset, field, session_label) else: if isinstance(asset, Future): if field == "price": diff --git a/zipline/gens/sim_engine.pyx b/zipline/gens/sim_engine.pyx index 02d0472c..5ad89682 100644 --- a/zipline/gens/sim_engine.pyx +++ b/zipline/gens/sim_engine.pyx @@ -84,17 +84,3 @@ cdef class MinuteSimulationClock: yield minute, MINUTE_END yield minutes[-1], DAY_END - - - -cdef class DailySimulationClock: - cdef object trading_days - - def __init__(self, trading_days): - self.trading_days = trading_days - - def __iter__(self): - for i, day in enumerate(self.trading_days): - yield day, DAY_START - yield day, BAR - yield day, DAY_END