From 2462929368552fcd3db7e662d47b4ba83166e87b Mon Sep 17 00:00:00 2001 From: Jean Bredeche Date: Tue, 26 Jul 2016 16:20:14 -0400 Subject: [PATCH] Revert "Merge pull request #1340 from quantopian/by-daily-i-mean-minutely" This reverts commit f4456719b047f24ca9f8226e78f21496608e1f51, reversing changes made to 4be07e4628d0870784994da31d6d74d53cb12c17. --- tests/pipeline/test_pipeline_algo.py | 3 +-- tests/test_algorithm.py | 28 +++++++--------------------- tests/test_fetcher.py | 3 +-- zipline/_protocol.pyx | 19 ------------------- zipline/algorithm.py | 27 ++++++++++++--------------- zipline/data/data_portal.py | 11 +++++------ zipline/gens/sim_engine.pyx | 14 ++++++++++++++ 7 files changed, 40 insertions(+), 65 deletions(-) diff --git a/tests/pipeline/test_pipeline_algo.py b/tests/pipeline/test_pipeline_algo.py index 436c08f6..bf176153 100644 --- a/tests/pipeline/test_pipeline_algo.py +++ b/tests/pipeline/test_pipeline_algo.py @@ -26,7 +26,6 @@ from pandas import ( Series, Timestamp, ) -from pandas.tseries.tools import normalize_date from six import iteritems, itervalues from zipline.algorithm import TradingAlgorithm @@ -513,7 +512,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs, attach_pipeline(pipeline, 'test') def handle_data(context, data): - today = normalize_date(get_datetime()) + today = get_datetime() results = pipeline_output('test') expect_over_300 = { AAPL: today < self.AAPL_split_date, diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index eacff880..5222517e 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -3731,18 +3731,12 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): transactions = output['transactions'] initial_fills = transactions.iloc[1] self.assertEqual(len(initial_fills), len(assets)) - - last_minute_of_session = \ - self.trading_calendar.open_and_close_for_session( - self.test_days[1] - )[1] - for sid, txn in zip(sids, initial_fills): self.assertDictContainsSubset( { 'amount': order_size, 'commission': None, - 'dt': last_minute_of_session, + 'dt': self.test_days[1], 'price': initial_fill_prices[sid], 'sid': sid, }, @@ -3809,17 +3803,15 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): context.portfolio.cash == context.portfolio.starting_cash ) - today_session = self.trading_calendar.minute_to_session_label( - context.get_datetime() - ) + now = context.get_datetime() - if today_session == first_asset_end_date: + if now == first_asset_end_date: # Equity 0 will no longer exist tomorrow, so this order will # never be filled. assert len(context.get_open_orders()) == 0 context.order(context.sid(0), 10) assert len(context.get_open_orders()) == 1 - elif today_session == first_asset_auto_close_date: + elif now == first_asset_auto_close_date: assert len(context.get_open_orders()) == 0 algo = TradingAlgorithm( @@ -3837,18 +3829,12 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): original_open_orders = orders_for_date(first_asset_end_date) assert len(original_open_orders) == 1 - - last_close_for_asset = \ - algo.trading_calendar.open_and_close_for_session( - first_asset_end_date - )[1] - self.assertDictContainsSubset( { 'amount': 10, 'commission': 0, - 'created': last_close_for_asset, - 'dt': last_close_for_asset, + 'created': first_asset_end_date, + 'dt': first_asset_end_date, 'sid': assets[0], 'status': ORDER_STATUS.OPEN, 'filled': 0, @@ -3862,7 +3848,7 @@ class TestEquityAutoClose(WithTmpDir, WithTradingCalendar, ZiplineTestCase): { 'amount': 10, 'commission': 0, - 'created': last_close_for_asset, + 'created': first_asset_end_date, 'dt': first_asset_auto_close_date, 'sid': assets[0], 'status': ORDER_STATUS.CANCELLED, diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index 4b5cc060..05760e8b 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -417,7 +417,6 @@ def handle_data(context, data): algocode = """ from pandas import Timestamp -from pandas.tseries.tools import normalize_date from zipline.api import fetch_csv, record, sid, get_datetime def initialize(context): @@ -433,7 +432,7 @@ def initialize(context): context.bar_count = 0 def handle_data(context, data): - expected = context.expected_sids[normalize_date(get_datetime())] + expected = context.expected_sids[get_datetime()] actual = data.fetcher_assets for stk in expected: if stk not in actual: diff --git a/zipline/_protocol.pyx b/zipline/_protocol.pyx index 33d0df5b..122f5f82 100644 --- a/zipline/_protocol.pyx +++ b/zipline/_protocol.pyx @@ -164,7 +164,6 @@ cdef class BarData: cdef object _universe_func cdef object _last_calculated_universe cdef object _universe_last_updated_at - cdef bool _daily_mode cdef bool _adjust_minutes @@ -178,8 +177,6 @@ cdef class BarData: self.data_frequency = data_frequency self._views = {} - self._daily_mode = (self.data_frequency == "daily") - self._universe_func = universe_func self._last_calculated_universe = None self._universe_last_updated_at = None @@ -223,28 +220,12 @@ cdef class BarData: ) cdef _get_current_minute(self): - """ - Internal utility method to get the current simulation time. - - Possible answers are: - - whatever the algorithm's get_datetime() method returns (this is what - `self.simulation_dt_func()` points to) - - sometimes we're knowingly not in a market minute, like if we're in - before_trading_start. In that case, `self._adjust_minutes` is - True, and we get the previous market minute. - - if we're in daily mode, get the session label for this minute. - """ dt = self.simulation_dt_func() if self._adjust_minutes: dt = \ self.data_portal.trading_calendar.previous_minute(dt) - if self._daily_mode: - # if we're in daily mode, take the given dt (which is the last - # minute of the session) and get the session label for it. - dt = self.data_portal.trading_calendar.minute_to_session_label(dt) - return dt @check_parameters(('assets', 'fields'), ((Asset, str), str)) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index c8f68078..0584d337 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -119,7 +119,10 @@ from zipline.utils.preprocess import preprocess import zipline.protocol from zipline.sources.requests_csv import PandasRequestsCSV -from zipline.gens.sim_engine import MinuteSimulationClock +from zipline.gens.sim_engine import ( + MinuteSimulationClock, + DailySimulationClock, +) from zipline.sources.benchmark_source import BenchmarkSource from zipline.zipline_warnings import ZiplineDeprecationWarning @@ -494,31 +497,25 @@ class TradingAlgorithm(object): """ If the clock property is not set, then create one based on frequency. """ - trading_o_and_c = self.trading_calendar.schedule.ix[ - self.sim_params.sessions] - market_closes = trading_o_and_c['market_close'].values.astype(np.int64) - if self.sim_params.data_frequency == 'minute': + trading_o_and_c = self.trading_calendar.schedule.ix[ + self.sim_params.sessions] market_opens = trading_o_and_c['market_open'].values.astype( - np.int64) + 'datetime64[ns]').astype(np.int64) + market_closes = trading_o_and_c['market_close'].values.astype( + 'datetime64[ns]').astype(np.int64) minutely_emission = self.sim_params.emission_rate == "minute" - return MinuteSimulationClock( + clock = MinuteSimulationClock( self.sim_params.sessions, market_opens, market_closes, minutely_emission ) + return clock else: - # in daily mode, we want to have one bar per session, timestamped - # as the last minute of the session. - return MinuteSimulationClock( - self.sim_params.sessions, - market_closes, - market_closes, - False - ) + return DailySimulationClock(self.sim_params.sessions) def _create_benchmark_source(self): return BenchmarkSource( diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index dcb3e0b1..50850015 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -773,20 +773,19 @@ class DataPortal(object): if field not in BASE_FIELDS: raise KeyError("Invalid column: " + str(field)) - session_label = self.trading_calendar.minute_to_session_label(dt) - if dt < asset.start_date or \ - (data_frequency == "daily" and - session_label > asset.end_date) or \ + (data_frequency == "daily" and dt > asset.end_date) or \ (data_frequency == "minute" and - session_label > asset.end_date): + normalize_date(dt) > asset.end_date): if field == "volume": return 0 elif field != "last_traded": return np.NaN if data_frequency == "daily": - return self._get_daily_data(asset, field, session_label) + day_to_use = dt + day_to_use = normalize_date(day_to_use) + return self._get_daily_data(asset, field, day_to_use) else: if isinstance(asset, Future): if field == "price": diff --git a/zipline/gens/sim_engine.pyx b/zipline/gens/sim_engine.pyx index 5ad89682..02d0472c 100644 --- a/zipline/gens/sim_engine.pyx +++ b/zipline/gens/sim_engine.pyx @@ -84,3 +84,17 @@ cdef class MinuteSimulationClock: yield minute, MINUTE_END yield minutes[-1], DAY_END + + + +cdef class DailySimulationClock: + cdef object trading_days + + def __init__(self, trading_days): + self.trading_days = trading_days + + def __iter__(self): + for i, day in enumerate(self.trading_days): + yield day, DAY_START + yield day, BAR + yield day, DAY_END