From b1428aaad153621787f6e9bf84424cf0446e3034 Mon Sep 17 00:00:00 2001 From: Jean Bredeche Date: Mon, 6 Jun 2016 16:28:30 -0400 Subject: [PATCH] DEV: Cleaned up `trading_minute_window` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed it from ExchangeCalendar. Fixed TradingSchedule’s implementation to be much faster. Removed the `step` parameter. --- tests/test_exchange_calendar.py | 10 ---- tests/test_trading_schedule.py | 51 +++++++++++++++++ zipline/data/data_portal.py | 35 +++++++----- zipline/utils/calendars/calendar_helpers.py | 58 -------------------- zipline/utils/calendars/exchange_calendar.py | 10 ---- zipline/utils/calendars/trading_schedule.py | 19 +++---- 6 files changed, 81 insertions(+), 102 deletions(-) diff --git a/tests/test_exchange_calendar.py b/tests/test_exchange_calendar.py index cb75db97..6c9ae0ad 100644 --- a/tests/test_exchange_calendar.py +++ b/tests/test_exchange_calendar.py @@ -156,16 +156,6 @@ class ExchangeCalendarTestBase(object): self.assertIsNotNone(open, "Open value is None") self.assertIsNotNone(close, "Close value is None") - # def test_minutes_for_date(self): - # for date in self.answers.index: - # mins_for_date = self.calendar.minutes_for_date(date) - - def test_minute_window(self): - for open in self.answers.market_open: - open_tz = open.tz_localize('UTC') - window = self.calendar.trading_minute_window(open_tz, 390, step=1) - self.assertEqual(len(window), 390) - class NYSECalendarTestCase(ExchangeCalendarTestBase, TestCase): diff --git a/tests/test_trading_schedule.py b/tests/test_trading_schedule.py index cba11678..f897204e 100644 --- a/tests/test_trading_schedule.py +++ b/tests/test_trading_schedule.py @@ -3,8 +3,11 @@ from unittest import TestCase from pandas import ( Timestamp, date_range, + DatetimeIndex ) +import numpy as np + from zipline.utils.calendars import ( get_calendar, ExchangeTradingSchedule, @@ -56,3 +59,51 @@ class TestExchangeTradingSchedule(TestCase): "Mismatch between schedule: %s and calendar: %s at time %s" % (cal_open, sched_exec, dt) ) + + def test_execution_minute_window_forward(self): + dt = Timestamp("11/23/2016 15:00", tz='EST').tz_convert("UTC") + + # 61 minutes left on 11/23, closed 11/24, only 210 minutes on 11/25 + minutes = self.nyse_exchange_schedule.execution_minute_window(dt, 300) + + np.testing.assert_array_equal( + minutes[0:61], + DatetimeIndex( + start=Timestamp("2016-11-23 20:00", tz='UTC'), + end=Timestamp("2016-11-23 21:00", tz='UTC'), + freq="min" + ) + ) + + np.testing.assert_array_equal( + minutes[61:271], + DatetimeIndex( + start=Timestamp("2016-11-25 14:31", tz='UTC'), + end=Timestamp("2016-11-25 18:00", tz='UTC'), + freq="min" + ) + ) + + np.testing.assert_array_equal( + minutes[271:], + DatetimeIndex( + start=Timestamp("2016-11-28 14:31", tz='UTC'), + end=Timestamp("2016-11-28 14:59", tz='UTC'), + freq="min" + ) + ) + + def test_execution_minute_window_backward(self): + end_dt = Timestamp("2016-11-28 14:59", tz='UTC') + start_dt = Timestamp("2016-11-23 20:00", tz='UTC') + + from_end_minutes = \ + self.nyse_exchange_schedule.execution_minute_window(end_dt, -300) + + from_start_minutes = \ + self.nyse_exchange_schedule.execution_minute_window(start_dt, 300) + + np.testing.assert_array_equal( + from_end_minutes, + from_start_minutes + ) diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 14523c68..57806ef9 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -1182,6 +1182,19 @@ class DataPortal(object): return daily_data + def _handle_history_out_of_bounds(self, bar_count): + suggested_start_day = ( + self.trading_schedule.all_execution_minutes[ + self._first_trading_minute_loc + bar_count + ] + self.trading_schedule.day + ).date() + + raise HistoryWindowStartsBeforeData( + first_trading_day=self._first_trading_day.date(), + bar_count=bar_count, + suggested_start_day=suggested_start_day, + ) + def _get_history_minute_window(self, assets, end_dt, bar_count, field_to_use): """ @@ -1189,21 +1202,15 @@ class DataPortal(object): of minute frequency for the given sids. """ # get all the minutes for this window - mm = self.trading_schedule.all_execution_minutes - end_loc = mm.get_loc(end_dt) - start_loc = end_loc - bar_count + 1 - if start_loc < self._first_trading_minute_loc: - suggested_start_day = ( - mm[ - self._first_trading_minute_loc + bar_count - ] + self.trading_schedule.day - ).date() - raise HistoryWindowStartsBeforeData( - first_trading_day=self._first_trading_day.date(), - bar_count=bar_count, - suggested_start_day=suggested_start_day, + try: + minutes_for_window = self.trading_schedule.execution_minute_window( + end_dt, -bar_count ) - minutes_for_window = mm[start_loc:end_loc + 1] + except KeyError: + self._handle_history_out_of_bounds(bar_count) + + if minutes_for_window[0] < self._first_trading_minute: + self._handle_history_out_of_bounds(bar_count) asset_minute_data = self._get_minute_window_for_assets( assets, diff --git a/zipline/utils/calendars/calendar_helpers.py b/zipline/utils/calendars/calendar_helpers.py index 2e2f7162..e6ea29cc 100644 --- a/zipline/utils/calendars/calendar_helpers.py +++ b/zipline/utils/calendars/calendar_helpers.py @@ -16,7 +16,6 @@ import pandas as pd import numpy as np import bisect -from pytz import timezone from zipline.errors import NoFurtherDataError @@ -238,60 +237,3 @@ def previous_scheduled_minute(start, is_scheduled_day_hook, # If start is not a trading day, or is before the market open # then return the close of the *previous* trading day. return previous_open_and_close_hook(start)[1] - - -def minute_window(start, count, step, schedule, is_scheduled_minute_hook, - session_date_hook, minutes_for_date_hook): - """ - Returns a DatetimeIndex containing `count` market minutes, starting - with `start` and continuing `step` minutes at a time. - - Parameters - ---------- - start : Timestamp - The start of the window. - count : int - The number of minutes needed. - step : int - The step size by which to increment. - - Returns - ------- - DatetimeIndex - A window with @count minutes, start with @start. - """ - if not is_scheduled_minute_hook(start): - raise ValueError("minute_window starting at non-market time " - "{minute}".format(minute=start)) - - start_utc = start.astimezone(timezone('UTC')) - - session = session_date_hook(start) - session_idx = schedule.index.get_loc(session) - - mins_in_session = minutes_for_date_hook(session) - start_idx = mins_in_session.searchsorted(start_utc) - - # Use a list instead of a pandas DatetimeIndex, as using .append() - # with DatetimeIndex can become expensive if used several times, since - # it makes a full copy of the data. list.extend() will not typically - # copy the data unless there is not enough memory to extend into, which - # is usually not problem. - all_minutes = list(mins_in_session[start_idx::np.sign(step)]) - - while True: - - step_minutes = all_minutes[0::np.absolute(step)] - - if len(step_minutes) >= count: - step_minutes = step_minutes[:count] - return pd.DatetimeIndex(step_minutes, copy=False) - - # Iterate session forward or backward - session_idx += np.sign(step) - # Get the minutes in the next exchange session - session = schedule.index[session_idx] - session_minutes = minutes_for_date_hook(session)[::np.sign(step)] - - # A these new session_minutes to the `all_minutes` candidate list - all_minutes.extend(list(session_minutes)) diff --git a/zipline/utils/calendars/exchange_calendar.py b/zipline/utils/calendars/exchange_calendar.py index 5ec6c4a2..3f1cbb4f 100644 --- a/zipline/utils/calendars/exchange_calendar.py +++ b/zipline/utils/calendars/exchange_calendar.py @@ -47,7 +47,6 @@ from .calendar_helpers import ( add_scheduled_days, next_scheduled_minute, previous_scheduled_minute, - minute_window, ) start_default = pd.Timestamp('1990-01-01', tz='UTC') @@ -286,15 +285,6 @@ class ExchangeCalendar(with_metaclass(ABCMeta)): previous_open_and_close_hook=self.previous_open_and_close, ) - def trading_minute_window(self, start, count, step=1): - return minute_window( - start, count, step, - schedule=self.schedule, - is_scheduled_minute_hook=self.is_open_on_minute, - session_date_hook=self.session_date, - minutes_for_date_hook=self.trading_minutes_for_day, - ) - def _special_dates(self, calendars, ad_hoc_dates, start_date, end_date): """ Union an iterable of pairs of the form diff --git a/zipline/utils/calendars/trading_schedule.py b/zipline/utils/calendars/trading_schedule.py index 3cf02cb6..f0aaa0f0 100644 --- a/zipline/utils/calendars/trading_schedule.py +++ b/zipline/utils/calendars/trading_schedule.py @@ -33,8 +33,7 @@ from .calendar_helpers import ( add_scheduled_days, all_scheduled_minutes, next_scheduled_minute, - previous_scheduled_minute, - minute_window, + previous_scheduled_minute ) @@ -137,14 +136,14 @@ class TradingSchedule(with_metaclass(ABCMeta)): previous_open_and_close_hook=self.previous_start_and_end, ) - def execution_minute_window(self, start, count, step=1): - return minute_window( - start, count, step, - schedule=self.schedule, - is_scheduled_minute_hook=self.is_executing_on_minute, - session_date_hook=self.session_date, - minutes_for_date_hook=self.execution_minutes_for_day, - ) + def execution_minute_window(self, start, count): + start_idx = self.all_execution_minutes.get_loc(start) + end_idx = start_idx + count + + if start_idx > end_idx: + return self.all_execution_minutes[(end_idx + 1):(start_idx + 1)] + else: + return self.all_execution_minutes[start_idx:end_idx] @abstractproperty def day(self):