From d155d894fe7175bdac7ba5b18c9562229f96b7d3 Mon Sep 17 00:00:00 2001 From: Andrew Daniels Date: Thu, 4 May 2017 16:06:15 -0400 Subject: [PATCH] MAINT: Pass data_frequency to get_history_window This allows us to remove the check for whether the provided dt had a time of midnight, which was a flimsy way to infer if the data frequency was 'daily'. Besides the explicit check being preferable, this method was broken on the futures calendar, since midnight is a valid market minute. --- tests/test_api_shim.py | 2 + tests/test_benchmark.py | 8 +- tests/test_continuous_futures.py | 52 +++++++----- tests/test_history.py | 126 ++++++++++++++++++---------- zipline/_protocol.pyx | 9 +- zipline/algorithm.py | 2 + zipline/data/data_portal.py | 58 +++++++++---- zipline/sources/benchmark_source.py | 3 + zipline/testing/core.py | 2 +- 9 files changed, 176 insertions(+), 86 deletions(-) diff --git a/tests/test_api_shim.py b/tests/test_api_shim.py index 4773e57f..614df1e0 100644 --- a/tests/test_api_shim.py +++ b/tests/test_api_shim.py @@ -265,6 +265,7 @@ class TestAPIShim(WithCreateBarData, 5, "1m", "volume", + "minute", True ) else: @@ -274,6 +275,7 @@ class TestAPIShim(WithCreateBarData, 5, "1m", "volume", + "minute", ) test_sim_params = SimulationParameters( diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 7abe4604..fced35a9 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -101,7 +101,12 @@ class TestBenchmark(WithDataPortal, WithSimParams, WithTradingCalendars, # should be the equivalent of getting the price history, then doing # a pct_change on it manually_calculated = self.data_portal.get_history_window( - [1], days_to_use[-1], len(days_to_use), "1d", "close" + [1], + days_to_use[-1], + len(days_to_use), + "1d", + "close", + "daily", )[1].pct_change() # compare all the fields except the first one, for which we don't have @@ -187,6 +192,7 @@ class TestBenchmark(WithDataPortal, WithSimParams, WithTradingCalendars, len(days_to_use), "1d", "close", + "daily", )[2].pct_change() for idx, day in enumerate(days_to_use[1:]): diff --git a/tests/test_continuous_futures.py b/tests/test_continuous_futures.py index 4f3d73d8..b05287c3 100644 --- a/tests/test_continuous_futures.py +++ b/tests/test_continuous_futures.py @@ -670,7 +670,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-01-26', cf], 0, @@ -696,7 +696,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-02-25', cf], 1, @@ -724,7 +724,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-01-11 18:01', tz='US/Eastern').tz_convert('UTC'), - 3, '1d', 'sid') + 3, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-01-08', cf], 10, @@ -745,7 +745,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-01-26', cf], 1, @@ -771,7 +771,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-02-25', cf], 2, @@ -799,7 +799,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') # Volume cuts out for FOF16 on 2016-01-25 self.assertEqual(window.loc['2016-01-26', cf], @@ -826,7 +826,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1d', 'sid') + 30, '1d', 'sid', 'minute') self.assertEqual(window.loc['2016-02-25', cf], 1, @@ -863,7 +863,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf.sid], Timestamp('2016-01-26 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'sid') + 30, '1m', 'sid', 'minute') self.assertEqual(window.loc['2016-01-26 22:32', cf], 0, @@ -882,7 +882,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-01-27 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'sid') + 30, '1m', 'sid', 'minute') self.assertEqual(window.loc['2016-01-27 22:32', cf], 1, @@ -896,7 +896,9 @@ def record_current_contract(algo, data): cf = self.data_portal.asset_finder.create_continuous_future( 'FO', 0, 'calendar', None) window = self.data_portal.get_history_window( - [cf.sid], Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close') + [cf.sid], + Timestamp('2016-03-06', tz='UTC'), + 30, '1d', 'close', 'daily') assert_almost_equal( window.loc['2016-01-26', cf], @@ -915,7 +917,9 @@ def record_current_contract(algo, data): # Advance the window a month. window = self.data_portal.get_history_window( - [cf.sid], Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close') + [cf.sid], + Timestamp('2016-04-06', tz='UTC'), + 30, '1d', 'close', 'daily') assert_almost_equal( window.loc['2016-02-24', cf], @@ -946,7 +950,9 @@ def record_current_contract(algo, data): cf = self.data_portal.asset_finder.create_continuous_future( 'MA', 0, 'volume', None) window = self.data_portal.get_history_window( - [cf.sid], Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close') + [cf.sid], + Timestamp('2016-03-06', tz='UTC'), + 30, '1d', 'close', 'daily') assert_almost_equal( window.loc['2016-01-26', cf], @@ -965,7 +971,9 @@ def record_current_contract(algo, data): # Advance the window a month. window = self.data_portal.get_history_window( - [cf.sid], Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close') + [cf.sid], + Timestamp('2016-04-06', tz='UTC'), + 30, '1d', 'close', 'daily') assert_almost_equal( window.loc['2016-02-24', cf], @@ -991,7 +999,8 @@ def record_current_contract(algo, data): 'FO', 0, 'calendar', 'add') window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], - Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close') + Timestamp('2016-03-06', tz='UTC'), + 30, '1d', 'close', 'daily') # Unadjusted value is: 115011.44 # Adjustment is based on hop from 115231.44 to 125231.44 @@ -1034,7 +1043,8 @@ def record_current_contract(algo, data): # Advance the window a month. window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], - Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close') + Timestamp('2016-04-06', tz='UTC'), + 30, '1d', 'close', 'daily') # Unadjusted value: 115221.44 # Adjustments based on hops: @@ -1116,7 +1126,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf.sid], Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') self.assertEqual(window.loc['2016-02-25 22:32', cf], 115231.412, @@ -1135,7 +1145,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf], Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') self.assertEqual(window.loc['2016-02-26 22:32', cf], 125241.412, @@ -1155,7 +1165,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') # Unadjusted: 115231.412 # Adjustment based on roll: @@ -1198,7 +1208,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') # No adjustments in this window. self.assertEqual(window.loc['2016-02-26 22:32', cf_mul], @@ -1219,7 +1229,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') # Unadjusted: 115231.412 # Adjustment based on roll: @@ -1262,7 +1272,7 @@ def record_current_contract(algo, data): window = self.data_portal.get_history_window( [cf, cf_mul, cf_add], Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'), - 30, '1m', 'close') + 30, '1m', 'close', 'minute') # No adjustments in this window. self.assertEqual(window.loc['2016-02-26 22:32', cf_mul], diff --git a/tests/test_history.py b/tests/test_history.py index e1f197ce..cdc36d02 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -654,7 +654,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): self.trading_calendar.open_and_close_for_session(jan5)[1], 2, '1d', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal(np.array([np.nan, 8389]), window1) @@ -665,7 +666,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06 14:35', tz='UTC'), 2, '1d', - 'close' + 'close', + 'minute', )[asset] # Value from 1/5 should be quartered @@ -683,7 +685,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07 14:35', tz='UTC'), 3, '1d', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal( @@ -697,7 +700,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-08 14:40', tz='UTC'), 2, '1d', - 'close' + 'close', + 'minute', )[asset] # should not be adjusted @@ -715,7 +719,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): self.trading_calendar.open_and_close_for_session(jan5)[1], 2, '1d', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal(np.array([nan, 391]), window1) @@ -726,7 +731,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06 14:35', tz='UTC'), 2, '1d', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal( @@ -742,7 +748,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07 14:35', tz='UTC'), 3, '1d', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal( @@ -758,7 +765,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-08 14:40', tz='UTC'), 2, '1d', - 'close' + 'close', + 'minute', )[asset] # should not be adjusted, should be 787 to 791 @@ -977,7 +985,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): equity_cal.open_and_close_for_session(jan5)[1], 10, '1m', - 'close' + 'close', + 'minute', )[asset] np.testing.assert_array_equal( @@ -997,7 +1006,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06 14:35', tz='UTC'), window2_count, '1m', - 'close' + 'close', + 'minute', )[asset] # five minutes from 1/5 should be halved @@ -1034,7 +1044,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07 14:35', tz='UTC'), window3_count, '1m', - 'close' + 'close', + 'minute', )[asset] # first five minutes should be 4385-4390, but eigthed @@ -1065,7 +1076,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07 14:40', tz='UTC'), 5, '1m', - 'close' + 'close', + 'minute', )[asset] # should not be adjusted, should be 1005 to 1009 @@ -1080,7 +1092,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-05 21:00', tz='UTC'), 10, '1m', - 'close' + 'close', + 'minute', )[self.DIVIDEND_ASSET] np.testing.assert_array_equal(np.array(range(382, 392)), window1) @@ -1096,7 +1109,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): window2_end, window2_count, '1m', - 'close' + 'close', + 'minute', )[self.DIVIDEND_ASSET] # first dividend is 2%, so the first five values should be 2% lower @@ -1123,7 +1137,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): window3_end, window3_count, '1m', - 'close' + 'close', + 'minute', )[self.DIVIDEND_ASSET] # first five minute from 1/7 should be hit by 0.9408 (= 0.98 * 0.96) @@ -1278,7 +1293,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): window_end, bar_count, '1m', - 'close' + 'close', + 'minute', )[self.HALF_DAY_TEST_ASSET] # 390 minutes for 7/2, 210 minutes for 7/3, 7/4-7/6 closed @@ -1334,7 +1350,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): first_equity_open, bar_count, '1m', - 'close' + 'close', + 'minute', ) expected = range(asset1_idx - 97, asset1_idx + 3) @@ -1376,7 +1393,12 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): with self.assertRaisesRegexp( HistoryWindowStartsBeforeData, exp_msg): self.data_portal.get_history_window( - [self.ASSET1], first_day_minutes[5], 15, '1m', 'price' + [self.ASSET1], + first_day_minutes[5], + 15, + '1m', + 'price', + 'minute', )[self.ASSET1] def test_daily_history_blended(self): @@ -1405,7 +1427,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): minute, 3, '1d', - field + field, + 'minute', )[self.ASSET2] self.assertEqual(len(window), 3) @@ -1428,10 +1451,6 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): last_val = -1 - # XXX - if minute == day: - continue - if minute < equity_open: # If before the equity calendar open, we don't yet # have data (but price is forward-filled). @@ -1486,7 +1505,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): minute, 3, '1d', - field + field, + 'minute', )[self.ASSET2] self.assertEqual(len(window), 3) @@ -1510,10 +1530,6 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase): self.assertEqual(window[0], 22873500) self.assertEqual(window[1], 38083500) - # XXX - if minute == day: - continue - last_val = -1 if minute < equity_open: @@ -1779,7 +1795,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-05', tz='UTC'), 1, '1d', - 'close' + 'close', + 'daily', )[asset] np.testing.assert_array_equal(window1, [2]) @@ -1789,7 +1806,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-05', tz='UTC'), 1, '1d', - 'volume' + 'volume', + 'daily', )[asset] np.testing.assert_array_equal(window1_volume, [200]) @@ -1800,7 +1818,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06', tz='UTC'), 2, '1d', - 'close' + 'close', + 'daily', )[asset] # first value should be halved, second value unadjusted @@ -1811,7 +1830,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06', tz='UTC'), 2, '1d', - 'volume' + 'volume', + 'daily', )[asset] if asset == self.SPLIT_ASSET: @@ -1826,7 +1846,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07', tz='UTC'), 3, '1d', - 'close' + 'close', + 'daily', )[asset] np.testing.assert_array_equal([0.25, 1.5, 4], window3) @@ -1836,7 +1857,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07', tz='UTC'), 3, '1d', - 'volume' + 'volume', + 'daily', )[asset] if asset == self.SPLIT_ASSET: @@ -1853,7 +1875,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-05', tz='UTC'), 1, '1d', - 'close' + 'close', + 'daily', )[self.DIVIDEND_ASSET] np.testing.assert_array_equal(window1, [2]) @@ -1864,7 +1887,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-06', tz='UTC'), 2, '1d', - 'close' + 'close', + 'daily', )[self.DIVIDEND_ASSET] # first dividend is 2%, so the first value should be 2% lower than @@ -1877,7 +1901,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2015-01-07', tz='UTC'), 3, '1d', - 'close' + 'close', + 'daily', )[self.DIVIDEND_ASSET] # second dividend is 0.96 @@ -1937,7 +1962,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): second_day, 4, '1d', - 'price' + 'price', + 'daily', )[self.ASSET1] with self.assertRaisesRegexp(HistoryWindowStartsBeforeData, exp_msg): @@ -1946,7 +1972,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): second_day, 4, '1d', - 'volume' + 'volume', + 'daily', )[self.ASSET1] # Use a minute to force minute mode. @@ -1959,7 +1986,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): first_minute, 4, '1d', - 'close' + 'close', + 'daily', )[self.ASSET2] def test_history_window_different_order(self): @@ -1976,7 +2004,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): day, 4, "1d", - "close" + "close", + 'daily', ) window_2 = self.data_portal.get_history_window( @@ -1984,7 +2013,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): day, 4, "1d", - "close" + "close", + 'daily', ) np.testing.assert_almost_equal(window_1[self.ASSET1].values, @@ -2004,7 +2034,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2014-02-07', tz='UTC'), 4, "1d", - "close" + "close", + 'daily', ) window_2 = self.data_portal.get_history_window( @@ -2012,7 +2043,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2014-02-05', tz='UTC'), 4, "1d", - "close" + "close", + 'daily', ) window_3 = self.data_portal.get_history_window( @@ -2020,7 +2052,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2014-02-07', tz='UTC'), 4, "1d", - "close" + "close", + 'daily', ) window_4 = self.data_portal.get_history_window( @@ -2028,7 +2061,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase): pd.Timestamp('2014-01-22', tz='UTC'), 4, "1d", - "close" + "close", + 'daily', ) # Calling 02-07 after resetting the window should not affect the diff --git a/zipline/_protocol.pyx b/zipline/_protocol.pyx index e6ee6a33..4b401431 100644 --- a/zipline/_protocol.pyx +++ b/zipline/_protocol.pyx @@ -648,7 +648,8 @@ cdef class BarData: self._get_current_minute(), bar_count, frequency, - fields + fields, + self.data_frequency, ) if self._adjust_minutes: @@ -680,7 +681,8 @@ cdef class BarData: self._get_current_minute(), bar_count, frequency, - field + field, + self.data_frequency, )[assets] for field in fields } @@ -708,7 +710,8 @@ cdef class BarData: self._get_current_minute(), bar_count, frequency, - field + field, + self.data_frequency, ) for field in fields } diff --git a/zipline/algorithm.py b/zipline/algorithm.py index ccbd1fae..5341c87d 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -2160,6 +2160,7 @@ class TradingAlgorithm(object): bar_count, frequency, field, + self.data_frequency, ffill, ) else: @@ -2176,6 +2177,7 @@ class TradingAlgorithm(object): bar_count, frequency, field, + self.data_frequency, ffill, ) diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 997cee49..450f74ba 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -735,8 +735,12 @@ class DataPortal(object): ) return tds[start_loc:end_loc + 1] - def _get_history_daily_window(self, assets, end_dt, bar_count, - field_to_use): + def _get_history_daily_window(self, + assets, + end_dt, + bar_count, + field_to_use, + data_frequency): """ Internal method that returns a dataframe containing history bars of daily frequency for the given sids. @@ -750,7 +754,7 @@ class DataPortal(object): columns=None) data = self._get_history_daily_window_data( - assets, days_for_window, end_dt, field_to_use + assets, days_for_window, end_dt, field_to_use, data_frequency ) return pd.DataFrame( data, @@ -762,10 +766,9 @@ class DataPortal(object): assets, days_for_window, end_dt, - field_to_use): - ends_at_midnight = (end_dt.hour == end_dt.minute == 0) - - if ends_at_midnight: + field_to_use, + data_frequency): + if data_frequency == 'daily': # two cases where we use daily data for the whole range: # 1) the history window ends at midnight utc. # 2) the last desired day of the window is after the @@ -860,7 +863,13 @@ class DataPortal(object): columns=assets ) - def get_history_window(self, assets, end_dt, bar_count, frequency, field, + def get_history_window(self, + assets, + end_dt, + bar_count, + frequency, + field, + data_frequency, ffill=True): """ Public API method that returns a dataframe containing the requested @@ -880,6 +889,10 @@ class DataPortal(object): field: string The desired field of the asset. + data_frequency: string + The frequency of the data to query; i.e. whether the data is + 'daily' or 'minute' bars. + ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. @@ -894,10 +907,10 @@ class DataPortal(object): if frequency == "1d": if field == "price": df = self._get_history_daily_window(assets, end_dt, bar_count, - "close") + "close", data_frequency) else: df = self._get_history_daily_window(assets, end_dt, bar_count, - field) + field, data_frequency) elif frequency == "1m": if field == "price": df = self._get_history_minute_window(assets, end_dt, bar_count, @@ -1299,7 +1312,13 @@ class DataPortal(object): # returns is always calculated over the last 2 days, regardless # of the simulation's data frequency. hst = self.get_history_window( - [asset], dt, 2, "1d", "price", ffill=True + [asset], + dt, + 2, + "1d", + "price", + data_frequency, + ffill=True, )[asset] return (hst.iloc[-1] - hst.iloc[0]) / hst.iloc[0] @@ -1317,7 +1336,13 @@ class DataPortal(object): calculated_bar_count = bars price_arr = self.get_history_window( - [asset], dt, calculated_bar_count, freq_str, "price", ffill=True + [asset], + dt, + calculated_bar_count, + freq_str, + "price", + data_frequency, + ffill=True, )[asset] if transform_name == "mavg": @@ -1326,8 +1351,13 @@ class DataPortal(object): return nanstd(price_arr, ddof=1) elif transform_name == "vwap": volume_arr = self.get_history_window( - [asset], dt, calculated_bar_count, freq_str, "volume", - ffill=True + [asset], + dt, + calculated_bar_count, + freq_str, + "volume", + data_frequency, + ffill=True, )[asset] vol_sum = nansum(volume_arr) diff --git a/zipline/sources/benchmark_source.py b/zipline/sources/benchmark_source.py index b1816bcf..c20892f4 100644 --- a/zipline/sources/benchmark_source.py +++ b/zipline/sources/benchmark_source.py @@ -146,6 +146,7 @@ class BenchmarkSource(object): bar_count=len(minutes) + 1, frequency="1m", field="price", + data_frequency=self.emission_rate, ffill=True )[asset] @@ -163,6 +164,7 @@ class BenchmarkSource(object): bar_count=len(trading_days) + 1, frequency="1d", field="price", + data_frequency=self.emission_rate, ffill=True )[asset] return benchmark_series.pct_change()[1:] @@ -175,6 +177,7 @@ class BenchmarkSource(object): bar_count=len(trading_days), frequency="1d", field="price", + data_frequency=self.emission_rate, ffill=True )[asset] diff --git a/zipline/testing/core.py b/zipline/testing/core.py index 47eee138..34be29a5 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -714,7 +714,7 @@ class FakeDataPortal(DataPortal): return 1.0 def get_history_window(self, assets, end_dt, bar_count, frequency, field, - ffill=True): + data_frequency, ffill=True): if frequency == "1d": end_idx = \ self.trading_calendar.all_sessions.searchsorted(end_dt)