MAINT: Pass data_frequency to get_history_window

This allows us to remove the check for whether the provided dt had a
time of midnight, which was a flimsy way to infer if the data frequency
was 'daily'. Besides the explicit check being preferable, this method
was broken on the futures calendar, since midnight is a valid market
minute.

tests/test_api_shim.py

@@ -265,6 +265,7 @@ class TestAPIShim(WithCreateBarData,
                         5,
                         "1m",
                         "volume",
+                        "minute",
                         True
                     )
                 else:
@@ -274,6 +275,7 @@ class TestAPIShim(WithCreateBarData,
                         5,
                         "1m",
                         "volume",
+                        "minute",
                     )
 
         test_sim_params = SimulationParameters(

tests/test_benchmark.py

@@ -101,7 +101,12 @@ class TestBenchmark(WithDataPortal, WithSimParams, WithTradingCalendars,
         # should be the equivalent of getting the price history, then doing
         # a pct_change on it
         manually_calculated = self.data_portal.get_history_window(
-            [1], days_to_use[-1], len(days_to_use), "1d", "close"
+            [1],
+            days_to_use[-1],
+            len(days_to_use),
+            "1d",
+            "close",
+            "daily",
         )[1].pct_change()
 
         # compare all the fields except the first one, for which we don't have
@@ -187,6 +192,7 @@ class TestBenchmark(WithDataPortal, WithSimParams, WithTradingCalendars,
                 len(days_to_use),
                 "1d",
                 "close",
+                "daily",
             )[2].pct_change()
 
             for idx, day in enumerate(days_to_use[1:]):

tests/test_continuous_futures.py

@@ -670,7 +670,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-01-26', cf],
                          0,
@@ -696,7 +696,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-02-25', cf],
                          1,
@@ -724,7 +724,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-01-11 18:01', tz='US/Eastern').tz_convert('UTC'),
-            3, '1d', 'sid')
+            3, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-01-08', cf],
                          10,
@@ -745,7 +745,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-01-26', cf],
                          1,
@@ -771,7 +771,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-02-25', cf],
                          2,
@@ -799,7 +799,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-03-04 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         # Volume cuts out for FOF16 on 2016-01-25
         self.assertEqual(window.loc['2016-01-26', cf],
@@ -826,7 +826,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-04-06 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1d', 'sid')
+            30, '1d', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-02-25', cf],
                          1,
@@ -863,7 +863,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf.sid],
             Timestamp('2016-01-26 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'sid')
+            30, '1m', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-01-26 22:32', cf],
                          0,
@@ -882,7 +882,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-01-27 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'sid')
+            30, '1m', 'sid', 'minute')
 
         self.assertEqual(window.loc['2016-01-27 22:32', cf],
                          1,
@@ -896,7 +896,9 @@ def record_current_contract(algo, data):
         cf = self.data_portal.asset_finder.create_continuous_future(
             'FO', 0, 'calendar', None)
         window = self.data_portal.get_history_window(
-            [cf.sid], Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close')
+            [cf.sid],
+            Timestamp('2016-03-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         assert_almost_equal(
             window.loc['2016-01-26', cf],
@@ -915,7 +917,9 @@ def record_current_contract(algo, data):
 
         # Advance the window a month.
         window = self.data_portal.get_history_window(
-            [cf.sid], Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close')
+            [cf.sid],
+            Timestamp('2016-04-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         assert_almost_equal(
             window.loc['2016-02-24', cf],
@@ -946,7 +950,9 @@ def record_current_contract(algo, data):
         cf = self.data_portal.asset_finder.create_continuous_future(
             'MA', 0, 'volume', None)
         window = self.data_portal.get_history_window(
-            [cf.sid], Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close')
+            [cf.sid],
+            Timestamp('2016-03-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         assert_almost_equal(
             window.loc['2016-01-26', cf],
@@ -965,7 +971,9 @@ def record_current_contract(algo, data):
 
         # Advance the window a month.
         window = self.data_portal.get_history_window(
-            [cf.sid], Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close')
+            [cf.sid],
+            Timestamp('2016-04-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         assert_almost_equal(
             window.loc['2016-02-24', cf],
@@ -991,7 +999,8 @@ def record_current_contract(algo, data):
             'FO', 0, 'calendar', 'add')
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
-            Timestamp('2016-03-06', tz='UTC'), 30, '1d', 'close')
+            Timestamp('2016-03-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         # Unadjusted value is: 115011.44
         # Adjustment is based on hop from 115231.44 to 125231.44
@@ -1034,7 +1043,8 @@ def record_current_contract(algo, data):
         # Advance the window a month.
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
-            Timestamp('2016-04-06', tz='UTC'), 30, '1d', 'close')
+            Timestamp('2016-04-06', tz='UTC'),
+            30, '1d', 'close', 'daily')
 
         # Unadjusted value: 115221.44
         # Adjustments based on hops:
@@ -1116,7 +1126,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf.sid],
             Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         self.assertEqual(window.loc['2016-02-25 22:32', cf],
                          115231.412,
@@ -1135,7 +1145,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf],
             Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         self.assertEqual(window.loc['2016-02-26 22:32', cf],
                          125241.412,
@@ -1155,7 +1165,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
             Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         # Unadjusted: 115231.412
         # Adjustment based on roll:
@@ -1198,7 +1208,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
             Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         # No adjustments in this window.
         self.assertEqual(window.loc['2016-02-26 22:32', cf_mul],
@@ -1219,7 +1229,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
             Timestamp('2016-02-25 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         # Unadjusted: 115231.412
         # Adjustment based on roll:
@@ -1262,7 +1272,7 @@ def record_current_contract(algo, data):
         window = self.data_portal.get_history_window(
             [cf, cf_mul, cf_add],
             Timestamp('2016-02-28 18:01', tz='US/Eastern').tz_convert('UTC'),
-            30, '1m', 'close')
+            30, '1m', 'close', 'minute')
 
         # No adjustments in this window.
         self.assertEqual(window.loc['2016-02-26 22:32', cf_mul],

tests/test_history.py

@@ -654,7 +654,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 self.trading_calendar.open_and_close_for_session(jan5)[1],
                 2,
                 '1d',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             np.testing.assert_array_equal(np.array([np.nan, 8389]), window1)
@@ -665,7 +666,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-06 14:35', tz='UTC'),
                 2,
                 '1d',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             # Value from 1/5 should be quartered
@@ -683,7 +685,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-07 14:35', tz='UTC'),
                 3,
                 '1d',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             np.testing.assert_array_equal(
@@ -697,7 +700,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-08 14:40', tz='UTC'),
                 2,
                 '1d',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             # should not be adjusted
@@ -715,7 +719,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             self.trading_calendar.open_and_close_for_session(jan5)[1],
             2,
             '1d',
-            'close'
+            'close',
+            'minute',
         )[asset]
 
         np.testing.assert_array_equal(np.array([nan, 391]), window1)
@@ -726,7 +731,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-06 14:35', tz='UTC'),
             2,
             '1d',
-            'close'
+            'close',
+            'minute',
         )[asset]
 
         np.testing.assert_array_equal(
@@ -742,7 +748,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-07 14:35', tz='UTC'),
             3,
             '1d',
-            'close'
+            'close',
+            'minute',
         )[asset]
 
         np.testing.assert_array_equal(
@@ -758,7 +765,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-08 14:40', tz='UTC'),
             2,
             '1d',
-            'close'
+            'close',
+            'minute',
         )[asset]
 
         # should not be adjusted, should be 787 to 791
@@ -977,7 +985,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 equity_cal.open_and_close_for_session(jan5)[1],
                 10,
                 '1m',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             np.testing.assert_array_equal(
@@ -997,7 +1006,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-06 14:35', tz='UTC'),
                 window2_count,
                 '1m',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             # five minutes from 1/5 should be halved
@@ -1034,7 +1044,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-07 14:35', tz='UTC'),
                 window3_count,
                 '1m',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             # first five minutes should be 4385-4390, but eigthed
@@ -1065,7 +1076,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-07 14:40', tz='UTC'),
                 5,
                 '1m',
-                'close'
+                'close',
+                'minute',
             )[asset]
 
             # should not be adjusted, should be 1005 to 1009
@@ -1080,7 +1092,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-05 21:00', tz='UTC'),
             10,
             '1m',
-            'close'
+            'close',
+            'minute',
         )[self.DIVIDEND_ASSET]
 
         np.testing.assert_array_equal(np.array(range(382, 392)), window1)
@@ -1096,7 +1109,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             window2_end,
             window2_count,
             '1m',
-            'close'
+            'close',
+            'minute',
         )[self.DIVIDEND_ASSET]
 
         # first dividend is 2%, so the first five values should be 2% lower
@@ -1123,7 +1137,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             window3_end,
             window3_count,
             '1m',
-            'close'
+            'close',
+            'minute',
         )[self.DIVIDEND_ASSET]
 
         # first five minute from 1/7 should be hit by 0.9408 (= 0.98 * 0.96)
@@ -1278,7 +1293,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             window_end,
             bar_count,
             '1m',
-            'close'
+            'close',
+            'minute',
         )[self.HALF_DAY_TEST_ASSET]
 
         # 390 minutes for 7/2, 210 minutes for 7/3, 7/4-7/6 closed
@@ -1334,7 +1350,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             first_equity_open,
             bar_count,
             '1m',
-            'close'
+            'close',
+            'minute',
         )
 
         expected = range(asset1_idx - 97, asset1_idx + 3)
@@ -1376,7 +1393,12 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             with self.assertRaisesRegexp(
                     HistoryWindowStartsBeforeData, exp_msg):
                 self.data_portal.get_history_window(
-                    [self.ASSET1], first_day_minutes[5], 15, '1m', 'price'
+                    [self.ASSET1],
+                    first_day_minutes[5],
+                    15,
+                    '1m',
+                    'price',
+                    'minute',
                 )[self.ASSET1]
 
     def test_daily_history_blended(self):
@@ -1405,7 +1427,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                     minute,
                     3,
                     '1d',
-                    field
+                    field,
+                    'minute',
                 )[self.ASSET2]
 
                 self.assertEqual(len(window), 3)
@@ -1428,10 +1451,6 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
 
                 last_val = -1
 
-                # XXX
-                if minute == day:
-                    continue
-
                 if minute < equity_open:
                     # If before the equity calendar open, we don't yet
                     # have data (but price is forward-filled).
@@ -1486,7 +1505,8 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 minute,
                 3,
                 '1d',
-                field
+                field,
+                'minute',
             )[self.ASSET2]
 
             self.assertEqual(len(window), 3)
@@ -1510,10 +1530,6 @@ class MinuteEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 self.assertEqual(window[0], 22873500)
                 self.assertEqual(window[1], 38083500)
 
-            # XXX
-            if minute == day:
-                continue
-
             last_val = -1
 
             if minute < equity_open:
@@ -1779,7 +1795,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-05', tz='UTC'),
                 1,
                 '1d',
-                'close'
+                'close',
+                'daily',
             )[asset]
 
             np.testing.assert_array_equal(window1, [2])
@@ -1789,7 +1806,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-05', tz='UTC'),
                 1,
                 '1d',
-                'volume'
+                'volume',
+                'daily',
             )[asset]
 
             np.testing.assert_array_equal(window1_volume, [200])
@@ -1800,7 +1818,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-06', tz='UTC'),
                 2,
                 '1d',
-                'close'
+                'close',
+                'daily',
             )[asset]
 
             # first value should be halved, second value unadjusted
@@ -1811,7 +1830,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-06', tz='UTC'),
                 2,
                 '1d',
-                'volume'
+                'volume',
+                'daily',
             )[asset]
 
             if asset == self.SPLIT_ASSET:
@@ -1826,7 +1846,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-07', tz='UTC'),
                 3,
                 '1d',
-                'close'
+                'close',
+                'daily',
             )[asset]
 
             np.testing.assert_array_equal([0.25, 1.5, 4], window3)
@@ -1836,7 +1857,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 pd.Timestamp('2015-01-07', tz='UTC'),
                 3,
                 '1d',
-                'volume'
+                'volume',
+                'daily',
             )[asset]
 
             if asset == self.SPLIT_ASSET:
@@ -1853,7 +1875,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-05', tz='UTC'),
             1,
             '1d',
-            'close'
+            'close',
+            'daily',
         )[self.DIVIDEND_ASSET]
 
         np.testing.assert_array_equal(window1, [2])
@@ -1864,7 +1887,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-06', tz='UTC'),
             2,
             '1d',
-            'close'
+            'close',
+            'daily',
         )[self.DIVIDEND_ASSET]
 
         # first dividend is 2%, so the first value should be 2% lower than
@@ -1877,7 +1901,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2015-01-07', tz='UTC'),
             3,
             '1d',
-            'close'
+            'close',
+            'daily',
         )[self.DIVIDEND_ASSET]
 
         # second dividend is 0.96
@@ -1937,7 +1962,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 second_day,
                 4,
                 '1d',
-                'price'
+                'price',
+                'daily',
             )[self.ASSET1]
 
         with self.assertRaisesRegexp(HistoryWindowStartsBeforeData, exp_msg):
@@ -1946,7 +1972,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 second_day,
                 4,
                 '1d',
-                'volume'
+                'volume',
+                'daily',
             )[self.ASSET1]
 
         # Use a minute to force minute mode.
@@ -1959,7 +1986,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
                 first_minute,
                 4,
                 '1d',
-                'close'
+                'close',
+                'daily',
             )[self.ASSET2]
 
     def test_history_window_different_order(self):
@@ -1976,7 +2004,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             day,
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         window_2 = self.data_portal.get_history_window(
@@ -1984,7 +2013,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             day,
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         np.testing.assert_almost_equal(window_1[self.ASSET1].values,
@@ -2004,7 +2034,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2014-02-07', tz='UTC'),
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         window_2 = self.data_portal.get_history_window(
@@ -2012,7 +2043,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2014-02-05', tz='UTC'),
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         window_3 = self.data_portal.get_history_window(
@@ -2020,7 +2052,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2014-02-07', tz='UTC'),
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         window_4 = self.data_portal.get_history_window(
@@ -2028,7 +2061,8 @@ class DailyEquityHistoryTestCase(WithHistory, ZiplineTestCase):
             pd.Timestamp('2014-01-22', tz='UTC'),
             4,
             "1d",
-            "close"
+            "close",
+            'daily',
         )
 
         # Calling 02-07 after resetting the window should not affect the