diff --git a/tests/pipeline/test_events.py b/tests/pipeline/test_events.py index 054d3acf..85c85d51 100644 --- a/tests/pipeline/test_events.py +++ b/tests/pipeline/test_events.py @@ -598,10 +598,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase): boundary_dates] moscow_boundary_dates = [date.tz_localize('Europe/Moscow') for date in boundary_dates] - mixed_tz_dates = [pd.Timestamp('2013-01-24'), + mixed_tz_dates = [pd.Timestamp('2013-12-30'), + pd.Timestamp('2013-01-24'), pd.Timestamp('2013-01-31 20:00:00'), pd.Timestamp('2013-04-04'), - pd.Timestamp('2013-04-21')] + pd.Timestamp('2013-04-21'), + pd.Timestamp('2013-06-01')] us_dates = pd.to_datetime(us_boundary_dates + mixed_tz_dates, utc=True).tz_localize(None) moscow_dates = pd.to_datetime(moscow_boundary_dates + mixed_tz_dates, @@ -619,10 +621,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase): [pd.Timestamp('2013-01-04'), pd.Timestamp('2013-01-05'), pd.Timestamp('2013-01-05'), + pd.Timestamp('2013-12-30'), pd.Timestamp('2013-01-24'), pd.Timestamp('2013-02-01'), pd.Timestamp('2013-04-04'), - pd.Timestamp('2013-04-21')] + pd.Timestamp('2013-04-21'), + pd.Timestamp('2013-06-01')] ).values # Russia's TZ offset is +4 @@ -630,10 +634,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase): [pd.Timestamp('2013-01-04'), pd.Timestamp('2013-01-05'), pd.Timestamp('2013-01-05'), + pd.Timestamp('2013-12-30'), pd.Timestamp('2013-01-24'), pd.Timestamp('2013-01-31'), pd.Timestamp('2013-04-04'), - pd.Timestamp('2013-04-21')] + pd.Timestamp('2013-04-21'), + pd.Timestamp('2013-06-01')] ).values # Test with timezones on either side of the meridian @@ -652,7 +658,4 @@ class EventLoaderUtilsTestCase(ZiplineTestCase): ts_field='timestamp') timestamps = result['timestamp'].values - check_arrays( - timestamps, - expected[scrambler] - ) + check_arrays(np.sort(timestamps), np.sort(expected[scrambler])) diff --git a/zipline/pipeline/loaders/utils.py b/zipline/pipeline/loaders/utils.py index f5385bf3..3033cfdc 100644 --- a/zipline/pipeline/loaders/utils.py +++ b/zipline/pipeline/loaders/utils.py @@ -231,6 +231,12 @@ def normalize_timestamp_to_query_time(df, # don't mutate the dataframe in place df = df.copy() + # There is a pandas bug (0.18.1) where if the timestamps in a + # normalized DatetimeIndex are not sorted and one calls `tz_localize(None)` + # on tha DatetimeIndex, some of the dates will be shifted by an hour + # (similarly to the previously mentioned bug). Therefore, we must sort + # the df here to ensure that we get the normalize correctly. + df.sort_values(ts_field, inplace=True) dtidx = pd.DatetimeIndex(df.loc[:, ts_field], tz='utc') dtidx_local_time = dtidx.tz_convert(tz) to_roll_forward = mask_between_time( diff --git a/zipline/testing/core.py b/zipline/testing/core.py index df628888..abfe110b 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -406,7 +406,7 @@ def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True): ) # Fill NaTs with zero for comparison. x = np.where(x_isnat, np.zeros_like(x), x) - y = np.where(x_isnat, np.zeros_like(x), x) + y = np.where(y_isnat, np.zeros_like(y), y) return assert_array_equal(x, y, err_msg=err_msg, verbose=verbose)