Merge pull request #1739 from quantopian/fix-zipline-and-pandas-bug

Fix zipline and pandas bug

tests/pipeline/test_events.py

@@ -598,10 +598,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase):
                          boundary_dates]
     moscow_boundary_dates = [date.tz_localize('Europe/Moscow') for date in
                              boundary_dates]
-    mixed_tz_dates = [pd.Timestamp('2013-01-24'),
+    mixed_tz_dates = [pd.Timestamp('2013-12-30'),
+                      pd.Timestamp('2013-01-24'),
                       pd.Timestamp('2013-01-31 20:00:00'),
                       pd.Timestamp('2013-04-04'),
-                      pd.Timestamp('2013-04-21')]
+                      pd.Timestamp('2013-04-21'),
+                      pd.Timestamp('2013-06-01')]
     us_dates = pd.to_datetime(us_boundary_dates + mixed_tz_dates,
                               utc=True).tz_localize(None)
     moscow_dates = pd.to_datetime(moscow_boundary_dates + mixed_tz_dates,
@@ -619,10 +621,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase):
         [pd.Timestamp('2013-01-04'),
          pd.Timestamp('2013-01-05'),
          pd.Timestamp('2013-01-05'),
+         pd.Timestamp('2013-12-30'),
          pd.Timestamp('2013-01-24'),
          pd.Timestamp('2013-02-01'),
          pd.Timestamp('2013-04-04'),
-         pd.Timestamp('2013-04-21')]
+         pd.Timestamp('2013-04-21'),
+         pd.Timestamp('2013-06-01')]
     ).values
 
     # Russia's TZ offset is +4
@@ -630,10 +634,12 @@ class EventLoaderUtilsTestCase(ZiplineTestCase):
         [pd.Timestamp('2013-01-04'),
          pd.Timestamp('2013-01-05'),
          pd.Timestamp('2013-01-05'),
+         pd.Timestamp('2013-12-30'),
          pd.Timestamp('2013-01-24'),
          pd.Timestamp('2013-01-31'),
          pd.Timestamp('2013-04-04'),
-         pd.Timestamp('2013-04-21')]
+         pd.Timestamp('2013-04-21'),
+         pd.Timestamp('2013-06-01')]
     ).values
 
     # Test with timezones on either side of the meridian
@@ -652,7 +658,4 @@ class EventLoaderUtilsTestCase(ZiplineTestCase):
                                                        ts_field='timestamp')
 
             timestamps = result['timestamp'].values
-            check_arrays(
-                timestamps,
-                expected[scrambler]
-            )
+            check_arrays(np.sort(timestamps), np.sort(expected[scrambler]))

zipline/pipeline/loaders/utils.py

@@ -231,6 +231,12 @@ def normalize_timestamp_to_query_time(df,
         # don't mutate the dataframe in place
         df = df.copy()
 
+    # There is a pandas bug (0.18.1) where if the timestamps in a
+    # normalized DatetimeIndex are not sorted and one calls `tz_localize(None)`
+    #  on tha DatetimeIndex, some of the dates will be shifted by an hour
+    # (similarly to the previously mentioned bug). Therefore, we must sort
+    # the df here to ensure that we get the normalize correctly.
+    df.sort_values(ts_field, inplace=True)
     dtidx = pd.DatetimeIndex(df.loc[:, ts_field], tz='utc')
     dtidx_local_time = dtidx.tz_convert(tz)
     to_roll_forward = mask_between_time(

zipline/testing/core.py

@@ -406,7 +406,7 @@ def check_arrays(x, y, err_msg='', verbose=True, check_dtypes=True):
         )
         # Fill NaTs with zero for comparison.
         x = np.where(x_isnat, np.zeros_like(x), x)
-        y = np.where(x_isnat, np.zeros_like(x), x)
+        y = np.where(y_isnat, np.zeros_like(y), y)
 
     return assert_array_equal(x, y, err_msg=err_msg, verbose=verbose)