BUG: fixed a standardization issue with historical data in live mode

catalyst/examples/simple_loop.py

@@ -23,7 +23,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='5T'
+        frequency='30T'
     )
     last_traded = prices.index[-1]
     print('last candle date: {}'.format(last_traded))
@@ -127,7 +127,7 @@ run_algorithm(
     initialize=initialize,
     handle_data=handle_data,
     analyze=None,
-    exchange_name='binance',
+    exchange_name='gdax',
     live=True,
     algo_namespace='simple_loop',
     base_currency='eth',

catalyst/exchange/exchange.py

@@ -409,7 +409,7 @@ class Exchange:
             method='ffill',
             fill_value=previous_value,
         )
-
+        series.sort_index(inplace=True)
         return series
 
     def get_history_window(self,
@@ -419,7 +419,7 @@ class Exchange:
                            frequency,
                            field,
                            data_frequency=None,
-                           ffill=True):
+                           is_current=False):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -446,10 +446,9 @@ class Exchange:
             The frequency of the data to query; i.e. whether the data is
             'daily' or 'minute' bars.
 
-        # TODO: fill how?
-        ffill: boolean
-            Forward-fill missing values. Only has effect if field
-            is 'price'.
+        is_current: bool
+            Skip date filters when current data is requested (last few bars
+            until now).
 
         Notes
         -----
@@ -475,24 +474,12 @@ class Exchange:
             freq=freq,
             assets=assets,
             bar_count=bar_count,
-            start_dt=start_dt,
-            end_dt=end_dt,
+            start_dt=start_dt if not is_current else None,
+            end_dt=end_dt if not is_current else None,
         )
 
         series = dict()
         for asset in candles:
-            if end_dt is not None and candles[asset]:
-                delta = get_delta(candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_candle = candles[asset][-1]
-
-                if last_candle['last_traded'] < adj_end_dt:
-                    raise LastCandleTooEarlyError(
-                        last_traded=last_candle['last_traded'],
-                        end_dt=adj_end_dt,
-                        exchange=self.name,
-                    )
-
             asset_series = self.get_series_from_candles(
                 candles=candles[asset],
                 start_dt=start_dt,
@@ -500,6 +487,17 @@ class Exchange:
                 data_frequency=frequency,
                 field=field,
             )
+            if end_dt is not None:
+                delta = get_delta(candle_size, data_frequency)
+                adj_end_dt = end_dt - delta
+                last_traded = asset_series.index[-1]
+
+                if last_traded < adj_end_dt:
+                    raise LastCandleTooEarlyError(
+                        last_traded=last_traded,
+                        end_dt=adj_end_dt,
+                        exchange=self.name,
+                    )
             series[asset] = asset_series
 
         df = pd.DataFrame(series)

catalyst/exchange/exchange_data_portal.py

@@ -238,6 +238,12 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
         """
         exchange = self.exchanges[exchange_name]
 
+        if end_dt >= pd.Timestamp.utcnow().floor('1T'):
+            is_current = True
+
+        else:
+            is_current = False
+
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -245,7 +251,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
             frequency,
             field,
             data_frequency,
-            ffill)
+            is_current)
         return df
 
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,