BLD: fix 'on the clock' candles fetch and request extra candles using a fixed time interval

catalyst/exchange/exchange.py

@@ -16,7 +16,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     TickerNotFoundError, NotEnoughCashError
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
-    get_periods, get_start_dt, get_frequency
+    get_periods, get_start_dt, get_frequency, \
+    get_candles_number_from_minutes
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
     resample_history_df, has_bundle, get_candles_df
 from logbook import Logger
@@ -511,7 +512,12 @@ class Exchange:
         # so we request more than needed
         # TODO: consider defining a const per asset
         # and/or some retry mechanism (in each iteration request more data)
-        requested_bar_count = bar_count + 30
+        kExtra_minutes_candles = 150
+        requested_bar_count = bar_count + \
+            get_candles_number_from_minutes(unit,
+                                            candle_size,
+                                            kExtra_minutes_candles)
+
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
@@ -529,11 +535,14 @@ class Exchange:
                     asset=asset,
                     exchange=self.name)
 
+        # for avoiding unnecessary forward fill end_dt is taken back one second
+        forward_fill_till_dt = end_dt - timedelta(seconds=1)
+
         series = get_candles_df(candles=candles,
                                 field=field,
                                 freq=frequency,
                                 bar_count=requested_bar_count,
-                                end_dt=end_dt)
+                                end_dt=forward_fill_till_dt)
 
         # TODO: consider how to approach this edge case
         # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size

catalyst/exchange/utils/datetime_utils.py

@@ -1,4 +1,5 @@
 import calendar
+import math
 import re
 from datetime import datetime, timedelta, date
 
@@ -326,3 +327,33 @@ def from_ms_timestamp(ms):
 
 def get_epoch():
     return pd.to_datetime('1970-1-1', utc=True)
+
+
+def get_candles_number_from_minutes(unit, candle_size, minutes):
+    """
+    Get the number of bars needed for the given time interval
+    in minutes.
+
+    Notes
+    -----
+    Supports only "T", "D" and "H" units
+
+    Parameters
+    ----------
+    unit: str
+    candle_size : int
+    minutes: int
+
+    Returns
+    -------
+    int
+
+    """
+    if unit == "T":
+        res = (float(minutes) / candle_size)
+    elif unit == "H":
+        res = (minutes / 60.0) / candle_size
+    else:  # unit == "D"
+        res = (minutes / 1440.0) / candle_size
+
+    return int(math.ceil(res))