BLD: cleanup and minor adjustments to get_candles() in live mode

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -24,7 +24,8 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
     get_exchange_folder, get_catalyst_symbol, \
     get_exchange_auth
-from exchange.utils.datetime_utils import from_ms_timestamp, get_epoch, \
+from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
+    get_epoch, \
     get_periods_range
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
@@ -424,27 +425,36 @@ class CCXT(Exchange):
             )
 
         elif end_dt is not None:
-            dt_range = get_periods_range(
-                end_dt=end_dt,
-                periods=bar_count,
-                freq=freq,
+            # Make sure that end_dt really wants data in the past
+            # if it's close to now, we skip the 'since' parameters to
+            # lower the probability of error
+            bars_to_now = pd.date_range(
+                end_dt, pd.Timestamp.utcnow(), freq=freq
             )
-            # skip the left bound of the range since the open range is
-            # on the right bound
-            start_dt = dt_range[1]
+            if len(bars_to_now) > 1:
+                dt_range = get_periods_range(
+                    end_dt=end_dt,
+                    periods=bar_count,
+                    freq=freq,
+                )
+                # with some exchanges, skip the left bound of the range
+                # since the open range is on the right bound
+                if self.name in ['poloniex']:
+                    start_dt = dt_range[1]
+                else:
+                    start_dt = dt_range[0]
 
-        ms = None
+        since = None
         if start_dt is not None:
-            if end_dt is not None:
-                delta = start_dt - get_epoch()
-            ms = int(delta.total_seconds()) * 1000
+            delta = start_dt - get_epoch()
+            since = int(delta.total_seconds()) * 1000
 
         candles = dict()
         for index, asset in enumerate(assets):
             ohlcvs = self.api.fetch_ohlcv(
                 symbol=symbols[index],
                 timeframe=timeframe,
-                since=ms,
+                since=since,
                 limit=bar_count,
                 params={}
             )

catalyst/exchange/exchange.py

@@ -13,7 +13,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     PricingDataNotLoadedError, \
     NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
     TickerNotFoundError, NotEnoughCashError
-from exchange.utils.datetime_utils import get_delta, get_periods_range, \
+from catalyst.exchange.utils.datetime_utils import get_delta, \
+    get_periods_range, \
     get_periods, get_start_dt, get_frequency
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
     resample_history_df, has_bundle
@@ -497,39 +498,37 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
-        adj_bar_count = candle_size * bar_count
-
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
             bar_count=bar_count,
-            start_dt=start_dt if not is_current else None,
             end_dt=end_dt if not is_current else None,
         )
 
         series = dict()
         for asset in candles:
+            first_candle = candles[asset][0]
             asset_series = self.get_series_from_candles(
                 candles=candles[asset],
-                start_dt=start_dt,
+                start_dt=first_candle['last_traded'],
                 end_dt=end_dt,
                 data_frequency=frequency,
                 field=field,
             )
-            if end_dt is not None:
-                delta = get_delta(candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_traded = asset_series.index[-1]
 
-                if last_traded < adj_end_dt:
-                    raise LastCandleTooEarlyError(
-                        last_traded=last_traded,
-                        end_dt=adj_end_dt,
-                        exchange=self.name,
-                    )
+            # Checking to make sure that the dates match
+            delta = get_delta(candle_size, data_frequency)
+            adj_end_dt = end_dt - delta
+            last_traded = asset_series.index[-1]
+
+            if last_traded < adj_end_dt:
+                raise LastCandleTooEarlyError(
+                    last_traded=last_traded,
+                    end_dt=adj_end_dt,
+                    exchange=self.name,
+                )
+
             series[asset] = asset_series
 
         df = pd.DataFrame(series)

catalyst/exchange/exchange_bundle.py

@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_df_from_arrays, get_assets
-from exchange.utils.datetime_utils import get_delta, get_start_dt, \
+from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
     get_period_label, get_month_start_end, get_year_start_end
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol

catalyst/exchange/exchange_data_portal.py

@@ -10,7 +10,7 @@ from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     PricingDataNotLoadedError)
 from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
-from exchange.utils.datetime_utils import get_frequency
+from catalyst.exchange.utils.datetime_utils import get_frequency
 from logbook import Logger
 from redo import retry
 

catalyst/exchange/utils/datetime_utils.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta, date
 import pandas as pd
 import pytz
 
-from exchange.exchange_errors import InvalidHistoryFrequencyError, \
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     InvalidHistoryFrequencyAlias
 
 

tests/exchange/test_ccxt.py

@@ -15,7 +15,7 @@ log = Logger('test_ccxt')
 class TestCCXT(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        exchange_name = 'binance'
+        exchange_name = 'bitfinex'
         auth = get_exchange_auth(exchange_name)
         self.exchange = CCXT(
             exchange_name=exchange_name,
@@ -58,7 +58,7 @@ class TestCCXT(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         candles = self.exchange.get_candles(
-            freq='5T',
+            freq='30T',
             assets=[self.exchange.get_asset('eth_btc')],
             bar_count=200,
             start_dt=pd.to_datetime('2017-09-01', utc=True)

tests/exchange/test_suites/test_suite_bundle.py

@@ -85,6 +85,8 @@ class TestSuiteBundle:
                     df, assets, '{}_{}'.format(freq, source)
                 )
 
+            print('saved {} test results: {}'.format(end_dt, folder))
+
             assert_frame_equal(
                 right=data['bundle'],
                 left=data['exchange'],
@@ -102,7 +104,6 @@ class TestSuiteBundle:
                 with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
                     handle.write(e.args[0])
 
-            print('saved test results: {}'.format(folder))
             pass
 
     def test_validate_bundles(self):
@@ -116,7 +117,7 @@ class TestSuiteBundle:
         #     population=exchange_population,
         #     features=[bundle],
         # )  # Type: list[Exchange]
-        exchanges = [get_exchange('poloniex', skip_init=True)]
+        exchanges = [get_exchange('bitfinex', skip_init=True)]
 
         data_portal = TestSuiteBundle.get_data_portal(exchanges)
         for exchange in exchanges: