BLD: unit tests for historical data

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -17,7 +17,7 @@ from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
-    ExchangeNotFoundError, CreateOrderError
+    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.exchange_utils import mixin_market_params, \
     from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol
@@ -148,6 +148,23 @@ class CCXT(Exchange):
     def time_skew(self):
         return None
 
+    def get_candle_frequencies(self):
+        frequencies = []
+        try:
+            for timeframe in self.api.timeframes:
+                frequencies.append(
+                    CCXT.get_frequency(timeframe, raise_error=False)
+                )
+
+        except Exception:
+            log.warn(
+                'candle frequencies not available for exchange {}'.format(
+                    self.name
+                )
+            )
+
+        return frequencies
+
     def get_market(self, symbol):
         """
         The CCXT market.
@@ -188,7 +205,8 @@ class CCXT(Exchange):
         parts = symbol.split('_')
         return '{}/{}'.format(parts[0].upper(), parts[1].upper())
 
-    def get_timeframe(self, freq):
+    @staticmethod
+    def get_timeframe(freq, raise_error=True):
         """
         The CCXT timeframe from the Catalyst frequency.
 
@@ -221,8 +239,56 @@ class CCXT(Exchange):
         elif unit.lower() == 'h' or unit == 'T':
             timeframe = '{}h'.format(candle_size)
 
+        elif raise_error:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
         return timeframe
 
+    @staticmethod
+    def get_frequency(timeframe, raise_error=True):
+        """
+        Test Catalyst frequency from the CCXT timeframe
+
+        Catalyst uses the Pandas offset alias convention:
+        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+
+        Parameters
+        ----------
+        timeframe
+
+        Returns
+        -------
+
+        """
+        timeframe_match = re.match(
+            r'([0-9].*)?(m|M|d|h|w|y)', timeframe, re.M | re.I
+        )
+        if timeframe_match:
+            candle_size = int(timeframe_match.group(1)) \
+                if timeframe_match.group(1) else 1
+
+            unit = timeframe_match.group(2)
+
+        else:
+            raise InvalidHistoryTimeframeError(timeframe=timeframe)
+
+        if unit.lower() == 'd':
+            freq = '{}D'.format(candle_size)
+
+        elif unit.lower() == 'm':
+            freq = '{}T'.format(candle_size)
+
+        elif unit.lower() == 'h':
+            freq = '{}H'.format(candle_size)
+
+        elif unit.lower() == 'w':
+            freq = '{}D'.format(candle_size * 7)
+
+        elif raise_error:
+            raise InvalidHistoryTimeframeError(timeframe=timeframe)
+
+        return freq
+
     def get_candles(self, freq, assets, bar_count=None, start_dt=None,
                     end_dt=None):
         is_single = (isinstance(assets, TradingPair))
@@ -230,7 +296,7 @@ class CCXT(Exchange):
             assets = [assets]
 
         symbols = self.get_symbols(assets)
-        timeframe = self.get_timeframe(freq)
+        timeframe = CCXT.get_timeframe(freq)
 
         ms = None
         if start_dt is not None:

catalyst/exchange/exchange_errors.py

@@ -33,22 +33,22 @@ class ExchangeRequestErrorTooManyAttempts(ZiplineError):
 
 class ExchangeBarDataError(ZiplineError):
     msg = (
-        'Unable to retrieve bar data: {data_type}, ' +
-        'giving up after {attempts} attempts: {error}'
+            'Unable to retrieve bar data: {data_type}, ' +
+            'giving up after {attempts} attempts: {error}'
     ).strip()
 
 
 class ExchangePortfolioDataError(ZiplineError):
     msg = (
-        'Unable to retrieve portfolio data: {data_type}, ' +
-        'giving up after {attempts} attempts: {error}'
+            'Unable to retrieve portfolio data: {data_type}, ' +
+            'giving up after {attempts} attempts: {error}'
     ).strip()
 
 
 class ExchangeTransactionError(ZiplineError):
     msg = (
-        'Unable to execute transaction: {transaction_type}, ' +
-        'giving up after {attempts} attempts: {error}'
+            'Unable to execute transaction: {transaction_type}, ' +
+            'giving up after {attempts} attempts: {error}'
     ).strip()
 
 
@@ -100,6 +100,12 @@ class InvalidHistoryFrequencyError(ZiplineError):
     ).strip()
 
 
+class InvalidHistoryTimeframeError(ZiplineError):
+    msg = (
+        'CCXT timeframe {timeframe} not supported by the exchange.'
+    ).strip()
+
+
 class MismatchingFrequencyError(ZiplineError):
     msg = (
         'Bar aggregate frequency {frequency} not compatible with '
@@ -264,6 +270,7 @@ class NotEnoughCapitalError(ZiplineError):
         'as the specified `capital_base`. The current balance {balance} is '
         'lower than the `capital_base`: {capital_base}').strip()
 
+
 class LastCandleTooEarlyError(ZiplineError):
     msg = (
         'The trade date of the last candle {last_traded} is before the '

tests/exchange/test_suite_exchange.py

@@ -11,7 +11,6 @@ from ccxt import AuthenticationError
 from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.exchange.factory import find_exchanges
-from catalyst.utils.paths import data_root
 
 log = Logger('TestSuiteExchange')
 
@@ -128,7 +127,8 @@ class TestSuiteExchange(unittest.TestCase):
         asset_population = 3
 
         exchanges = select_random_exchanges(
-            exchange_population
+            exchange_population,
+            features=['fetchTickers'],
         )  # Type: list[Exchange]
         for exchange in exchanges:
             exchange.init()
@@ -149,6 +149,44 @@ class TestSuiteExchange(unittest.TestCase):
         pass
 
     def test_candles(self):
+        exchange_population = 3
+        asset_population = 3
+
+        exchanges = select_random_exchanges(
+            exchange_population,
+            features=['fetchOHLCV'],
+        )  # Type: list[Exchange]
+        for exchange in exchanges:
+            exchange.init()
+
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                frequencies = exchange.get_candle_frequencies()
+                freq = random.sample(frequencies, 1)[0]
+
+                bar_count = random.randint(1, 10)
+                end_dt = pd.Timestamp.utcnow().floor('1T')
+                dt_range = pd.date_range(
+                    end=end_dt, periods=bar_count, freq=freq
+                )
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+
+                candles = exchange.get_candles(
+                    freq=freq,
+                    assets=assets,
+                    bar_count=bar_count,
+                    start_dt=dt_range[0],
+                    end_dt=dt_range[-1],
+                )
+
+                assert len(candles) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
         pass
 
     def test_orders(self):