BLD: added CCXT market data cache

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -1,7 +1,9 @@
+import json
 import re
 from collections import defaultdict
 
 import ccxt
+import os
 import pandas as pd
 import six
 from ccxt import ExchangeNotAvailable, InvalidOrder
@@ -18,7 +20,7 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeNotFoundError, CreateOrderError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.exchange_utils import mixin_market_params, \
-    from_ms_timestamp, get_epoch
+    from_ms_timestamp, get_epoch, get_exchange_folder
 from catalyst.finance.order import Order, ORDER_STATUS
 
 log = Logger('CCXT', level=LOG_LEVEL)
@@ -58,18 +60,8 @@ class CCXT(Exchange):
 
         self._symbol_maps = [None, None]
 
-        try:
-            markets_symbols = self.api.load_markets()
-            log.debug('the markets:\n{}'.format(markets_symbols))
-
-        except ExchangeNotAvailable as e:
-            raise ExchangeRequestError(error=e)
-
         self.name = exchange_name
 
-        self.markets = self.api.fetch_markets()
-        self.load_assets()
-
         self.base_currency = base_currency
         self.transactions = defaultdict(list)
 
@@ -78,6 +70,72 @@ class CCXT(Exchange):
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self.name)
+        self.markets = None
+
+    def init(self):
+        exchange_folder = get_exchange_folder(self.name)
+        filename = os.path.join(exchange_folder, 'cctx_markets.json')
+
+        if os.path.exists(filename):
+            timestamp = os.path.getmtime(filename)
+            dt = pd.to_datetime(timestamp, unit='s', utc=True)
+
+            if dt >= pd.Timestamp.utcnow().floor('1D'):
+                with open(filename) as f:
+                    self.markets = json.load(f)
+
+                log.debug('loaded markets for {}'.format(self.name))
+
+        if self.markets is None:
+            try:
+                markets_symbols = self.api.load_markets()
+                log.debug(
+                    'fetching {} markets:\n{}'.format(
+                        self.name, markets_symbols
+                    )
+                )
+
+                self.markets = self.api.fetch_markets()
+                with open(filename, 'w+') as f:
+                    json.dump(self.markets, f)
+
+            except ExchangeNotAvailable as e:
+                raise ExchangeRequestError(error=e)
+
+        self.load_assets()
+
+    @staticmethod
+    def find_exchanges(features=None):
+        exchange_names = []
+        for exchange_name in ccxt.exchanges:
+            log.debug('loading exchange: {}'.format(exchange_name))
+            exchange = getattr(ccxt, exchange_name)()
+
+            if features is None:
+                has_feature = True
+
+            else:
+                try:
+                    has_feature = all(
+                        [exchange.has[feature] for feature in features]
+                    )
+
+                except Exception:
+                    has_feature = False
+
+            if has_feature:
+                try:
+                    log.info('initializing {}'.format(exchange_name))
+                    exchange_names.append(exchange_name)
+
+                except Exception as e:
+                    log.warn(
+                        'unable to initialize exchange {}: {}'.format(
+                            exchange_name, e
+                        )
+                    )
+
+        return exchange_names
 
     def account(self):
         return None
@@ -346,6 +404,7 @@ class CCXT(Exchange):
         return TradingPair(**params)
 
     def load_assets(self):
+        log.debug('loading assets for {}'.format(self.name))
         self.assets = []
 
         for market in self.markets:

catalyst/exchange/exchange.py

@@ -3,6 +3,7 @@ from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
 
+import ccxt
 import numpy as np
 import pandas as pd
 from logbook import Logger
@@ -247,8 +248,16 @@ class Exchange:
             # The symbol provided may use the Catalyst or the exchange
             # convention
             key = a.exchange_symbol if is_exchange_symbol else a.symbol
-            if not asset and key.lower() == symbol.lower() and applies:
-                asset = a
+            if not asset and key.lower() == symbol.lower():
+                if applies:
+                    asset = a
+
+                else:
+                    raise NoDataAvailableOnExchange(
+                        symbol=key,
+                        exchange=self.name,
+                        data_frequency=data_frequency,
+                    )
 
         if asset is None:
             supported_symbols = sorted([a.symbol for a in self.assets])

catalyst/exchange/factory.py

@@ -1,12 +1,20 @@
 import os
 
+import ccxt
+from logbook import Logger
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
 from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
     get_exchange_folder
 
+log = Logger('factory', level=LOG_LEVEL)
 
-def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
+                 skip_init=False):
     exchange_auth = get_exchange_auth(exchange_name)
 
     has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
@@ -18,13 +26,18 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
             )
         )
 
-    return CCXT(
+    exchange = CCXT(
         exchange_name=exchange_name,
         key=exchange_auth['key'],
         secret=exchange_auth['secret'],
         base_currency=base_currency,
     )
 
+    if not skip_init:
+        exchange.init()
+
+    return exchange
+
 
 def get_exchanges(exchange_names):
     exchanges = dict()
@@ -32,3 +45,26 @@ def get_exchanges(exchange_names):
         exchanges[exchange_name] = get_exchange(exchange_name)
 
     return exchanges
+
+
+def find_exchanges(features=None):
+    """
+    Find exchanges filtered by a list of feature.
+
+    Parameters
+    ----------
+    features: str
+        The list of features.
+
+    Returns
+    -------
+    list[Exchange]
+
+    """
+    exchange_names = CCXT.find_exchanges(features)
+
+    exchanges = []
+    for exchange_name in exchange_names:
+        exchanges.append(get_exchange(exchange_name, skip_init=True))
+
+    return exchanges