diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 6af426e1..66944b88 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,7 +1,9 @@ +import json import re from collections import defaultdict import ccxt +import os import pandas as pd import six from ccxt import ExchangeNotAvailable, InvalidOrder @@ -18,7 +20,7 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeNotFoundError, CreateOrderError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.exchange_utils import mixin_market_params, \ - from_ms_timestamp, get_epoch + from_ms_timestamp, get_epoch, get_exchange_folder from catalyst.finance.order import Order, ORDER_STATUS log = Logger('CCXT', level=LOG_LEVEL) @@ -58,18 +60,8 @@ class CCXT(Exchange): self._symbol_maps = [None, None] - try: - markets_symbols = self.api.load_markets() - log.debug('the markets:\n{}'.format(markets_symbols)) - - except ExchangeNotAvailable as e: - raise ExchangeRequestError(error=e) - self.name = exchange_name - self.markets = self.api.fetch_markets() - self.load_assets() - self.base_currency = base_currency self.transactions = defaultdict(list) @@ -78,6 +70,72 @@ class CCXT(Exchange): self.request_cpt = dict() self.bundle = ExchangeBundle(self.name) + self.markets = None + + def init(self): + exchange_folder = get_exchange_folder(self.name) + filename = os.path.join(exchange_folder, 'cctx_markets.json') + + if os.path.exists(filename): + timestamp = os.path.getmtime(filename) + dt = pd.to_datetime(timestamp, unit='s', utc=True) + + if dt >= pd.Timestamp.utcnow().floor('1D'): + with open(filename) as f: + self.markets = json.load(f) + + log.debug('loaded markets for {}'.format(self.name)) + + if self.markets is None: + try: + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + + self.markets = self.api.fetch_markets() + with open(filename, 'w+') as f: + json.dump(self.markets, f) + + except ExchangeNotAvailable as e: + raise ExchangeRequestError(error=e) + + self.load_assets() + + @staticmethod + def find_exchanges(features=None): + exchange_names = [] + for exchange_name in ccxt.exchanges: + log.debug('loading exchange: {}'.format(exchange_name)) + exchange = getattr(ccxt, exchange_name)() + + if features is None: + has_feature = True + + else: + try: + has_feature = all( + [exchange.has[feature] for feature in features] + ) + + except Exception: + has_feature = False + + if has_feature: + try: + log.info('initializing {}'.format(exchange_name)) + exchange_names.append(exchange_name) + + except Exception as e: + log.warn( + 'unable to initialize exchange {}: {}'.format( + exchange_name, e + ) + ) + + return exchange_names def account(self): return None @@ -346,6 +404,7 @@ class CCXT(Exchange): return TradingPair(**params) def load_assets(self): + log.debug('loading assets for {}'.format(self.name)) self.assets = [] for market in self.markets: diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index cb76b885..87185fd1 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -3,6 +3,7 @@ from abc import ABCMeta, abstractmethod, abstractproperty from datetime import timedelta from time import sleep +import ccxt import numpy as np import pandas as pd from logbook import Logger @@ -247,8 +248,16 @@ class Exchange: # The symbol provided may use the Catalyst or the exchange # convention key = a.exchange_symbol if is_exchange_symbol else a.symbol - if not asset and key.lower() == symbol.lower() and applies: - asset = a + if not asset and key.lower() == symbol.lower(): + if applies: + asset = a + + else: + raise NoDataAvailableOnExchange( + symbol=key, + exchange=self.name, + data_frequency=data_frequency, + ) if asset is None: supported_symbols = sorted([a.symbol for a in self.assets]) diff --git a/catalyst/exchange/factory.py b/catalyst/exchange/factory.py index b0002f32..299f46f5 100644 --- a/catalyst/exchange/factory.py +++ b/catalyst/exchange/factory.py @@ -1,12 +1,20 @@ import os +import ccxt +from logbook import Logger + +from catalyst.constants import LOG_LEVEL +from catalyst.exchange.exchange import Exchange from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange_errors import ExchangeAuthEmpty from catalyst.exchange.exchange_utils import get_exchange_auth, \ get_exchange_folder +log = Logger('factory', level=LOG_LEVEL) -def get_exchange(exchange_name, base_currency=None, must_authenticate=False): + +def get_exchange(exchange_name, base_currency=None, must_authenticate=False, + skip_init=False): exchange_auth = get_exchange_auth(exchange_name) has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') @@ -18,13 +26,18 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False): ) ) - return CCXT( + exchange = CCXT( exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, ) + if not skip_init: + exchange.init() + + return exchange + def get_exchanges(exchange_names): exchanges = dict() @@ -32,3 +45,26 @@ def get_exchanges(exchange_names): exchanges[exchange_name] = get_exchange(exchange_name) return exchanges + + +def find_exchanges(features=None): + """ + Find exchanges filtered by a list of feature. + + Parameters + ---------- + features: str + The list of features. + + Returns + ------- + list[Exchange] + + """ + exchange_names = CCXT.find_exchanges(features) + + exchanges = [] + for exchange_name in exchange_names: + exchanges.append(get_exchange(exchange_name, skip_init=True)) + + return exchanges