diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 498db034..7e3a1fbc 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -20,7 +20,7 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeNotFoundError, CreateOrderError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.exchange_utils import mixin_market_params, \ - from_ms_timestamp, get_epoch, get_exchange_folder + from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol from catalyst.finance.order import Order, ORDER_STATUS log = Logger('CCXT', level=LOG_LEVEL) @@ -188,28 +188,6 @@ class CCXT(Exchange): parts = symbol.split('_') return '{}/{}'.format(parts[0].upper(), parts[1].upper()) - def get_catalyst_symbol(self, market_or_symbol): - """ - The Catalyst symbol. - - Parameters - ---------- - market_or_symbol - - Returns - ------- - - """ - if isinstance(market_or_symbol, string_types): - parts = market_or_symbol.split('/') - return '{}_{}'.format(parts[0].lower(), parts[1].lower()) - - else: - return '{}_{}'.format( - market_or_symbol['base'].lower(), - market_or_symbol['quote'].lower(), - ) - def get_timeframe(self, freq): """ The CCXT timeframe from the Catalyst frequency. @@ -402,7 +380,7 @@ class CCXT(Exchange): and asset_def['end_minute'] != 'N/A' else None else: - params['symbol'] = self.get_catalyst_symbol(market) + params['symbol'] = get_catalyst_symbol(market) # TODO: add as an optional column params['leverage'] = 1.0 @@ -656,30 +634,46 @@ class CCXT(Exchange): """ tickers = dict() - for asset in assets: - try: - ccxt_symbol = self.get_symbol(asset) - ticker = self.api.fetch_ticker(ccxt_symbol) + try: + symbols = [self.get_symbol(asset) for asset in assets] + ccxt_tickers = self.api.fetch_tickers(symbols) + for asset in assets: + symbol = self.get_symbol(asset) + if symbol not in ccxt_tickers: + log.warn('ticker not found for {} {}'.format( + self.name, symbol + )) + continue + + ticker = ccxt_tickers[symbol] ticker['last_traded'] = from_ms_timestamp(ticker['timestamp']) if 'last_price' not in ticker: # TODO: any more exceptions? ticker['last_price'] = ticker['last'] - # Using the volume represented in the base currency - ticker['volume'] = ticker['baseVolume'] \ - if 'baseVolume' in ticker else 0 + if 'baseVolume' in ticker and ticker['baseVolume'] is not None: + # Using the volume represented in the base currency + ticker['volume'] = ticker['baseVolume'] + + elif 'info' in ticker and 'bidQty' in ticker['info'] \ + and 'askQty' in ticker['info']: + ticker['volume'] = float(ticker['info']['bidQty']) + \ + float(ticker['info']['askQty']) + + else: + ticker['volume'] = 0 tickers[asset] = ticker - except ExchangeNotAvailable as e: - log.warn( - 'unable to fetch ticker: {} {}'.format( - self.name, asset.symbol - ) + except ExchangeNotAvailable as e: + log.warn( + 'unable to fetch ticker: {} {}'.format( + self.name, asset.symbol ) - raise ExchangeRequestError(error=e) + ) + raise ExchangeRequestError(error=e) return tickers diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 46c0c9e7..fae2a066 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -29,7 +29,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ NoDataAvailableOnExchange, \ PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError from catalyst.exchange.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params + save_exchange_symbols, mixin_market_params, get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory @@ -730,7 +730,7 @@ class ExchangeBundle: if data_frequency == 'minute' else asset_def['end_minute'] else: - params['symbol'] = self.exchange.get_catalyst_symbol(market) + params['symbol'] = get_catalyst_symbol(market) params['end_daily'] = end_dt \ if data_frequency == 'daily' else 'N/A' diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 19938d3e..f67ea9a4 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -62,6 +62,13 @@ def get_exchange_folder(exchange_name, environ=None): return exchange_folder +def is_blacklist(exchange_name, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, 'blacklist.txt') + + return os.path.exists(filename) + + def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -133,8 +140,8 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): filename = get_exchange_symbols_filename(exchange_name, is_local) if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): + pd.Timestamp('now', tz='UTC') - last_modified_time( + filename)).days > 1): download_exchange_symbols(exchange_name, environ) if os.path.isfile(filename): @@ -646,3 +653,25 @@ def group_assets_by_exchange(assets): exchange_assets[asset.exchange].append(asset) return exchange_assets + +def get_catalyst_symbol(market_or_symbol): + """ + The Catalyst symbol. + + Parameters + ---------- + market_or_symbol + + Returns + ------- + + """ + if isinstance(market_or_symbol, string_types): + parts = market_or_symbol.split('/') + return '{}_{}'.format(parts[0].lower(), parts[1].lower()) + + else: + return '{}_{}'.format( + market_or_symbol['base'].lower(), + market_or_symbol['quote'].lower(), + ) diff --git a/catalyst/exchange/factory.py b/catalyst/exchange/factory.py index 299f46f5..4451aff0 100644 --- a/catalyst/exchange/factory.py +++ b/catalyst/exchange/factory.py @@ -8,7 +8,7 @@ from catalyst.exchange.exchange import Exchange from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange_errors import ExchangeAuthEmpty from catalyst.exchange.exchange_utils import get_exchange_auth, \ - get_exchange_folder + get_exchange_folder, is_blacklist log = Logger('factory', level=LOG_LEVEL) @@ -47,7 +47,7 @@ def get_exchanges(exchange_names): return exchanges -def find_exchanges(features=None): +def find_exchanges(features=None, skip_blacklist=True): """ Find exchanges filtered by a list of feature. @@ -65,6 +65,9 @@ def find_exchanges(features=None): exchanges = [] for exchange_name in exchange_names: + if skip_blacklist and is_blacklist(exchange_name): + continue + exchanges.append(get_exchange(exchange_name, skip_init=True)) return exchanges diff --git a/tests/exchange/test_suite_exchange.py b/tests/exchange/test_suite_exchange.py index 87f8cb73..9b4d952c 100644 --- a/tests/exchange/test_suite_exchange.py +++ b/tests/exchange/test_suite_exchange.py @@ -30,24 +30,17 @@ def handle_exchange_error(exchange, e): is_blacklist = True if is_blacklist: - root = data_root() - filename = os.path.join(root, 'exchanges', 'blacklist.json') + try: + message = '{}: {}'.format( + e.__class__, e.message.decode('ascii', 'ignore') + ) + except Exception: + message = 'unexpected error' - if os.path.isfile(filename): - with open(filename) as handle: - try: - bl_data = json.load(handle) - - except ValueError: - bl_data = dict() - - else: - bl_data = dict() - - if exchange.name not in bl_data: - bl_data[exchange.name] = '{}: {}'.format(e.__class__, e.message) - with open(filename, 'wt') as handle: - json.dump(bl_data, handle, indent=4) + folder = get_exchange_folder(exchange.name) + filename = os.path.join(folder, 'blacklist.txt') + with open(filename, 'wt') as handle: + handle.write(message) def select_random_exchanges(population=3, features=None): @@ -62,8 +55,7 @@ def select_random_exchanges(population=3, features=None): return exchanges -def select_random_assets(exchange, population=3): - all_assets = exchange.assets +def select_random_assets(all_assets, population=3): assets = random.sample(all_assets, population) return assets @@ -93,6 +85,11 @@ class TestSuiteExchange(unittest.TestCase): handle_exchange_error(exchange, e) else: + print( + 're-trying an exchange request {} {}'.format( + exchange.name, attempts + ) + ) self._test_markets_exchange(exchange, attempts + 1) except Exception as e: @@ -101,17 +98,18 @@ class TestSuiteExchange(unittest.TestCase): return assets def test_markets(self): - population = None + population = 3 results = dict() exchanges = select_random_exchanges(population) # Type: list[Exchange] for exchange in exchanges: assets = self._test_markets_exchange(exchange) + if assets is not None: results[exchange.name] = len(assets) folder = get_exchange_folder(exchange.name) - filename = os.path.join(folder, 'supported_assets.json') + filename = os.path.join(folder, 'whitelist.json') symbols = [asset.symbol for asset in assets] with open(filename, 'wt') as handle: @@ -125,13 +123,29 @@ class TestSuiteExchange(unittest.TestCase): pass - def test_ticker(self): - exchanges = select_random_exchanges(3) # Type: list[Exchange] + def test_tickers(self): + exchange_population = 3 + asset_population = 3 + + exchanges = select_random_exchanges( + exchange_population + ) # Type: list[Exchange] for exchange in exchanges: exchange.init() - assets = select_random_assets(exchange, 3) - exchange.tickers() + if exchange.assets and len(exchange.assets) >= asset_population: + assets = select_random_assets( + exchange.assets, asset_population + ) + tickers = exchange.tickers(assets) + + assert len(tickers) == asset_population + + else: + print( + 'skipping exchange without assets {}'.format(exchange.name) + ) + exchange_population -= 1 pass def test_candles(self):