diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index f80fbf4a..bc896313 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -433,7 +433,7 @@ cdef class TradingPair(Asset): 'taker', 'trading_state', 'data_source', - 'decimals' + 'decimals', }) def __init__(self, object symbol, @@ -455,7 +455,7 @@ cdef class TradingPair(Asset): float taker=0.0025, float lot=0, int decimals = 8, - int trading_state=0, + int trading_state=1, object data_source='catalyst'): """ Replicates the Asset constructor with some built-in conventions @@ -600,14 +600,51 @@ cdef class TradingPair(Asset): cpdef to_dict(self): """ Convert to a python dict. + + Repeat constructor params: + object symbol, + object exchange, + object start_date=None, + object asset_name=None, + int sid=0, + float leverage=1.0, + object end_daily=None, + object end_minute=None, + object end_date=None, + object exchange_symbol=None, + object first_traded=None, + object auto_close_date=None, + object exchange_full=None, + float min_trade_size=0.0001, + float max_trade_size=1000000, + float maker=0.0015, + float taker=0.0025, + float lot=0, + int decimals = 8, + int trading_state=1, + object data_source='catalyst', """ - #TODO: missing fields - super_dict = super(TradingPair, self).to_dict() - super_dict['end_daily'] = self.end_daily - super_dict['end_minute'] = self.end_minute - super_dict['leverage'] = self.leverage - super_dict['min_trade_size'] = self.min_trade_size - return super_dict + trading_pair_dict = dict( + symbol=self.symbol, + exchange=self.exchange, + start_date=self.start_date, + asset_name=self.asset_name, + leverage=self.leverage, + end_daily=self.end_daily, + end_minute=self.end_minute, + end_date=self.end_date, + exchange_symbol=self.exchange_symbol, + exchange_full=self.exchange_full, + min_trade_size=self.min_trade_size, + max_trade_size=self.max_trade_size, + maker=self.maker, + taker=self.taker, + lot=self.lot, + decimals=self.decimals, + trading_state=self.trading_state, + data_source=self.data_source, + ) + return trading_pair_dict def is_exchange_open(self, dt_minute): """ @@ -623,6 +660,13 @@ cdef class TradingPair(Asset): #TODO: make more dymanic to catch holds return True + def set_end_date(self, dt, data_frequency): + if data_frequency == 'minute': + self.end_minute = dt + + else: + self.end_daily = dt + cpdef __reduce__(self): """ Function used by pickle to determine how to serialize/deserialize this @@ -646,7 +690,9 @@ cdef class TradingPair(Asset): self.lot, self.decimals, self.taker, - self.maker)) + self.maker, + self.trading_state, + self.data_source)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..375dbe21 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' - +EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json' DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 3e5d5b29..9a5dad60 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,5 +1,3 @@ -import json -import os import re from collections import defaultdict @@ -20,6 +18,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ UnsupportedHistoryFrequencyError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ + MarketsNotFoundError, InvalidMarketError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ get_exchange_folder, get_catalyst_symbol, \ @@ -27,8 +27,16 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ get_epoch, \ get_periods_range +from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \ + get_epoch, get_catalyst_symbol, \ + get_exchange_auth, get_exchange_config from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction +from ccxt import InvalidOrder, NetworkError, \ + ExchangeError +from logbook import Logger +from redo import retry +from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -62,6 +70,8 @@ class CCXT(Exchange): 'secret': secret, }) self.api.enableRateLimit = True + self.has = self.api.has + self.fees = self.api.fees except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) @@ -69,6 +79,7 @@ class CCXT(Exchange): self._symbol_maps = [None, None] self.name = exchange_name + self.assets = [] self.base_currency = base_currency self.transactions = defaultdict(list) @@ -80,50 +91,115 @@ class CCXT(Exchange): self._common_symbols = dict() self.bundle = ExchangeBundle(self.name) - self.markets = None self._is_init = False + self._config = None def init(self): if self._is_init: return - exchange_folder = get_exchange_folder(self.name) - filename = os.path.join(exchange_folder, 'cctx_markets.json') - - if os.path.exists(filename): - timestamp = os.path.getmtime(filename) - dt = pd.to_datetime(timestamp, unit='s', utc=True) - - if dt >= pd.Timestamp.utcnow().floor('1D'): - with open(filename) as f: - self.markets = json.load(f) - - log.debug('loaded markets for {}'.format(self.name)) - - if self.markets is None: - try: - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) + if self._config is None: + self._config = get_exchange_config(self.name) + log.debug( + 'got exchange config {}:\n{}'.format( + self.name, self._config ) - - self.markets = self.api.fetch_markets() - with open(filename, 'w+') as f: - json.dump(self.markets, f, indent=4) - - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch markets {}: {}'.format( - self.name, e - ) - ) - raise ExchangeRequestError(error=e) + ) self.load_assets() self._is_init = True + def load_assets(self): + if self._config is None: + raise ValueError('Exchange config not available.') + + self.assets = [] + for asset_dict in self._config['assets']: + asset = TradingPair(**asset_dict) + self.assets.append(asset) + + def _fetch_markets(self): + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + try: + markets = self.api.fetch_markets() + + except NetworkError as e: + raise ExchangeRequestError(error=e) + + if not markets: + raise MarketsNotFoundError( + exchange=self.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=self.name, + market=market, + ) + return markets + + def create_exchange_config(self): + config = dict( + name=self.name, + features=[feature for feature in self.has if self.has[feature]] + ) + markets = retry( + action=self._fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(self.name) + ), + ) + + config['assets'] = [] + for market in markets: + asset = self.create_trading_pair(market=market) + config['assets'].append(asset) + + return config + + def create_trading_pair(self, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=self.name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + self.apply_conditional_market_params(params, market) + + return TradingPair(**params) + @staticmethod def find_exchanges(features=None, is_authenticated=False): ccxt_features = [] @@ -202,42 +278,6 @@ class CCXT(Exchange): return frequencies - def get_market(self, symbol): - """ - The CCXT market. - - Parameters - ---------- - symbol: - The CCXT symbol. - - Returns - ------- - dict[str, Object] - - """ - s = self.get_symbol(symbol) - market = next( - (market for market in self.markets if market['symbol'] == s), - None, - ) - return market - - def substitute_currency_code(self, currency, source='catalyst'): - if source == 'catalyst': - currency = currency.upper() - - key = self.api.common_currency_code(currency) - self._common_symbols[key] = currency.lower() - return key - - else: - if currency in self._common_symbols: - return self._common_symbols[currency] - - else: - return currency.lower() - def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -485,144 +525,53 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def get_asset_defs(self, market): + def apply_conditional_market_params(self, params, market): """ - The local and Catalyst definitions of the specified market. + Applies a CCXT market dict to parameters of TradingPair init. Parameters ---------- - market: dict[str, Object] - The CCXT market dicts. + params: dict[Object] + market: dict[Object] Returns ------- - dict[str, Object] - The asset definition. """ - asset_defs = [] - - for is_local in (False, True): - asset_def = self.get_asset_def(market, is_local) - asset_defs.append((asset_def, is_local)) - - return asset_defs - - def get_asset_def(self, market, is_local=False): - """ - The asset definition (in symbols.json files) corresponding - to the the specified market. - - Parameters - ---------- - market: dict[str, Object] - The CCXT market dict. - is_local - Whether to search in local or Catalyst asset definitions. - - Returns - ------- - dict[str, Object] - The asset definition. - - """ - exchange_symbol = market['id'] - - symbol_map = self._fetch_symbol_map(is_local) - if symbol_map is not None: - assets_lower = {k.lower(): v for k, v in symbol_map.items()} - key = exchange_symbol.lower() - - asset = assets_lower[key] if key in assets_lower else None - if asset is not None: - return asset - - else: - return None + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 else: - return None + params['trading_state'] = 1 - def create_trading_pair(self, market, asset_def=None, is_local=False): - """ - Creating a TradingPair from market and asset data. + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] - Parameters - ---------- - market: dict[str, Object] - asset_def: dict[str, Object] - is_local: bool + if self.name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 - Returns - ------- - - """ - data_source = 'local' if is_local else 'catalyst' - params = dict( - exchange=self.name, - data_source=data_source, - exchange_symbol=market['id'], - ) - mixin_market_params(self.name, params, market) - - if asset_def is not None: - params['symbol'] = asset_def['symbol'] - - params['start_date'] = asset_def['start_date'] \ - if 'start_date' in asset_def else None - - params['end_date'] = asset_def['end_date'] \ - if 'end_date' in asset_def else None - - params['leverage'] = asset_def['leverage'] \ - if 'leverage' in asset_def else 1.0 - - params['asset_name'] = asset_def['asset_name'] \ - if 'asset_name' in asset_def else None - - params['end_daily'] = asset_def['end_daily'] \ - if 'end_daily' in asset_def \ - and asset_def['end_daily'] != 'N/A' else None - - params['end_minute'] = asset_def['end_minute'] \ - if 'end_minute' in asset_def \ - and asset_def['end_minute'] != 'N/A' else None + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] else: - params['symbol'] = get_catalyst_symbol(market) - # TODO: add as an optional column - params['leverage'] = 1.0 + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 - return TradingPair(**params) + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) - def load_assets(self): - log.debug('loading assets for {}'.format(self.name)) - self.assets = [] - - for market in self.markets: - if 'id' not in market: - log.warn('invalid market: {}'.format(market)) - continue - - asset_defs = self.get_asset_defs(market) - - asset = None - for asset_def in asset_defs: - if asset_def[0] is not None or not asset_defs[1]: - try: - asset = self.create_trading_pair( - market=market, - asset_def=asset_def[0], - is_local=asset_def[1] - ) - self.assets.append(asset) - - except TypeError as e: - log.warn('unable to add asset: {}'.format(e)) - - if asset is None: - asset = self.create_trading_pair(market=market) - self.assets.append(asset) + if 'lot' not in params: + params['lot'] = params['min_trade_size'] def get_balances(self): try: @@ -760,18 +709,14 @@ class CCXT(Exchange): side = 'buy' if amount > 0 else 'sell' if hasattr(self.api, 'amount_to_lots'): - # TODO: is this right? - if self.api.markets is None: - self.api.load_markets() - - # https://github.com/ccxt/ccxt/issues/1483 - adj_amount = abs(amount) - market = self.api.markets[symbol] - if 'lots' in market and market['lots'] > amount: - raise CreateOrderError( - exchange=self.name, - e='order amount lower than the smallest lot: {}'.format( - amount + adj_amount = self.api.amount_to_lots( + symbol=symbol, + amount=abs(amount), + ) + if adj_amount != abs(amount): + log.info( + 'adjusted order amount {} to {} based on lot size'.format( + abs(amount), adj_amount, ) ) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 38aed7c7..91ac07a3 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -5,6 +5,7 @@ from functools import partial from itertools import chain from operator import is_not +import copy import numpy as np import pandas as pd import pytz @@ -25,7 +26,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \ get_period_label, get_month_start_end, get_year_start_end from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params, get_catalyst_symbol + get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory from logbook import Logger @@ -700,42 +701,36 @@ class ExchangeBundle: for symbol in symbols: start_dt = df.index.get_level_values(1).min() end_dt = df.index.get_level_values(1).max() - end_dt_key = 'end_{}'.format(data_frequency) - market = self.exchange.get_market(symbol) - if market is None: - raise ValueError('symbol not available in the exchange.') + try: + asset = self.exchange.get_asset(symbol, is_local=True) + except: + asset = copy.deepcopy(self.exchange.get_asset(symbol)) - params = dict( - exchange=self.exchange.name, - data_source='local', - exchange_symbol=market['id'], - ) - mixin_market_params(self.exchange_name, params, market) + if asset.data_source == 'local': + asset.start_date = asset.start_date \ + if asset.start_date < start_dt else start_dt - asset_def = self.exchange.get_asset_def(market, True) - if asset_def is not None: - params['symbol'] = asset_def['symbol'] + if data_frequency == 'daily': + asset.end_date = asset.end_daily = asset.end_daily \ + if asset.end_daily > end_dt else end_dt - params['start_date'] = asset_def['start_date'] \ - if asset_def['start_date'] < start_dt else start_dt - - params['end_date'] = asset_def[end_dt_key] \ - if asset_def[end_dt_key] > end_dt else end_dt - - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else asset_def['end_daily'] - - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else asset_def['end_minute'] + else: + asset.end_date = asset.end_minute = asset.end_minute \ + if asset.end_minute > end_dt else end_dt else: - params['symbol'] = get_catalyst_symbol(market) + asset.data_source = 'local' + asset.start_date = start_dt + asset.end_dt = end_dt - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else 'N/A' - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else 'N/A' + if data_frequency == 'daily': + asset.end_daily = end_dt + asset.end_minute = None + + else: + asset.end_daily = None + asset.end_minute = end_dt if min_start_dt is None or start_dt < min_start_dt: min_start_dt = start_dt @@ -743,11 +738,9 @@ class ExchangeBundle: if max_end_dt is None or end_dt > max_end_dt: max_end_dt = end_dt - asset = TradingPair(**params) - assets[market['id']] = asset - - save_exchange_symbols(self.exchange_name, assets, True) + assets[symbol] = asset + # TODO: update config.json writer = self.get_writer( start_dt=min_start_dt.replace(hour=00, minute=00), end_dt=max_end_dt.replace(hour=23, minute=59), diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index d5af87c4..91fba0da 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -322,3 +322,17 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() + + +class MarketsNotFoundError(ZiplineError): + msg = ( + 'Exchange {exchange} contains no valid market so it is unusable in ' + 'Catalyst.' + ).strip() + + +class InvalidMarketError(ZiplineError): + msg = ( + 'Exchange {exchange} contains at least one incorrectly structured ' + 'market: {market}, so it is unusable in Catalyst.' + ).strip() diff --git a/tests/exchange/test_config.py b/tests/exchange/test_config.py new file mode 100644 index 00000000..f13f526b --- /dev/null +++ b/tests/exchange/test_config.py @@ -0,0 +1,8 @@ +from catalyst.exchange.utils.factory import get_exchange + + +class TestConfig: + def test_create_config(self): + exchange = get_exchange('binance', skip_init=True) + config = exchange.create_exchange_config() + pass