diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 38b2ae32..e3a8a5f4 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -12,7 +12,8 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ UnsupportedHistoryFrequencyError, \ - ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ + MarketsNotFoundError, InvalidMarketError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.ccxt_utils import get_exchange_config from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ @@ -23,6 +24,7 @@ from catalyst.finance.transaction import Transaction from ccxt import InvalidOrder, NetworkError, \ ExchangeError from logbook import Logger +from redo import retry from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -107,6 +109,97 @@ class CCXT(Exchange): asset = TradingPair(**asset_dict) self.assets.append(asset) + def _fetch_markets(self): + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + try: + markets = self.api.fetch_markets() + + except NetworkError as e: + raise ExchangeRequestError(error=e) + + if not markets: + raise MarketsNotFoundError( + exchange=self.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=self.name, + market=market, + ) + return markets + + def create_exchange_config(self): + config = dict( + name=self.name, + features=[feature for feature in self.has if self.has[feature]] + ) + markets = retry( + action=self._fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(self.name) + ), + ) + + config['assets'] = [] + for market in markets: + asset = self.create_trading_pair(market=market) + config['assets'].append(asset) + + return config + + def create_trading_pair(self, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=self.name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + self.apply_conditional_market_params(params, market) + + return TradingPair(**params) + + def load_assets(self): + if self._config is None: + raise ValueError('Exchange config not available.') + + self.assets = [] + for asset_dict in self._config['assets']: + asset = TradingPair(**asset_dict) + self.assets.append(asset) + def account(self): return None @@ -365,6 +458,54 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None + def apply_conditional_market_params(self, params, market): + """ + Applies a CCXT market dict to parameters of TradingPair init. + + Parameters + ---------- + params: dict[Object] + market: dict[Object] + + Returns + ------- + + """ + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 + + else: + params['trading_state'] = 1 + + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] + + if self.name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 + + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] + + else: + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 + + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) + + if 'lot' not in params: + params['lot'] = params['min_trade_size'] + def get_balances(self): try: log.debug('retrieving wallets balances') diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c1b64f7c..84d4b596 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -16,7 +16,7 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ from catalyst.exchange.utils.datetime_utils import get_delta, \ get_periods_range, \ get_periods, get_start_dt, get_frequency -from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ +from catalyst.exchange.utils.exchange_utils import \ resample_history_df, has_bundle from logbook import Logger @@ -290,16 +290,6 @@ class Exchange: log.debug('found asset: {}'.format(asset)) return asset - def fetch_symbol_map(self, is_local=False): - index = 1 if is_local else 0 - if self._symbol_maps[index] is not None: - return self._symbol_maps[index] - - else: - symbol_map = get_exchange_symbols(self.name, is_local) - self._symbol_maps[index] = symbol_map - return symbol_map - @abstractmethod def init(self): """ @@ -311,24 +301,13 @@ class Exchange: """ @abstractmethod - def load_assets(self, is_local=False): + def create_exchange_config(self): """ - Populate the 'assets' attribute with a dictionary of Assets. - The key of the resulting dictionary is the exchange specific - currency pair symbol. The universal symbol is contained in the - 'symbol' attribute of each asset. - - Notes - ----- - The sid of each asset is calculated based on a numeric hash of the - universal symbol. This simple approach avoids maintaining a mapping - of sids. - - This method can be omerridden if an exchange offers equivalent data - via its api. + Fetch the exchange market data and generate a config object + Returns + ------- """ - pass def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index ea57a845..343e38bb 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -626,13 +626,13 @@ class ExchangeBundle: key=lambda chunk: pd.to_datetime(chunk['period']) ) with maybe_show_progress( - all_chunks, - show_progress, - label='Ingesting {frequency} price data on ' - '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - )) as it: + all_chunks, + show_progress, + label='Ingesting {frequency} price data on ' + '{exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index fb24f1c8..49d347b1 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -11,11 +11,14 @@ from six import string_types from six.moves.urllib import request from catalyst.constants import DATE_FORMAT, SYMBOLS_URL -from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound +from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ + InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ - ExchangeJSONDecoder + ExchangeJSONDecoder, ConfigJSONEncoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time +from six import string_types +from six.moves.urllib import request def get_sid(symbol): @@ -69,7 +72,7 @@ def is_blacklist(exchange_name, environ=None): return os.path.exists(filename) -def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): +def get_exchange_config_filename(exchange_name, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -83,12 +86,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): str """ - name = 'symbols.json' if not is_local else 'symbols_local.json' + name = 'config.json' exchange_folder = get_exchange_folder(exchange_name, environ) return os.path.join(exchange_folder, name) -def download_exchange_symbols(exchange_name, environ=None): +def download_exchange_config(exchange_name, filename, environ=None): """ Downloads the exchange's symbols.json from the repository. @@ -102,15 +105,13 @@ def download_exchange_symbols(exchange_name, environ=None): str """ - filename = get_exchange_symbols_filename(exchange_name) - url = SYMBOLS_URL.format(exchange=exchange_name) - response = request.urlretrieve(url=url, filename=filename) - return response + url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) + request.urlretrieve(url=url, filename=filename) -def get_exchange_symbols(exchange_name, is_local=False, environ=None): +def get_exchange_config(exchange_name, filename=None, environ=None): """ - The de-serialized content of the exchange's symbols.json. + The de-serialized content of the exchange's config.json. Parameters ---------- @@ -123,55 +124,47 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): Object """ - filename = get_exchange_symbols_filename(exchange_name, is_local) - - if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): - try: - download_exchange_symbols(exchange_name, environ) - except Exception: - pass + if filename is None: + filename = get_exchange_config_filename(exchange_name) if os.path.isfile(filename): - with open(filename) as data_file: - try: - data = json.load(data_file, cls=ExchangeJSONDecoder) - return data + now = pd.Timestamp.utcnow() + limit = pd.Timedelta('2H') + if pd.Timedelta(now - last_modified_time(filename)) > limit: + download_exchange_config(exchange_name, filename, environ) - except ValueError: - return dict() else: - raise ExchangeSymbolsNotFound( - exchange=exchange_name, - filename=filename - ) + download_exchange_config(exchange_name, filename, environ) + with open(filename) as data_file: + try: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data -def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None): + except ValueError: + return dict() + +def save_exchange_config(exchange_name, config, filename=None, environ=None): """ - Save assets into an exchange_symbols file. + Save assets into an exchange_config file. Parameters ---------- exchange_name: str - assets: list[dict[str, object]] - is_local: bool + config environ Returns ------- """ - asset_dicts = dict() - for symbol in assets: - asset_dicts[symbol] = assets[symbol].to_dict() + if filename is None: + name = 'config.json' + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, name) - filename = get_exchange_symbols_filename( - exchange_name, is_local, environ - ) - with open(filename, 'wt') as handle: - json.dump(asset_dicts, handle, indent=4, default=symbols_serial) + with open(filename, 'w+') as handle: + json.dump(config, handle, indent=4, cls=ConfigJSONEncoder) def get_symbols_string(assets): @@ -508,25 +501,6 @@ def has_bundle(exchange_name, data_frequency, environ=None): return os.path.isdir(folder) -def symbols_serial(obj): - """ - JSON serializer for objects not serializable by default json code - - Parameters - ---------- - obj: Object - - Returns - ------- - str - - """ - if isinstance(obj, (datetime, date)): - return obj.floor('1D').strftime(DATE_FORMAT) - - raise TypeError("Type %s not serializable" % type(obj)) - - def perf_serial(obj): """ JSON serializer for objects not serializable by default json code @@ -616,46 +590,12 @@ def resample_history_df(df, freq, field, start_dt=None): return resampled_df -def mixin_market_params(exchange_name, params, market): - """ - Applies a CCXT market dict to parameters of TradingPair init. +def from_ms_timestamp(ms): + return pd.to_datetime(ms, unit='ms', utc=True) - Parameters - ---------- - params: dict[Object] - market: dict[Object] - Returns - ------- - - """ - # TODO: make this more externalized / configurable - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] - - if exchange_name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 - - elif 'maker' in market and 'taker' in market and \ - market['maker'] is not None and market['taker'] is not None: - - params['maker'] = market['maker'] - params['taker'] = market['taker'] - - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 - - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) - - if 'lot' not in params: - params['lot'] = params['min_trade_size'] +def get_epoch(): + return pd.to_datetime('1970-1-1', utc=True) def group_assets_by_exchange(assets): diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 6e1d21e3..4f849c3b 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -3,10 +3,33 @@ import re from json import JSONEncoder import pandas as pd -from catalyst.assets._assets import TradingPair -from catalyst.constants import DATE_TIME_FORMAT from six import string_types +from datetime import date, datetime +from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT +from catalyst.assets._assets import TradingPair + + +class ConfigJSONEncoder(json.JSONEncoder): + def default(self, obj): + """ + JSON serializer for objects not serializable by default json code + + Parameters + ---------- + obj: Object + + Returns + ------- + str + + """ + if isinstance(obj, (datetime, date)): + return obj.floor('1D').strftime(DATE_FORMAT) + + elif isinstance(obj, TradingPair): + return obj.to_dict() + class ExchangeJSONEncoder(json.JSONEncoder): def default(self, obj):