From e872b1fc822b5feb0db873fba8ec236d8cde935c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 17:28:32 -0500 Subject: [PATCH] BLD: replacing symbols.json and fetching markets with a single config --- catalyst/assets/_assets.pyx | 66 +++- catalyst/constants.py | 2 +- catalyst/exchange/ccxt/ccxt_exchange.py | 316 ++++++++---------- catalyst/exchange/exchange.py | 34 +- catalyst/exchange/exchange_bundle.py | 89 +++-- catalyst/exchange/exchange_errors.py | 14 + catalyst/exchange/utils/exchange_utils.py | 140 ++------ .../exchange/utils/serialization_utils.py | 25 +- etc/requirements.txt | 2 +- tests/exchange/test_config.py | 8 + 10 files changed, 323 insertions(+), 373 deletions(-) create mode 100644 tests/exchange/test_config.py diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index f80fbf4a..bc896313 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -433,7 +433,7 @@ cdef class TradingPair(Asset): 'taker', 'trading_state', 'data_source', - 'decimals' + 'decimals', }) def __init__(self, object symbol, @@ -455,7 +455,7 @@ cdef class TradingPair(Asset): float taker=0.0025, float lot=0, int decimals = 8, - int trading_state=0, + int trading_state=1, object data_source='catalyst'): """ Replicates the Asset constructor with some built-in conventions @@ -600,14 +600,51 @@ cdef class TradingPair(Asset): cpdef to_dict(self): """ Convert to a python dict. + + Repeat constructor params: + object symbol, + object exchange, + object start_date=None, + object asset_name=None, + int sid=0, + float leverage=1.0, + object end_daily=None, + object end_minute=None, + object end_date=None, + object exchange_symbol=None, + object first_traded=None, + object auto_close_date=None, + object exchange_full=None, + float min_trade_size=0.0001, + float max_trade_size=1000000, + float maker=0.0015, + float taker=0.0025, + float lot=0, + int decimals = 8, + int trading_state=1, + object data_source='catalyst', """ - #TODO: missing fields - super_dict = super(TradingPair, self).to_dict() - super_dict['end_daily'] = self.end_daily - super_dict['end_minute'] = self.end_minute - super_dict['leverage'] = self.leverage - super_dict['min_trade_size'] = self.min_trade_size - return super_dict + trading_pair_dict = dict( + symbol=self.symbol, + exchange=self.exchange, + start_date=self.start_date, + asset_name=self.asset_name, + leverage=self.leverage, + end_daily=self.end_daily, + end_minute=self.end_minute, + end_date=self.end_date, + exchange_symbol=self.exchange_symbol, + exchange_full=self.exchange_full, + min_trade_size=self.min_trade_size, + max_trade_size=self.max_trade_size, + maker=self.maker, + taker=self.taker, + lot=self.lot, + decimals=self.decimals, + trading_state=self.trading_state, + data_source=self.data_source, + ) + return trading_pair_dict def is_exchange_open(self, dt_minute): """ @@ -623,6 +660,13 @@ cdef class TradingPair(Asset): #TODO: make more dymanic to catch holds return True + def set_end_date(self, dt, data_frequency): + if data_frequency == 'minute': + self.end_minute = dt + + else: + self.end_daily = dt + cpdef __reduce__(self): """ Function used by pickle to determine how to serialize/deserialize this @@ -646,7 +690,9 @@ cdef class TradingPair(Asset): self.lot, self.decimals, self.taker, - self.maker)) + self.maker, + self.trading_state, + self.data_source)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..375dbe21 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' - +EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json' DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index ed3b2b24..2498c217 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,5 +1,3 @@ -import json -import os import re from collections import defaultdict @@ -13,15 +11,17 @@ from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ + MarketsNotFoundError, InvalidMarketError from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ - from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \ - get_exchange_auth +from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \ + get_epoch, get_catalyst_symbol, \ + get_exchange_auth, get_exchange_config from catalyst.finance.order import Order, ORDER_STATUS from ccxt import InvalidOrder, NetworkError, \ ExchangeError from logbook import Logger +from redo import retry from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -55,6 +55,8 @@ class CCXT(Exchange): 'apiKey': key, 'secret': secret, }) + self.has = self.api.has + self.fees = self.api.fees except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) @@ -62,6 +64,7 @@ class CCXT(Exchange): self._symbol_maps = [None, None] self.name = exchange_name + self.assets = [] self.base_currency = base_currency self.transactions = defaultdict(list) @@ -72,50 +75,115 @@ class CCXT(Exchange): self.request_cpt = dict() self.bundle = ExchangeBundle(self.name) - self.markets = None self._is_init = False + self._config = None def init(self): if self._is_init: return - exchange_folder = get_exchange_folder(self.name) - filename = os.path.join(exchange_folder, 'cctx_markets.json') - - if os.path.exists(filename): - timestamp = os.path.getmtime(filename) - dt = pd.to_datetime(timestamp, unit='s', utc=True) - - if dt >= pd.Timestamp.utcnow().floor('1D'): - with open(filename) as f: - self.markets = json.load(f) - - log.debug('loaded markets for {}'.format(self.name)) - - if self.markets is None: - try: - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) + if self._config is None: + self._config = get_exchange_config(self.name) + log.debug( + 'got exchange config {}:\n{}'.format( + self.name, self._config ) - - self.markets = self.api.fetch_markets() - with open(filename, 'w+') as f: - json.dump(self.markets, f, indent=4) - - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch markets {}: {}'.format( - self.name, e - ) - ) - raise ExchangeRequestError(error=e) + ) self.load_assets() self._is_init = True + def load_assets(self): + if self._config is None: + raise ValueError('Exchange config not available.') + + self.assets = [] + for asset_dict in self._config['assets']: + asset = TradingPair(**asset_dict) + self.assets.append(asset) + + def _fetch_markets(self): + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + try: + markets = self.api.fetch_markets() + + except NetworkError as e: + raise ExchangeRequestError(error=e) + + if not markets: + raise MarketsNotFoundError( + exchange=self.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=self.name, + market=market, + ) + return markets + + def create_exchange_config(self): + config = dict( + name=self.name, + features=[feature for feature in self.has if self.has[feature]] + ) + markets = retry( + action=self._fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(self.name) + ), + ) + + config['assets'] = [] + for market in markets: + asset = self.create_trading_pair(market=market) + config['assets'].append(asset) + + return config + + def create_trading_pair(self, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=self.name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + self.apply_conditional_market_params(params, market) + + return TradingPair(**params) + @staticmethod def find_exchanges(features=None, is_authenticated=False): ccxt_features = [] @@ -194,27 +262,6 @@ class CCXT(Exchange): return frequencies - def get_market(self, symbol): - """ - The CCXT market. - - Parameters - ---------- - symbol: - The CCXT symbol. - - Returns - ------- - dict[str, Object] - - """ - s = self.get_symbol(symbol) - market = next( - (market for market in self.markets if market['symbol'] == s), - None, - ) - return market - def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -417,144 +464,53 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def get_asset_defs(self, market): + def apply_conditional_market_params(self, params, market): """ - The local and Catalyst definitions of the specified market. + Applies a CCXT market dict to parameters of TradingPair init. Parameters ---------- - market: dict[str, Object] - The CCXT market dicts. + params: dict[Object] + market: dict[Object] Returns ------- - dict[str, Object] - The asset definition. """ - asset_defs = [] - - for is_local in (False, True): - asset_def = self.get_asset_def(market, is_local) - asset_defs.append((asset_def, is_local)) - - return asset_defs - - def get_asset_def(self, market, is_local=False): - """ - The asset definition (in symbols.json files) corresponding - to the the specified market. - - Parameters - ---------- - market: dict[str, Object] - The CCXT market dict. - is_local - Whether to search in local or Catalyst asset definitions. - - Returns - ------- - dict[str, Object] - The asset definition. - - """ - exchange_symbol = market['id'] - - symbol_map = self._fetch_symbol_map(is_local) - if symbol_map is not None: - assets_lower = {k.lower(): v for k, v in symbol_map.items()} - key = exchange_symbol.lower() - - asset = assets_lower[key] if key in assets_lower else None - if asset is not None: - return asset - - else: - return None + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 else: - return None + params['trading_state'] = 1 - def create_trading_pair(self, market, asset_def=None, is_local=False): - """ - Creating a TradingPair from market and asset data. + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] - Parameters - ---------- - market: dict[str, Object] - asset_def: dict[str, Object] - is_local: bool + if self.name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 - Returns - ------- - - """ - data_source = 'local' if is_local else 'catalyst' - params = dict( - exchange=self.name, - data_source=data_source, - exchange_symbol=market['id'], - ) - mixin_market_params(self.name, params, market) - - if asset_def is not None: - params['symbol'] = asset_def['symbol'] - - params['start_date'] = asset_def['start_date'] \ - if 'start_date' in asset_def else None - - params['end_date'] = asset_def['end_date'] \ - if 'end_date' in asset_def else None - - params['leverage'] = asset_def['leverage'] \ - if 'leverage' in asset_def else 1.0 - - params['asset_name'] = asset_def['asset_name'] \ - if 'asset_name' in asset_def else None - - params['end_daily'] = asset_def['end_daily'] \ - if 'end_daily' in asset_def \ - and asset_def['end_daily'] != 'N/A' else None - - params['end_minute'] = asset_def['end_minute'] \ - if 'end_minute' in asset_def \ - and asset_def['end_minute'] != 'N/A' else None + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] else: - params['symbol'] = get_catalyst_symbol(market) - # TODO: add as an optional column - params['leverage'] = 1.0 + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 - return TradingPair(**params) + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) - def load_assets(self): - log.debug('loading assets for {}'.format(self.name)) - self.assets = [] - - for market in self.markets: - if 'id' not in market: - log.warn('invalid market: {}'.format(market)) - continue - - asset_defs = self.get_asset_defs(market) - - asset = None - for asset_def in asset_defs: - if asset_def[0] is not None or not asset_defs[1]: - try: - asset = self.create_trading_pair( - market=market, - asset_def=asset_def[0], - is_local=asset_def[1] - ) - self.assets.append(asset) - - except TypeError as e: - log.warn('unable to add asset: {}'.format(e)) - - if asset is None: - asset = self.create_trading_pair(market=market) - self.assets.append(asset) + if 'lot' not in params: + params['lot'] = params['min_trade_size'] def get_balances(self): try: diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index b6b24d20..863089fa 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -5,8 +5,6 @@ from time import sleep import numpy as np import pandas as pd -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -17,8 +15,9 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ TickerNotFoundError, NotEnoughCashError from catalyst.exchange.utils.bundle_utils import get_start_dt, \ get_delta, get_periods, get_periods_range -from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ +from catalyst.exchange.utils.exchange_utils import \ get_frequency, resample_history_df, has_bundle +from logbook import Logger log = Logger('Exchange', level=LOG_LEVEL) @@ -284,16 +283,6 @@ class Exchange: log.debug('found asset: {}'.format(asset)) return asset - def fetch_symbol_map(self, is_local=False): - index = 1 if is_local else 0 - if self._symbol_maps[index] is not None: - return self._symbol_maps[index] - - else: - symbol_map = get_exchange_symbols(self.name, is_local) - self._symbol_maps[index] = symbol_map - return symbol_map - @abstractmethod def init(self): """ @@ -305,24 +294,13 @@ class Exchange: """ @abstractmethod - def load_assets(self, is_local=False): + def create_exchange_config(self): """ - Populate the 'assets' attribute with a dictionary of Assets. - The key of the resulting dictionary is the exchange specific - currency pair symbol. The universal symbol is contained in the - 'symbol' attribute of each asset. - - Notes - ----- - The sid of each asset is calculated based on a numeric hash of the - universal symbol. This simple approach avoids maintaining a mapping - of sids. - - This method can be omerridden if an exchange offers equivalent data - via its api. + Fetch the exchange market data and generate a config object + Returns + ------- """ - pass def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index c3d032d2..113bd2ec 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -5,6 +5,7 @@ from functools import partial from itertools import chain from operator import is_not +import copy import numpy as np import pandas as pd import pytz @@ -29,7 +30,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \ get_delta, get_assets from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params, get_catalyst_symbol + get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory @@ -233,12 +234,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has empty ' \ 'periods: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime( - DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - dates=[date.strftime( - DATE_TIME_FORMAT) for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime( + DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + dates=[date.strftime( + DATE_TIME_FORMAT) for date in dates]) if empty_rows_behavior == 'warn': log.warn(problem) @@ -287,12 +288,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \ 'identical close values on: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - threshold=threshold, - dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) - for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + threshold=threshold, + dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) + for date in dates]) problems.append(problem) @@ -630,8 +631,8 @@ class ExchangeBundle: show_progress, label='Ingesting {frequency} price data on ' '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, + exchange=self.exchange_name, + frequency=data_frequency, )) as it: for chunk in it: problems += self.ingest_ctable( @@ -701,42 +702,36 @@ class ExchangeBundle: for symbol in symbols: start_dt = df.index.get_level_values(1).min() end_dt = df.index.get_level_values(1).max() - end_dt_key = 'end_{}'.format(data_frequency) - market = self.exchange.get_market(symbol) - if market is None: - raise ValueError('symbol not available in the exchange.') + try: + asset = self.exchange.get_asset(symbol, is_local=True) + except: + asset = copy.deepcopy(self.exchange.get_asset(symbol)) - params = dict( - exchange=self.exchange.name, - data_source='local', - exchange_symbol=market['id'], - ) - mixin_market_params(self.exchange_name, params, market) + if asset.data_source == 'local': + asset.start_date = asset.start_date \ + if asset.start_date < start_dt else start_dt - asset_def = self.exchange.get_asset_def(market, True) - if asset_def is not None: - params['symbol'] = asset_def['symbol'] + if data_frequency == 'daily': + asset.end_date = asset.end_daily = asset.end_daily \ + if asset.end_daily > end_dt else end_dt - params['start_date'] = asset_def['start_date'] \ - if asset_def['start_date'] < start_dt else start_dt - - params['end_date'] = asset_def[end_dt_key] \ - if asset_def[end_dt_key] > end_dt else end_dt - - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else asset_def['end_daily'] - - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else asset_def['end_minute'] + else: + asset.end_date = asset.end_minute = asset.end_minute \ + if asset.end_minute > end_dt else end_dt else: - params['symbol'] = get_catalyst_symbol(market) + asset.data_source = 'local' + asset.start_date = start_dt + asset.end_dt = end_dt - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else 'N/A' - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else 'N/A' + if data_frequency == 'daily': + asset.end_daily = end_dt + asset.end_minute = None + + else: + asset.end_daily = None + asset.end_minute = end_dt if min_start_dt is None or start_dt < min_start_dt: min_start_dt = start_dt @@ -744,11 +739,9 @@ class ExchangeBundle: if max_end_dt is None or end_dt > max_end_dt: max_end_dt = end_dt - asset = TradingPair(**params) - assets[market['id']] = asset - - save_exchange_symbols(self.exchange_name, assets, True) + assets[symbol] = asset + # TODO: update config.json writer = self.get_writer( start_dt=min_start_dt.replace(hour=00, minute=00), end_dt=max_end_dt.replace(hour=23, minute=59), diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 0e22868f..4e7cb244 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -315,3 +315,17 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() + + +class MarketsNotFoundError(ZiplineError): + msg = ( + 'Exchange {exchange} contains no valid market so it is unusable in ' + 'Catalyst.' + ).strip() + + +class InvalidMarketError(ZiplineError): + msg = ( + 'Exchange {exchange} contains at least one incorrectly structured ' + 'market: {market}, so it is unusable in Catalyst.' + ).strip() diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index ab16f71d..6b3c68a6 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -8,16 +8,15 @@ from datetime import date, datetime import pandas as pd from catalyst.assets._assets import TradingPair -from six import string_types -from six.moves.urllib import request - -from catalyst.constants import DATE_FORMAT, SYMBOLS_URL -from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ - InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias +from catalyst.constants import EXCHANGE_CONFIG_URL +from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ + InvalidHistoryFrequencyAlias from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ - ExchangeJSONDecoder + ExchangeJSONDecoder, ConfigJSONEncoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time +from six import string_types +from six.moves.urllib import request def get_sid(symbol): @@ -71,7 +70,7 @@ def is_blacklist(exchange_name, environ=None): return os.path.exists(filename) -def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): +def get_exchange_config_filename(exchange_name, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -85,12 +84,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): str """ - name = 'symbols.json' if not is_local else 'symbols_local.json' + name = 'config.json' exchange_folder = get_exchange_folder(exchange_name, environ) return os.path.join(exchange_folder, name) -def download_exchange_symbols(exchange_name, environ=None): +def download_exchange_config(exchange_name, filename, environ=None): """ Downloads the exchange's symbols.json from the repository. @@ -104,15 +103,13 @@ def download_exchange_symbols(exchange_name, environ=None): str """ - filename = get_exchange_symbols_filename(exchange_name) - url = SYMBOLS_URL.format(exchange=exchange_name) - response = request.urlretrieve(url=url, filename=filename) - return response + url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) + request.urlretrieve(url=url, filename=filename) -def get_exchange_symbols(exchange_name, is_local=False, environ=None): +def get_exchange_config(exchange_name, filename=None, environ=None): """ - The de-serialized content of the exchange's symbols.json. + The de-serialized content of the exchange's config.json. Parameters ---------- @@ -125,52 +122,47 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): Object """ - filename = get_exchange_symbols_filename(exchange_name, is_local) - - if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): - download_exchange_symbols(exchange_name, environ) + if filename is None: + filename = get_exchange_config_filename(exchange_name) if os.path.isfile(filename): - with open(filename) as data_file: - try: - data = json.load(data_file, cls=ExchangeJSONDecoder) - return data + now = pd.Timestamp.utcnow() + limit = pd.Timedelta('2H') + if pd.Timedelta(now - last_modified_time(filename)) > limit: + download_exchange_config(exchange_name, filename, environ) - except ValueError: - return dict() else: - raise ExchangeSymbolsNotFound( - exchange=exchange_name, - filename=filename - ) + download_exchange_config(exchange_name, filename, environ) + with open(filename) as data_file: + try: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data -def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None): + except ValueError: + return dict() + +def save_exchange_config(exchange_name, config, filename=None, environ=None): """ - Save assets into an exchange_symbols file. + Save assets into an exchange_config file. Parameters ---------- exchange_name: str - assets: list[dict[str, object]] - is_local: bool + config environ Returns ------- """ - asset_dicts = dict() - for symbol in assets: - asset_dicts[symbol] = assets[symbol].to_dict() + if filename is None: + name = 'config.json' + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, name) - filename = get_exchange_symbols_filename( - exchange_name, is_local, environ - ) - with open(filename, 'wt') as handle: - json.dump(asset_dicts, handle, indent=4, default=symbols_serial) + with open(filename, 'w+') as handle: + json.dump(config, handle, indent=4, cls=ConfigJSONEncoder) def get_symbols_string(assets): @@ -443,25 +435,6 @@ def has_bundle(exchange_name, data_frequency, environ=None): return os.path.isdir(folder) -def symbols_serial(obj): - """ - JSON serializer for objects not serializable by default json code - - Parameters - ---------- - obj: Object - - Returns - ------- - str - - """ - if isinstance(obj, (datetime, date)): - return obj.floor('1D').strftime(DATE_FORMAT) - - raise TypeError("Type %s not serializable" % type(obj)) - - def perf_serial(obj): """ JSON serializer for objects not serializable by default json code @@ -608,47 +581,6 @@ def resample_history_df(df, freq, field): return resampled_df -def mixin_market_params(exchange_name, params, market): - """ - Applies a CCXT market dict to parameters of TradingPair init. - - Parameters - ---------- - params: dict[Object] - market: dict[Object] - - Returns - ------- - - """ - # TODO: make this more externalized / configurable - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] - - if exchange_name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 - - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None and market['taker'] is not None: - params['maker'] = market['maker'] - params['taker'] = market['taker'] - - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 - - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) - - if 'lot' not in params: - params['lot'] = params['min_trade_size'] - - def from_ms_timestamp(ms): return pd.to_datetime(ms, unit='ms', utc=True) diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 4b098a02..e0e202b0 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -5,7 +5,30 @@ from json import JSONEncoder import pandas as pd from six import string_types -from catalyst.constants import DATE_TIME_FORMAT +from datetime import date, datetime +from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT +from catalyst.assets._assets import TradingPair + + +class ConfigJSONEncoder(json.JSONEncoder): + def default(self, obj): + """ + JSON serializer for objects not serializable by default json code + + Parameters + ---------- + obj: Object + + Returns + ------- + str + + """ + if isinstance(obj, (datetime, date)): + return obj.floor('1D').strftime(DATE_FORMAT) + + elif isinstance(obj, TradingPair): + return obj.to_dict() class ExchangeJSONEncoder(json.JSONEncoder): diff --git a/etc/requirements.txt b/etc/requirements.txt index f01a54b5..bb5352c5 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -82,5 +82,5 @@ tables==3.3.0 #Catalyst dependencies ccxt==1.10.565 -boto3==1.4.8 +boto3==1.5.7 redo==1.6 diff --git a/tests/exchange/test_config.py b/tests/exchange/test_config.py new file mode 100644 index 00000000..f13f526b --- /dev/null +++ b/tests/exchange/test_config.py @@ -0,0 +1,8 @@ +from catalyst.exchange.utils.factory import get_exchange + + +class TestConfig: + def test_create_config(self): + exchange = get_exchange('binance', skip_init=True) + config = exchange.create_exchange_config() + pass