From e872b1fc822b5feb0db873fba8ec236d8cde935c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 17:28:32 -0500 Subject: [PATCH 1/9] BLD: replacing symbols.json and fetching markets with a single config --- catalyst/assets/_assets.pyx | 66 +++- catalyst/constants.py | 2 +- catalyst/exchange/ccxt/ccxt_exchange.py | 316 ++++++++---------- catalyst/exchange/exchange.py | 34 +- catalyst/exchange/exchange_bundle.py | 89 +++-- catalyst/exchange/exchange_errors.py | 14 + catalyst/exchange/utils/exchange_utils.py | 140 ++------ .../exchange/utils/serialization_utils.py | 25 +- etc/requirements.txt | 2 +- tests/exchange/test_config.py | 8 + 10 files changed, 323 insertions(+), 373 deletions(-) create mode 100644 tests/exchange/test_config.py diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index f80fbf4a..bc896313 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -433,7 +433,7 @@ cdef class TradingPair(Asset): 'taker', 'trading_state', 'data_source', - 'decimals' + 'decimals', }) def __init__(self, object symbol, @@ -455,7 +455,7 @@ cdef class TradingPair(Asset): float taker=0.0025, float lot=0, int decimals = 8, - int trading_state=0, + int trading_state=1, object data_source='catalyst'): """ Replicates the Asset constructor with some built-in conventions @@ -600,14 +600,51 @@ cdef class TradingPair(Asset): cpdef to_dict(self): """ Convert to a python dict. + + Repeat constructor params: + object symbol, + object exchange, + object start_date=None, + object asset_name=None, + int sid=0, + float leverage=1.0, + object end_daily=None, + object end_minute=None, + object end_date=None, + object exchange_symbol=None, + object first_traded=None, + object auto_close_date=None, + object exchange_full=None, + float min_trade_size=0.0001, + float max_trade_size=1000000, + float maker=0.0015, + float taker=0.0025, + float lot=0, + int decimals = 8, + int trading_state=1, + object data_source='catalyst', """ - #TODO: missing fields - super_dict = super(TradingPair, self).to_dict() - super_dict['end_daily'] = self.end_daily - super_dict['end_minute'] = self.end_minute - super_dict['leverage'] = self.leverage - super_dict['min_trade_size'] = self.min_trade_size - return super_dict + trading_pair_dict = dict( + symbol=self.symbol, + exchange=self.exchange, + start_date=self.start_date, + asset_name=self.asset_name, + leverage=self.leverage, + end_daily=self.end_daily, + end_minute=self.end_minute, + end_date=self.end_date, + exchange_symbol=self.exchange_symbol, + exchange_full=self.exchange_full, + min_trade_size=self.min_trade_size, + max_trade_size=self.max_trade_size, + maker=self.maker, + taker=self.taker, + lot=self.lot, + decimals=self.decimals, + trading_state=self.trading_state, + data_source=self.data_source, + ) + return trading_pair_dict def is_exchange_open(self, dt_minute): """ @@ -623,6 +660,13 @@ cdef class TradingPair(Asset): #TODO: make more dymanic to catch holds return True + def set_end_date(self, dt, data_frequency): + if data_frequency == 'minute': + self.end_minute = dt + + else: + self.end_daily = dt + cpdef __reduce__(self): """ Function used by pickle to determine how to serialize/deserialize this @@ -646,7 +690,9 @@ cdef class TradingPair(Asset): self.lot, self.decimals, self.taker, - self.maker)) + self.maker, + self.trading_state, + self.data_source)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..375dbe21 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' - +EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json' DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index ed3b2b24..2498c217 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,5 +1,3 @@ -import json -import os import re from collections import defaultdict @@ -13,15 +11,17 @@ from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ + MarketsNotFoundError, InvalidMarketError from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ - from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \ - get_exchange_auth +from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \ + get_epoch, get_catalyst_symbol, \ + get_exchange_auth, get_exchange_config from catalyst.finance.order import Order, ORDER_STATUS from ccxt import InvalidOrder, NetworkError, \ ExchangeError from logbook import Logger +from redo import retry from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -55,6 +55,8 @@ class CCXT(Exchange): 'apiKey': key, 'secret': secret, }) + self.has = self.api.has + self.fees = self.api.fees except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) @@ -62,6 +64,7 @@ class CCXT(Exchange): self._symbol_maps = [None, None] self.name = exchange_name + self.assets = [] self.base_currency = base_currency self.transactions = defaultdict(list) @@ -72,50 +75,115 @@ class CCXT(Exchange): self.request_cpt = dict() self.bundle = ExchangeBundle(self.name) - self.markets = None self._is_init = False + self._config = None def init(self): if self._is_init: return - exchange_folder = get_exchange_folder(self.name) - filename = os.path.join(exchange_folder, 'cctx_markets.json') - - if os.path.exists(filename): - timestamp = os.path.getmtime(filename) - dt = pd.to_datetime(timestamp, unit='s', utc=True) - - if dt >= pd.Timestamp.utcnow().floor('1D'): - with open(filename) as f: - self.markets = json.load(f) - - log.debug('loaded markets for {}'.format(self.name)) - - if self.markets is None: - try: - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) + if self._config is None: + self._config = get_exchange_config(self.name) + log.debug( + 'got exchange config {}:\n{}'.format( + self.name, self._config ) - - self.markets = self.api.fetch_markets() - with open(filename, 'w+') as f: - json.dump(self.markets, f, indent=4) - - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch markets {}: {}'.format( - self.name, e - ) - ) - raise ExchangeRequestError(error=e) + ) self.load_assets() self._is_init = True + def load_assets(self): + if self._config is None: + raise ValueError('Exchange config not available.') + + self.assets = [] + for asset_dict in self._config['assets']: + asset = TradingPair(**asset_dict) + self.assets.append(asset) + + def _fetch_markets(self): + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + try: + markets = self.api.fetch_markets() + + except NetworkError as e: + raise ExchangeRequestError(error=e) + + if not markets: + raise MarketsNotFoundError( + exchange=self.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=self.name, + market=market, + ) + return markets + + def create_exchange_config(self): + config = dict( + name=self.name, + features=[feature for feature in self.has if self.has[feature]] + ) + markets = retry( + action=self._fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(self.name) + ), + ) + + config['assets'] = [] + for market in markets: + asset = self.create_trading_pair(market=market) + config['assets'].append(asset) + + return config + + def create_trading_pair(self, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=self.name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + self.apply_conditional_market_params(params, market) + + return TradingPair(**params) + @staticmethod def find_exchanges(features=None, is_authenticated=False): ccxt_features = [] @@ -194,27 +262,6 @@ class CCXT(Exchange): return frequencies - def get_market(self, symbol): - """ - The CCXT market. - - Parameters - ---------- - symbol: - The CCXT symbol. - - Returns - ------- - dict[str, Object] - - """ - s = self.get_symbol(symbol) - market = next( - (market for market in self.markets if market['symbol'] == s), - None, - ) - return market - def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -417,144 +464,53 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def get_asset_defs(self, market): + def apply_conditional_market_params(self, params, market): """ - The local and Catalyst definitions of the specified market. + Applies a CCXT market dict to parameters of TradingPair init. Parameters ---------- - market: dict[str, Object] - The CCXT market dicts. + params: dict[Object] + market: dict[Object] Returns ------- - dict[str, Object] - The asset definition. """ - asset_defs = [] - - for is_local in (False, True): - asset_def = self.get_asset_def(market, is_local) - asset_defs.append((asset_def, is_local)) - - return asset_defs - - def get_asset_def(self, market, is_local=False): - """ - The asset definition (in symbols.json files) corresponding - to the the specified market. - - Parameters - ---------- - market: dict[str, Object] - The CCXT market dict. - is_local - Whether to search in local or Catalyst asset definitions. - - Returns - ------- - dict[str, Object] - The asset definition. - - """ - exchange_symbol = market['id'] - - symbol_map = self._fetch_symbol_map(is_local) - if symbol_map is not None: - assets_lower = {k.lower(): v for k, v in symbol_map.items()} - key = exchange_symbol.lower() - - asset = assets_lower[key] if key in assets_lower else None - if asset is not None: - return asset - - else: - return None + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 else: - return None + params['trading_state'] = 1 - def create_trading_pair(self, market, asset_def=None, is_local=False): - """ - Creating a TradingPair from market and asset data. + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] - Parameters - ---------- - market: dict[str, Object] - asset_def: dict[str, Object] - is_local: bool + if self.name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 - Returns - ------- - - """ - data_source = 'local' if is_local else 'catalyst' - params = dict( - exchange=self.name, - data_source=data_source, - exchange_symbol=market['id'], - ) - mixin_market_params(self.name, params, market) - - if asset_def is not None: - params['symbol'] = asset_def['symbol'] - - params['start_date'] = asset_def['start_date'] \ - if 'start_date' in asset_def else None - - params['end_date'] = asset_def['end_date'] \ - if 'end_date' in asset_def else None - - params['leverage'] = asset_def['leverage'] \ - if 'leverage' in asset_def else 1.0 - - params['asset_name'] = asset_def['asset_name'] \ - if 'asset_name' in asset_def else None - - params['end_daily'] = asset_def['end_daily'] \ - if 'end_daily' in asset_def \ - and asset_def['end_daily'] != 'N/A' else None - - params['end_minute'] = asset_def['end_minute'] \ - if 'end_minute' in asset_def \ - and asset_def['end_minute'] != 'N/A' else None + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] else: - params['symbol'] = get_catalyst_symbol(market) - # TODO: add as an optional column - params['leverage'] = 1.0 + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 - return TradingPair(**params) + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) - def load_assets(self): - log.debug('loading assets for {}'.format(self.name)) - self.assets = [] - - for market in self.markets: - if 'id' not in market: - log.warn('invalid market: {}'.format(market)) - continue - - asset_defs = self.get_asset_defs(market) - - asset = None - for asset_def in asset_defs: - if asset_def[0] is not None or not asset_defs[1]: - try: - asset = self.create_trading_pair( - market=market, - asset_def=asset_def[0], - is_local=asset_def[1] - ) - self.assets.append(asset) - - except TypeError as e: - log.warn('unable to add asset: {}'.format(e)) - - if asset is None: - asset = self.create_trading_pair(market=market) - self.assets.append(asset) + if 'lot' not in params: + params['lot'] = params['min_trade_size'] def get_balances(self): try: diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index b6b24d20..863089fa 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -5,8 +5,6 @@ from time import sleep import numpy as np import pandas as pd -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -17,8 +15,9 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ TickerNotFoundError, NotEnoughCashError from catalyst.exchange.utils.bundle_utils import get_start_dt, \ get_delta, get_periods, get_periods_range -from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ +from catalyst.exchange.utils.exchange_utils import \ get_frequency, resample_history_df, has_bundle +from logbook import Logger log = Logger('Exchange', level=LOG_LEVEL) @@ -284,16 +283,6 @@ class Exchange: log.debug('found asset: {}'.format(asset)) return asset - def fetch_symbol_map(self, is_local=False): - index = 1 if is_local else 0 - if self._symbol_maps[index] is not None: - return self._symbol_maps[index] - - else: - symbol_map = get_exchange_symbols(self.name, is_local) - self._symbol_maps[index] = symbol_map - return symbol_map - @abstractmethod def init(self): """ @@ -305,24 +294,13 @@ class Exchange: """ @abstractmethod - def load_assets(self, is_local=False): + def create_exchange_config(self): """ - Populate the 'assets' attribute with a dictionary of Assets. - The key of the resulting dictionary is the exchange specific - currency pair symbol. The universal symbol is contained in the - 'symbol' attribute of each asset. - - Notes - ----- - The sid of each asset is calculated based on a numeric hash of the - universal symbol. This simple approach avoids maintaining a mapping - of sids. - - This method can be omerridden if an exchange offers equivalent data - via its api. + Fetch the exchange market data and generate a config object + Returns + ------- """ - pass def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index c3d032d2..113bd2ec 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -5,6 +5,7 @@ from functools import partial from itertools import chain from operator import is_not +import copy import numpy as np import pandas as pd import pytz @@ -29,7 +30,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \ get_delta, get_assets from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params, get_catalyst_symbol + get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory @@ -233,12 +234,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has empty ' \ 'periods: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime( - DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - dates=[date.strftime( - DATE_TIME_FORMAT) for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime( + DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + dates=[date.strftime( + DATE_TIME_FORMAT) for date in dates]) if empty_rows_behavior == 'warn': log.warn(problem) @@ -287,12 +288,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \ 'identical close values on: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - threshold=threshold, - dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) - for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + threshold=threshold, + dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) + for date in dates]) problems.append(problem) @@ -630,8 +631,8 @@ class ExchangeBundle: show_progress, label='Ingesting {frequency} price data on ' '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, + exchange=self.exchange_name, + frequency=data_frequency, )) as it: for chunk in it: problems += self.ingest_ctable( @@ -701,42 +702,36 @@ class ExchangeBundle: for symbol in symbols: start_dt = df.index.get_level_values(1).min() end_dt = df.index.get_level_values(1).max() - end_dt_key = 'end_{}'.format(data_frequency) - market = self.exchange.get_market(symbol) - if market is None: - raise ValueError('symbol not available in the exchange.') + try: + asset = self.exchange.get_asset(symbol, is_local=True) + except: + asset = copy.deepcopy(self.exchange.get_asset(symbol)) - params = dict( - exchange=self.exchange.name, - data_source='local', - exchange_symbol=market['id'], - ) - mixin_market_params(self.exchange_name, params, market) + if asset.data_source == 'local': + asset.start_date = asset.start_date \ + if asset.start_date < start_dt else start_dt - asset_def = self.exchange.get_asset_def(market, True) - if asset_def is not None: - params['symbol'] = asset_def['symbol'] + if data_frequency == 'daily': + asset.end_date = asset.end_daily = asset.end_daily \ + if asset.end_daily > end_dt else end_dt - params['start_date'] = asset_def['start_date'] \ - if asset_def['start_date'] < start_dt else start_dt - - params['end_date'] = asset_def[end_dt_key] \ - if asset_def[end_dt_key] > end_dt else end_dt - - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else asset_def['end_daily'] - - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else asset_def['end_minute'] + else: + asset.end_date = asset.end_minute = asset.end_minute \ + if asset.end_minute > end_dt else end_dt else: - params['symbol'] = get_catalyst_symbol(market) + asset.data_source = 'local' + asset.start_date = start_dt + asset.end_dt = end_dt - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else 'N/A' - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else 'N/A' + if data_frequency == 'daily': + asset.end_daily = end_dt + asset.end_minute = None + + else: + asset.end_daily = None + asset.end_minute = end_dt if min_start_dt is None or start_dt < min_start_dt: min_start_dt = start_dt @@ -744,11 +739,9 @@ class ExchangeBundle: if max_end_dt is None or end_dt > max_end_dt: max_end_dt = end_dt - asset = TradingPair(**params) - assets[market['id']] = asset - - save_exchange_symbols(self.exchange_name, assets, True) + assets[symbol] = asset + # TODO: update config.json writer = self.get_writer( start_dt=min_start_dt.replace(hour=00, minute=00), end_dt=max_end_dt.replace(hour=23, minute=59), diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 0e22868f..4e7cb244 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -315,3 +315,17 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() + + +class MarketsNotFoundError(ZiplineError): + msg = ( + 'Exchange {exchange} contains no valid market so it is unusable in ' + 'Catalyst.' + ).strip() + + +class InvalidMarketError(ZiplineError): + msg = ( + 'Exchange {exchange} contains at least one incorrectly structured ' + 'market: {market}, so it is unusable in Catalyst.' + ).strip() diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index ab16f71d..6b3c68a6 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -8,16 +8,15 @@ from datetime import date, datetime import pandas as pd from catalyst.assets._assets import TradingPair -from six import string_types -from six.moves.urllib import request - -from catalyst.constants import DATE_FORMAT, SYMBOLS_URL -from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ - InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias +from catalyst.constants import EXCHANGE_CONFIG_URL +from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ + InvalidHistoryFrequencyAlias from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ - ExchangeJSONDecoder + ExchangeJSONDecoder, ConfigJSONEncoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time +from six import string_types +from six.moves.urllib import request def get_sid(symbol): @@ -71,7 +70,7 @@ def is_blacklist(exchange_name, environ=None): return os.path.exists(filename) -def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): +def get_exchange_config_filename(exchange_name, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -85,12 +84,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): str """ - name = 'symbols.json' if not is_local else 'symbols_local.json' + name = 'config.json' exchange_folder = get_exchange_folder(exchange_name, environ) return os.path.join(exchange_folder, name) -def download_exchange_symbols(exchange_name, environ=None): +def download_exchange_config(exchange_name, filename, environ=None): """ Downloads the exchange's symbols.json from the repository. @@ -104,15 +103,13 @@ def download_exchange_symbols(exchange_name, environ=None): str """ - filename = get_exchange_symbols_filename(exchange_name) - url = SYMBOLS_URL.format(exchange=exchange_name) - response = request.urlretrieve(url=url, filename=filename) - return response + url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) + request.urlretrieve(url=url, filename=filename) -def get_exchange_symbols(exchange_name, is_local=False, environ=None): +def get_exchange_config(exchange_name, filename=None, environ=None): """ - The de-serialized content of the exchange's symbols.json. + The de-serialized content of the exchange's config.json. Parameters ---------- @@ -125,52 +122,47 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): Object """ - filename = get_exchange_symbols_filename(exchange_name, is_local) - - if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): - download_exchange_symbols(exchange_name, environ) + if filename is None: + filename = get_exchange_config_filename(exchange_name) if os.path.isfile(filename): - with open(filename) as data_file: - try: - data = json.load(data_file, cls=ExchangeJSONDecoder) - return data + now = pd.Timestamp.utcnow() + limit = pd.Timedelta('2H') + if pd.Timedelta(now - last_modified_time(filename)) > limit: + download_exchange_config(exchange_name, filename, environ) - except ValueError: - return dict() else: - raise ExchangeSymbolsNotFound( - exchange=exchange_name, - filename=filename - ) + download_exchange_config(exchange_name, filename, environ) + with open(filename) as data_file: + try: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data -def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None): + except ValueError: + return dict() + +def save_exchange_config(exchange_name, config, filename=None, environ=None): """ - Save assets into an exchange_symbols file. + Save assets into an exchange_config file. Parameters ---------- exchange_name: str - assets: list[dict[str, object]] - is_local: bool + config environ Returns ------- """ - asset_dicts = dict() - for symbol in assets: - asset_dicts[symbol] = assets[symbol].to_dict() + if filename is None: + name = 'config.json' + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, name) - filename = get_exchange_symbols_filename( - exchange_name, is_local, environ - ) - with open(filename, 'wt') as handle: - json.dump(asset_dicts, handle, indent=4, default=symbols_serial) + with open(filename, 'w+') as handle: + json.dump(config, handle, indent=4, cls=ConfigJSONEncoder) def get_symbols_string(assets): @@ -443,25 +435,6 @@ def has_bundle(exchange_name, data_frequency, environ=None): return os.path.isdir(folder) -def symbols_serial(obj): - """ - JSON serializer for objects not serializable by default json code - - Parameters - ---------- - obj: Object - - Returns - ------- - str - - """ - if isinstance(obj, (datetime, date)): - return obj.floor('1D').strftime(DATE_FORMAT) - - raise TypeError("Type %s not serializable" % type(obj)) - - def perf_serial(obj): """ JSON serializer for objects not serializable by default json code @@ -608,47 +581,6 @@ def resample_history_df(df, freq, field): return resampled_df -def mixin_market_params(exchange_name, params, market): - """ - Applies a CCXT market dict to parameters of TradingPair init. - - Parameters - ---------- - params: dict[Object] - market: dict[Object] - - Returns - ------- - - """ - # TODO: make this more externalized / configurable - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] - - if exchange_name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 - - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None and market['taker'] is not None: - params['maker'] = market['maker'] - params['taker'] = market['taker'] - - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 - - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) - - if 'lot' not in params: - params['lot'] = params['min_trade_size'] - - def from_ms_timestamp(ms): return pd.to_datetime(ms, unit='ms', utc=True) diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 4b098a02..e0e202b0 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -5,7 +5,30 @@ from json import JSONEncoder import pandas as pd from six import string_types -from catalyst.constants import DATE_TIME_FORMAT +from datetime import date, datetime +from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT +from catalyst.assets._assets import TradingPair + + +class ConfigJSONEncoder(json.JSONEncoder): + def default(self, obj): + """ + JSON serializer for objects not serializable by default json code + + Parameters + ---------- + obj: Object + + Returns + ------- + str + + """ + if isinstance(obj, (datetime, date)): + return obj.floor('1D').strftime(DATE_FORMAT) + + elif isinstance(obj, TradingPair): + return obj.to_dict() class ExchangeJSONEncoder(json.JSONEncoder): diff --git a/etc/requirements.txt b/etc/requirements.txt index f01a54b5..bb5352c5 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -82,5 +82,5 @@ tables==3.3.0 #Catalyst dependencies ccxt==1.10.565 -boto3==1.4.8 +boto3==1.5.7 redo==1.6 diff --git a/tests/exchange/test_config.py b/tests/exchange/test_config.py new file mode 100644 index 00000000..f13f526b --- /dev/null +++ b/tests/exchange/test_config.py @@ -0,0 +1,8 @@ +from catalyst.exchange.utils.factory import get_exchange + + +class TestConfig: + def test_create_config(self): + exchange = get_exchange('binance', skip_init=True) + config = exchange.create_exchange_config() + pass From 6b3f59ff76c4e870e1aa3abb76e441401160e174 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 17:28:32 -0500 Subject: [PATCH 2/9] BLD: replacing symbols.json and fetching markets with a single config --- catalyst/assets/_assets.pyx | 66 ++++- catalyst/constants.py | 2 +- catalyst/exchange/ccxt/ccxt_exchange.py | 351 ++++++++++-------------- catalyst/exchange/exchange_bundle.py | 61 ++-- catalyst/exchange/exchange_errors.py | 14 + tests/exchange/test_config.py | 8 + 6 files changed, 254 insertions(+), 248 deletions(-) create mode 100644 tests/exchange/test_config.py diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index f80fbf4a..bc896313 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -433,7 +433,7 @@ cdef class TradingPair(Asset): 'taker', 'trading_state', 'data_source', - 'decimals' + 'decimals', }) def __init__(self, object symbol, @@ -455,7 +455,7 @@ cdef class TradingPair(Asset): float taker=0.0025, float lot=0, int decimals = 8, - int trading_state=0, + int trading_state=1, object data_source='catalyst'): """ Replicates the Asset constructor with some built-in conventions @@ -600,14 +600,51 @@ cdef class TradingPair(Asset): cpdef to_dict(self): """ Convert to a python dict. + + Repeat constructor params: + object symbol, + object exchange, + object start_date=None, + object asset_name=None, + int sid=0, + float leverage=1.0, + object end_daily=None, + object end_minute=None, + object end_date=None, + object exchange_symbol=None, + object first_traded=None, + object auto_close_date=None, + object exchange_full=None, + float min_trade_size=0.0001, + float max_trade_size=1000000, + float maker=0.0015, + float taker=0.0025, + float lot=0, + int decimals = 8, + int trading_state=1, + object data_source='catalyst', """ - #TODO: missing fields - super_dict = super(TradingPair, self).to_dict() - super_dict['end_daily'] = self.end_daily - super_dict['end_minute'] = self.end_minute - super_dict['leverage'] = self.leverage - super_dict['min_trade_size'] = self.min_trade_size - return super_dict + trading_pair_dict = dict( + symbol=self.symbol, + exchange=self.exchange, + start_date=self.start_date, + asset_name=self.asset_name, + leverage=self.leverage, + end_daily=self.end_daily, + end_minute=self.end_minute, + end_date=self.end_date, + exchange_symbol=self.exchange_symbol, + exchange_full=self.exchange_full, + min_trade_size=self.min_trade_size, + max_trade_size=self.max_trade_size, + maker=self.maker, + taker=self.taker, + lot=self.lot, + decimals=self.decimals, + trading_state=self.trading_state, + data_source=self.data_source, + ) + return trading_pair_dict def is_exchange_open(self, dt_minute): """ @@ -623,6 +660,13 @@ cdef class TradingPair(Asset): #TODO: make more dymanic to catch holds return True + def set_end_date(self, dt, data_frequency): + if data_frequency == 'minute': + self.end_minute = dt + + else: + self.end_daily = dt + cpdef __reduce__(self): """ Function used by pickle to determine how to serialize/deserialize this @@ -646,7 +690,9 @@ cdef class TradingPair(Asset): self.lot, self.decimals, self.taker, - self.maker)) + self.maker, + self.trading_state, + self.data_source)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..375dbe21 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' - +EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json' DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 3e5d5b29..9a5dad60 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,5 +1,3 @@ -import json -import os import re from collections import defaultdict @@ -20,6 +18,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ UnsupportedHistoryFrequencyError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ + MarketsNotFoundError, InvalidMarketError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ get_exchange_folder, get_catalyst_symbol, \ @@ -27,8 +27,16 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ get_epoch, \ get_periods_range +from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \ + get_epoch, get_catalyst_symbol, \ + get_exchange_auth, get_exchange_config from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction +from ccxt import InvalidOrder, NetworkError, \ + ExchangeError +from logbook import Logger +from redo import retry +from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -62,6 +70,8 @@ class CCXT(Exchange): 'secret': secret, }) self.api.enableRateLimit = True + self.has = self.api.has + self.fees = self.api.fees except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) @@ -69,6 +79,7 @@ class CCXT(Exchange): self._symbol_maps = [None, None] self.name = exchange_name + self.assets = [] self.base_currency = base_currency self.transactions = defaultdict(list) @@ -80,50 +91,115 @@ class CCXT(Exchange): self._common_symbols = dict() self.bundle = ExchangeBundle(self.name) - self.markets = None self._is_init = False + self._config = None def init(self): if self._is_init: return - exchange_folder = get_exchange_folder(self.name) - filename = os.path.join(exchange_folder, 'cctx_markets.json') - - if os.path.exists(filename): - timestamp = os.path.getmtime(filename) - dt = pd.to_datetime(timestamp, unit='s', utc=True) - - if dt >= pd.Timestamp.utcnow().floor('1D'): - with open(filename) as f: - self.markets = json.load(f) - - log.debug('loaded markets for {}'.format(self.name)) - - if self.markets is None: - try: - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) + if self._config is None: + self._config = get_exchange_config(self.name) + log.debug( + 'got exchange config {}:\n{}'.format( + self.name, self._config ) - - self.markets = self.api.fetch_markets() - with open(filename, 'w+') as f: - json.dump(self.markets, f, indent=4) - - except (ExchangeError, NetworkError) as e: - log.warn( - 'unable to fetch markets {}: {}'.format( - self.name, e - ) - ) - raise ExchangeRequestError(error=e) + ) self.load_assets() self._is_init = True + def load_assets(self): + if self._config is None: + raise ValueError('Exchange config not available.') + + self.assets = [] + for asset_dict in self._config['assets']: + asset = TradingPair(**asset_dict) + self.assets.append(asset) + + def _fetch_markets(self): + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + try: + markets = self.api.fetch_markets() + + except NetworkError as e: + raise ExchangeRequestError(error=e) + + if not markets: + raise MarketsNotFoundError( + exchange=self.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=self.name, + market=market, + ) + return markets + + def create_exchange_config(self): + config = dict( + name=self.name, + features=[feature for feature in self.has if self.has[feature]] + ) + markets = retry( + action=self._fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(self.name) + ), + ) + + config['assets'] = [] + for market in markets: + asset = self.create_trading_pair(market=market) + config['assets'].append(asset) + + return config + + def create_trading_pair(self, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=self.name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + self.apply_conditional_market_params(params, market) + + return TradingPair(**params) + @staticmethod def find_exchanges(features=None, is_authenticated=False): ccxt_features = [] @@ -202,42 +278,6 @@ class CCXT(Exchange): return frequencies - def get_market(self, symbol): - """ - The CCXT market. - - Parameters - ---------- - symbol: - The CCXT symbol. - - Returns - ------- - dict[str, Object] - - """ - s = self.get_symbol(symbol) - market = next( - (market for market in self.markets if market['symbol'] == s), - None, - ) - return market - - def substitute_currency_code(self, currency, source='catalyst'): - if source == 'catalyst': - currency = currency.upper() - - key = self.api.common_currency_code(currency) - self._common_symbols[key] = currency.lower() - return key - - else: - if currency in self._common_symbols: - return self._common_symbols[currency] - - else: - return currency.lower() - def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -485,144 +525,53 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def get_asset_defs(self, market): + def apply_conditional_market_params(self, params, market): """ - The local and Catalyst definitions of the specified market. + Applies a CCXT market dict to parameters of TradingPair init. Parameters ---------- - market: dict[str, Object] - The CCXT market dicts. + params: dict[Object] + market: dict[Object] Returns ------- - dict[str, Object] - The asset definition. """ - asset_defs = [] - - for is_local in (False, True): - asset_def = self.get_asset_def(market, is_local) - asset_defs.append((asset_def, is_local)) - - return asset_defs - - def get_asset_def(self, market, is_local=False): - """ - The asset definition (in symbols.json files) corresponding - to the the specified market. - - Parameters - ---------- - market: dict[str, Object] - The CCXT market dict. - is_local - Whether to search in local or Catalyst asset definitions. - - Returns - ------- - dict[str, Object] - The asset definition. - - """ - exchange_symbol = market['id'] - - symbol_map = self._fetch_symbol_map(is_local) - if symbol_map is not None: - assets_lower = {k.lower(): v for k, v in symbol_map.items()} - key = exchange_symbol.lower() - - asset = assets_lower[key] if key in assets_lower else None - if asset is not None: - return asset - - else: - return None + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 else: - return None + params['trading_state'] = 1 - def create_trading_pair(self, market, asset_def=None, is_local=False): - """ - Creating a TradingPair from market and asset data. + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] - Parameters - ---------- - market: dict[str, Object] - asset_def: dict[str, Object] - is_local: bool + if self.name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 - Returns - ------- - - """ - data_source = 'local' if is_local else 'catalyst' - params = dict( - exchange=self.name, - data_source=data_source, - exchange_symbol=market['id'], - ) - mixin_market_params(self.name, params, market) - - if asset_def is not None: - params['symbol'] = asset_def['symbol'] - - params['start_date'] = asset_def['start_date'] \ - if 'start_date' in asset_def else None - - params['end_date'] = asset_def['end_date'] \ - if 'end_date' in asset_def else None - - params['leverage'] = asset_def['leverage'] \ - if 'leverage' in asset_def else 1.0 - - params['asset_name'] = asset_def['asset_name'] \ - if 'asset_name' in asset_def else None - - params['end_daily'] = asset_def['end_daily'] \ - if 'end_daily' in asset_def \ - and asset_def['end_daily'] != 'N/A' else None - - params['end_minute'] = asset_def['end_minute'] \ - if 'end_minute' in asset_def \ - and asset_def['end_minute'] != 'N/A' else None + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] else: - params['symbol'] = get_catalyst_symbol(market) - # TODO: add as an optional column - params['leverage'] = 1.0 + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 - return TradingPair(**params) + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) - def load_assets(self): - log.debug('loading assets for {}'.format(self.name)) - self.assets = [] - - for market in self.markets: - if 'id' not in market: - log.warn('invalid market: {}'.format(market)) - continue - - asset_defs = self.get_asset_defs(market) - - asset = None - for asset_def in asset_defs: - if asset_def[0] is not None or not asset_defs[1]: - try: - asset = self.create_trading_pair( - market=market, - asset_def=asset_def[0], - is_local=asset_def[1] - ) - self.assets.append(asset) - - except TypeError as e: - log.warn('unable to add asset: {}'.format(e)) - - if asset is None: - asset = self.create_trading_pair(market=market) - self.assets.append(asset) + if 'lot' not in params: + params['lot'] = params['min_trade_size'] def get_balances(self): try: @@ -760,18 +709,14 @@ class CCXT(Exchange): side = 'buy' if amount > 0 else 'sell' if hasattr(self.api, 'amount_to_lots'): - # TODO: is this right? - if self.api.markets is None: - self.api.load_markets() - - # https://github.com/ccxt/ccxt/issues/1483 - adj_amount = abs(amount) - market = self.api.markets[symbol] - if 'lots' in market and market['lots'] > amount: - raise CreateOrderError( - exchange=self.name, - e='order amount lower than the smallest lot: {}'.format( - amount + adj_amount = self.api.amount_to_lots( + symbol=symbol, + amount=abs(amount), + ) + if adj_amount != abs(amount): + log.info( + 'adjusted order amount {} to {} based on lot size'.format( + abs(amount), adj_amount, ) ) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 38aed7c7..91ac07a3 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -5,6 +5,7 @@ from functools import partial from itertools import chain from operator import is_not +import copy import numpy as np import pandas as pd import pytz @@ -25,7 +26,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \ get_period_label, get_month_start_end, get_year_start_end from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params, get_catalyst_symbol + get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory from logbook import Logger @@ -700,42 +701,36 @@ class ExchangeBundle: for symbol in symbols: start_dt = df.index.get_level_values(1).min() end_dt = df.index.get_level_values(1).max() - end_dt_key = 'end_{}'.format(data_frequency) - market = self.exchange.get_market(symbol) - if market is None: - raise ValueError('symbol not available in the exchange.') + try: + asset = self.exchange.get_asset(symbol, is_local=True) + except: + asset = copy.deepcopy(self.exchange.get_asset(symbol)) - params = dict( - exchange=self.exchange.name, - data_source='local', - exchange_symbol=market['id'], - ) - mixin_market_params(self.exchange_name, params, market) + if asset.data_source == 'local': + asset.start_date = asset.start_date \ + if asset.start_date < start_dt else start_dt - asset_def = self.exchange.get_asset_def(market, True) - if asset_def is not None: - params['symbol'] = asset_def['symbol'] + if data_frequency == 'daily': + asset.end_date = asset.end_daily = asset.end_daily \ + if asset.end_daily > end_dt else end_dt - params['start_date'] = asset_def['start_date'] \ - if asset_def['start_date'] < start_dt else start_dt - - params['end_date'] = asset_def[end_dt_key] \ - if asset_def[end_dt_key] > end_dt else end_dt - - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else asset_def['end_daily'] - - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else asset_def['end_minute'] + else: + asset.end_date = asset.end_minute = asset.end_minute \ + if asset.end_minute > end_dt else end_dt else: - params['symbol'] = get_catalyst_symbol(market) + asset.data_source = 'local' + asset.start_date = start_dt + asset.end_dt = end_dt - params['end_daily'] = end_dt \ - if data_frequency == 'daily' else 'N/A' - params['end_minute'] = end_dt \ - if data_frequency == 'minute' else 'N/A' + if data_frequency == 'daily': + asset.end_daily = end_dt + asset.end_minute = None + + else: + asset.end_daily = None + asset.end_minute = end_dt if min_start_dt is None or start_dt < min_start_dt: min_start_dt = start_dt @@ -743,11 +738,9 @@ class ExchangeBundle: if max_end_dt is None or end_dt > max_end_dt: max_end_dt = end_dt - asset = TradingPair(**params) - assets[market['id']] = asset - - save_exchange_symbols(self.exchange_name, assets, True) + assets[symbol] = asset + # TODO: update config.json writer = self.get_writer( start_dt=min_start_dt.replace(hour=00, minute=00), end_dt=max_end_dt.replace(hour=23, minute=59), diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index d5af87c4..91fba0da 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -322,3 +322,17 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() + + +class MarketsNotFoundError(ZiplineError): + msg = ( + 'Exchange {exchange} contains no valid market so it is unusable in ' + 'Catalyst.' + ).strip() + + +class InvalidMarketError(ZiplineError): + msg = ( + 'Exchange {exchange} contains at least one incorrectly structured ' + 'market: {market}, so it is unusable in Catalyst.' + ).strip() diff --git a/tests/exchange/test_config.py b/tests/exchange/test_config.py new file mode 100644 index 00000000..f13f526b --- /dev/null +++ b/tests/exchange/test_config.py @@ -0,0 +1,8 @@ +from catalyst.exchange.utils.factory import get_exchange + + +class TestConfig: + def test_create_config(self): + exchange = get_exchange('binance', skip_init=True) + config = exchange.create_exchange_config() + pass From 2a239fd5bbf71fb189f1148c85591ae4f19b9126 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 27 Jan 2018 20:53:38 -0500 Subject: [PATCH 3/9] BLD: made some adjustment to generate and use the exchange config more efficiently and without mapping. Currently testing. --- catalyst/exchange/ccxt/ccxt_exchange.py | 217 +----------- catalyst/exchange/utils/ccxt_utils.py | 311 ++++++++++++++++++ catalyst/exchange/utils/factory.py | 32 +- .../exchange/utils/serialization_utils.py | 4 + .../test_suites/test_suite_exchange.py | 36 +- 5 files changed, 336 insertions(+), 264 deletions(-) create mode 100644 catalyst/exchange/utils/ccxt_utils.py diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 9a5dad60..c804b9cb 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -4,11 +4,6 @@ from collections import defaultdict import ccxt import pandas as pd import six -from ccxt import InvalidOrder, NetworkError, \ - ExchangeError -from logbook import Logger -from six import string_types - from catalyst.algorithm import MarketOrder from catalyst.assets._assets import TradingPair from catalyst.constants import LOG_LEVEL @@ -16,26 +11,18 @@ from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ - UnsupportedHistoryFrequencyError - ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ - MarketsNotFoundError, InvalidMarketError + UnsupportedHistoryFrequencyError, \ + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ - get_exchange_folder, get_catalyst_symbol, \ - get_exchange_auth +from catalyst.exchange.utils.ccxt_utils import get_exchange_config from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ get_epoch, \ get_periods_range -from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \ - get_epoch, get_catalyst_symbol, \ - get_exchange_auth, get_exchange_config from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction from ccxt import InvalidOrder, NetworkError, \ ExchangeError from logbook import Logger -from redo import retry from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -51,7 +38,7 @@ SUPPORTED_EXCHANGES = dict( class CCXT(Exchange): - def __init__(self, exchange_name, key, secret, base_currency): + def __init__(self, exchange_name, key, secret, base_currency, config=None): log.debug( 'finding {} in CCXT exchanges:\n{}'.format( exchange_name, ccxt.exchanges @@ -92,7 +79,7 @@ class CCXT(Exchange): self.bundle = ExchangeBundle(self.name) self._is_init = False - self._config = None + self._config = config def init(self): if self._is_init: @@ -118,136 +105,6 @@ class CCXT(Exchange): asset = TradingPair(**asset_dict) self.assets.append(asset) - def _fetch_markets(self): - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) - ) - try: - markets = self.api.fetch_markets() - - except NetworkError as e: - raise ExchangeRequestError(error=e) - - if not markets: - raise MarketsNotFoundError( - exchange=self.name, - ) - - for market in markets: - if 'id' not in market: - raise InvalidMarketError( - exchange=self.name, - market=market, - ) - return markets - - def create_exchange_config(self): - config = dict( - name=self.name, - features=[feature for feature in self.has if self.has[feature]] - ) - markets = retry( - action=self._fetch_markets, - attempts=5, - sleeptime=5, - retry_exceptions=(ExchangeRequestError,), - cleanup=lambda: log.warn( - 'fetching markets again for {}'.format(self.name) - ), - ) - - config['assets'] = [] - for market in markets: - asset = self.create_trading_pair(market=market) - config['assets'].append(asset) - - return config - - def create_trading_pair(self, market, start_dt=None, end_dt=None, - leverage=1, end_daily=None, end_minute=None): - """ - Creating a TradingPair from market and asset data. - - Parameters - ---------- - market: dict[str, Object] - start_dt - end_dt - leverage - end_daily - end_minute - - Returns - ------- - - """ - params = dict( - exchange=self.name, - data_source='catalyst', - exchange_symbol=market['id'], - symbol=get_catalyst_symbol(market), - start_date=start_dt, - end_date=end_dt, - leverage=leverage, - asset_name=market['symbol'], - end_daily=end_daily, - end_minute=end_minute, - ) - self.apply_conditional_market_params(params, market) - - return TradingPair(**params) - - @staticmethod - def find_exchanges(features=None, is_authenticated=False): - ccxt_features = [] - if features is not None: - for feature in features: - if not feature.endswith('Bundle'): - ccxt_features.append(feature) - - exchange_names = [] - for exchange_name in ccxt.exchanges: - if is_authenticated: - exchange_auth = get_exchange_auth(exchange_name) - - has_auth = (exchange_auth['key'] != '' - and exchange_auth['secret'] != '') - - if not has_auth: - continue - - log.debug('loading exchange: {}'.format(exchange_name)) - exchange = getattr(ccxt, exchange_name)() - - if ccxt_features is None: - has_feature = True - - else: - try: - has_feature = all( - [exchange.has[feature] for feature in ccxt_features] - ) - - except Exception: - has_feature = False - - if has_feature: - try: - log.info('initializing {}'.format(exchange_name)) - exchange_names.append(exchange_name) - - except Exception as e: - log.warn( - 'unable to initialize exchange {}: {}'.format( - exchange_name, e - ) - ) - - return exchange_names - def account(self): return None @@ -295,13 +152,7 @@ class CCXT(Exchange): if source == 'ccxt': if isinstance(asset_or_symbol, string_types): parts = asset_or_symbol.split('/') - base_currency = self.substitute_currency_code( - parts[0], source - ) - quote_currency = self.substitute_currency_code( - parts[1], source - ) - return '{}_{}'.format(base_currency, quote_currency) + return '{}_{}'.format(parts[0].lower(), parts[1].lower()) else: return asset_or_symbol.symbol @@ -312,13 +163,7 @@ class CCXT(Exchange): ) else asset_or_symbol.symbol parts = symbol.split('_') - base_currency = self.substitute_currency_code( - parts[0], source - ) - quote_currency = self.substitute_currency_code( - parts[1], source - ) - return '{}/{}'.format(base_currency, quote_currency) + return '{}/{}'.format(parts[0].upper(), parts[1].upper()) @staticmethod def map_frequency(value, source='ccxt', raise_error=True): @@ -525,54 +370,6 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def apply_conditional_market_params(self, params, market): - """ - Applies a CCXT market dict to parameters of TradingPair init. - - Parameters - ---------- - params: dict[Object] - market: dict[Object] - - Returns - ------- - - """ - # TODO: make this more externalized / configurable - # Consider representing in some type of JSON structure - if 'active' in market: - params['trading_state'] = 1 if market['active'] else 0 - - else: - params['trading_state'] = 1 - - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] - - if self.name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 - - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None \ - and market['taker'] is not None: - params['maker'] = market['maker'] - params['taker'] = market['taker'] - - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 - - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) - - if 'lot' not in params: - params['lot'] = params['min_trade_size'] - def get_balances(self): try: log.debug('retrieving wallets balances') diff --git a/catalyst/exchange/utils/ccxt_utils.py b/catalyst/exchange/utils/ccxt_utils.py new file mode 100644 index 00000000..5c0484fb --- /dev/null +++ b/catalyst/exchange/utils/ccxt_utils.py @@ -0,0 +1,311 @@ +import json +import os +import pandas as pd +from six.moves.urllib import request + +from catalyst.assets._assets import TradingPair +from ccxt import NetworkError +from catalyst.constants import LOG_LEVEL, EXCHANGE_CONFIG_URL +from catalyst.exchange.exchange_errors import MarketsNotFoundError, \ + InvalidMarketError +from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol, \ + get_exchange_folder, get_exchange_auth +from catalyst.exchange.utils.serialization_utils import ExchangeJSONDecoder, \ + ExchangeJSONEncoder +from logbook import Logger +from redo import retry +from ccxt.base.exchange import Exchange +from catalyst.utils.paths import last_modified_time, data_root, \ + ensure_directory +import ccxt + +log = Logger('ccxt_utils', level=LOG_LEVEL) + + +def find_exchange_configs(features=None, history=None, is_authenticated=False, + path=None): + """ + Finding exchanges from their config files + + Parameters + ---------- + features + is_authenticated + + Returns + ------- + + """ + exchange_config = [] + for exchange_name in ccxt.exchanges: + config = get_exchange_config(exchange_name, path) + if not config or 'error' in config: + log.info( + 'skipping invalid exchange {}'.format(exchange_name) + ) + + # Check if the exchange has an auth.json file + if is_authenticated: + exchange_auth = get_exchange_auth(exchange_name) + has_auth = (exchange_auth['key'] != '' + and exchange_auth['secret'] != '') + + if not has_auth: + continue + + if features is None: + has_features = True + + else: + try: + supported_features = [ + feature for feature in features if + feature in config['features'] + ] + has_features = len(supported_features) > 0 + except Exception: + has_features = False + + # TODO: filter by history + if has_features: + try: + exchange_config.append(config) + + except Exception as e: + log.warn( + 'unable to initialize exchange {}: {}'.format( + exchange_name, e + ) + ) + + return exchange_config + + +def get_exchange_config(exchange_name, path=None, environ=None, + expiry='2H'): + """ + The de-serialized content of the exchange's config.json. + Parameters + ---------- + exchange_name: str + The exchange name + filename: str + The target file + environ: + + Returns + ------- + config: dict[srt, Object] + The config dictionary. + + """ + try: + if path is None: + root = data_root(environ) + path = os.path.join(root, 'exchanges') + + folder = os.path.join(path, exchange_name) + ensure_directory(folder) + + filename = os.path.join(folder, 'config.json') + url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) + if os.path.isfile(filename): + # If the file exists, only update periodically to avoid + # unnecessary calls + now = pd.Timestamp.utcnow() + limit = pd.Timedelta(expiry) + if pd.Timedelta(now - last_modified_time(filename)) > limit: + request.urlretrieve(url=url, filename=filename) + + else: + request.urlretrieve(url=url, filename=filename) + + with open(filename) as data_file: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data + + except Exception as e: + log.warn( + 'unable to download {} config: {}'.format( + exchange_name, e + ) + ) + return dict(error=e) + + +def save_exchange_config(config, filename=None, environ=None): + """ + Save assets into an exchange_config file. + Parameters + ---------- + exchange_name: str + config + environ + Returns + ------- + """ + if filename is None: + name = 'config.json' + exchange_folder = get_exchange_folder(config['id'], environ) + filename = os.path.join(exchange_folder, name) + + with open(filename, 'w+') as handle: + json.dump(config, handle, indent=4, cls=ExchangeJSONEncoder) + + +def fetch_markets(ccxt_exchange): + """ + Fetches CCXT market objects. + + Parameters + ---------- + ccxt_exchange: Exchange + + Returns + ------- + + """ + markets_symbols = ccxt_exchange.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + ccxt_exchange.name, markets_symbols + ) + ) + markets = ccxt_exchange.fetch_markets() + + if not markets: + raise MarketsNotFoundError( + exchange=ccxt_exchange.name, + ) + + for market in markets: + if 'id' not in market: + raise InvalidMarketError( + exchange=ccxt_exchange.name, + market=market, + ) + return markets + + +def create_exchange_config(ccxt_exchange): + """ + Creates an exchange config structure. + + Parameters + ---------- + ccxt_exchange: Exchange + + Returns + ------- + + """ + exchange_name = ccxt_exchange.__class__.__name__ + config = dict( + id=exchange_name, + name=ccxt_exchange.name, + features=[ + feature for feature in ccxt_exchange.has if + ccxt_exchange.has[feature] + ] + ) + markets = retry( + action=fetch_markets, + attempts=5, + sleeptime=5, + retry_exceptions=(NetworkError,), + cleanup=lambda: log.warn( + 'fetching markets again for {}'.format(exchange_name) + ), + args=(ccxt_exchange,) + ) + + config['assets'] = [] + for market in markets: + asset = create_trading_pair(exchange_name, market) + config['assets'].append(asset) + + return config + + +def create_trading_pair(exchange_name, market, start_dt=None, end_dt=None, + leverage=1, end_daily=None, end_minute=None): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + start_dt + end_dt + leverage + end_daily + end_minute + + Returns + ------- + + """ + params = dict( + exchange=exchange_name, + data_source='catalyst', + exchange_symbol=market['id'], + symbol=get_catalyst_symbol(market), + start_date=start_dt, + end_date=end_dt, + leverage=leverage, + asset_name=market['symbol'], + end_daily=end_daily, + end_minute=end_minute, + ) + apply_conditional_market_params(exchange_name, params, market) + + return TradingPair(**params) + + +def apply_conditional_market_params(exchange_name, params, market): + """ + Applies a CCXT market dict to parameters of TradingPair init. + + Parameters + ---------- + params: dict[Object] + market: dict[Object] + + Returns + ------- + + """ + # TODO: make this more externalized / configurable + # Consider representing in some type of JSON structure + if 'active' in market: + params['trading_state'] = 1 if market['active'] else 0 + + else: + params['trading_state'] = 1 + + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] + + if exchange_name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 + + elif 'maker' in market and 'taker' in market \ + and market['maker'] is not None \ + and market['taker'] is not None: + params['maker'] = market['maker'] + params['taker'] = market['taker'] + + else: + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 + + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) + + if 'lot' not in params: + params['lot'] = params['min_trade_size'] diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py index 77b2d708..d4e5c450 100644 --- a/catalyst/exchange/utils/factory.py +++ b/catalyst/exchange/utils/factory.py @@ -4,8 +4,9 @@ from catalyst.constants import LOG_LEVEL from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_errors import ExchangeAuthEmpty +from catalyst.exchange.utils.ccxt_utils import find_exchange_configs from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \ - get_exchange_folder, is_blacklist + get_exchange_folder from logbook import Logger log = Logger('factory', level=LOG_LEVEL) @@ -13,7 +14,7 @@ exchange_cache = dict() def get_exchange(exchange_name, base_currency=None, must_authenticate=False, - skip_init=False, auth_alias=None): + skip_init=False, auth_alias=None, config=None): key = (exchange_name, base_currency) if key in exchange_cache: return exchange_cache[key] @@ -34,6 +35,7 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False, key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, + config=config, ) exchange_cache[key] = exchange @@ -51,8 +53,8 @@ def get_exchanges(exchange_names): return exchanges -def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, - base_currency=None): +def find_exchanges(features=None, history=None, skip_blacklist=True, path=None, + is_authenticated=False, base_currency=None): """ Find exchanges filtered by a list of feature. @@ -70,28 +72,20 @@ def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, list[Exchange] """ - exchange_names = CCXT.find_exchanges(features, is_authenticated) - + exchange_configs = find_exchange_configs( + features, history, is_authenticated, path + ) exchanges = [] - for exchange_name in exchange_names: - if skip_blacklist and is_blacklist(exchange_name): + for config in exchange_configs: + if skip_blacklist and (config is None or 'error' in config): continue exchange = get_exchange( - exchange_name=exchange_name, + exchange_name=config['id'], skip_init=True, base_currency=base_currency, + config=config, ) - - if features is not None: - if 'dailyBundle' in features \ - and not exchange.has_bundle('daily'): - continue - - elif 'minuteBundle' in features \ - and not exchange.has_bundle('minute'): - continue - exchanges.append(exchange) return exchanges diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 62ab74d5..6e1d21e3 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -3,6 +3,7 @@ import re from json import JSONEncoder import pandas as pd +from catalyst.assets._assets import TradingPair from catalyst.constants import DATE_TIME_FORMAT from six import string_types @@ -12,6 +13,9 @@ class ExchangeJSONEncoder(json.JSONEncoder): if isinstance(obj, pd.Timestamp): return obj.strftime(DATE_TIME_FORMAT) + elif isinstance(obj, TradingPair): + return obj.to_dict() + # Let the base class default method raise the TypeError return JSONEncoder.default(self, obj) diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index 4088a675..ca4357a5 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -21,47 +21,13 @@ log = Logger('TestSuiteExchange') class TestSuiteExchange(WithLogger, ZiplineTestCase): - def _test_markets_exchange(self, exchange, attempts=0): - assets = None - try: - exchange.init() - - # Verify that the assets and markets are populated - if not exchange.markets: - raise ValueError( - 'no markets found' - ) - if not exchange.assets: - raise ValueError( - 'no assets derived from markets' - ) - assets = exchange.assets - - except ExchangeRequestError as e: - sleep(5) - - if attempts > 5: - handle_exchange_error(exchange, e) - - else: - print( - 're-trying an exchange request {} {}'.format( - exchange.name, attempts - ) - ) - self._test_markets_exchange(exchange, attempts + 1) - - except Exception as e: - handle_exchange_error(exchange, e) - - return assets - def test_markets(self): population = 3 results = dict() exchanges = select_random_exchanges(population) # Type: list[Exchange] for exchange in exchanges: + exchange.init() assets = self._test_markets_exchange(exchange) if assets is not None: From 2225c40b76c5c49fdb2fa7c5876caf954c3151cf Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 22:14:44 -0500 Subject: [PATCH 4/9] BLD: using an iterable to yield exchanges instead of populating a list --- catalyst/exchange/utils/ccxt_utils.py | 15 ++----------- catalyst/exchange/utils/factory.py | 31 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/catalyst/exchange/utils/ccxt_utils.py b/catalyst/exchange/utils/ccxt_utils.py index 5c0484fb..cbbf2220 100644 --- a/catalyst/exchange/utils/ccxt_utils.py +++ b/catalyst/exchange/utils/ccxt_utils.py @@ -22,7 +22,7 @@ import ccxt log = Logger('ccxt_utils', level=LOG_LEVEL) -def find_exchange_configs(features=None, history=None, is_authenticated=False, +def scan_exchange_configs(features=None, history=None, is_authenticated=False, path=None): """ Finding exchanges from their config files @@ -36,7 +36,6 @@ def find_exchange_configs(features=None, history=None, is_authenticated=False, ------- """ - exchange_config = [] for exchange_name in ccxt.exchanges: config = get_exchange_config(exchange_name, path) if not config or 'error' in config: @@ -68,17 +67,7 @@ def find_exchange_configs(features=None, history=None, is_authenticated=False, # TODO: filter by history if has_features: - try: - exchange_config.append(config) - - except Exception as e: - log.warn( - 'unable to initialize exchange {}: {}'.format( - exchange_name, e - ) - ) - - return exchange_config + yield config def get_exchange_config(exchange_name, path=None, environ=None, diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py index d4e5c450..dc528135 100644 --- a/catalyst/exchange/utils/factory.py +++ b/catalyst/exchange/utils/factory.py @@ -4,7 +4,7 @@ from catalyst.constants import LOG_LEVEL from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_errors import ExchangeAuthEmpty -from catalyst.exchange.utils.ccxt_utils import find_exchange_configs +from catalyst.exchange.utils.ccxt_utils import scan_exchange_configs from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \ get_exchange_folder from logbook import Logger @@ -72,20 +72,33 @@ def find_exchanges(features=None, history=None, skip_blacklist=True, path=None, list[Exchange] """ - exchange_configs = find_exchange_configs( - features, history, is_authenticated, path + + return list( + scan_exchanges( + features, + history, + skip_blacklist, + path, + is_authenticated, + base_currency + ) ) - exchanges = [] - for config in exchange_configs: + + +def scan_exchanges(features=None, history=None, skip_blacklist=True, path=None, + is_authenticated=False, base_currency=None): + for config in scan_exchange_configs( + features=features, + history=history, + is_authenticated=is_authenticated, + path=path, + ): if skip_blacklist and (config is None or 'error' in config): continue - exchange = get_exchange( + yield get_exchange( exchange_name=config['id'], skip_init=True, base_currency=base_currency, config=config, ) - exchanges.append(exchange) - - return exchanges From 48a89ad521fc32dc195913c51db83babff371f5a Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 23:14:37 -0500 Subject: [PATCH 5/9] BLD: created a report with the start and end time of all collected candles --- catalyst/exchange/utils/ccxt_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/catalyst/exchange/utils/ccxt_utils.py b/catalyst/exchange/utils/ccxt_utils.py index cbbf2220..964f966c 100644 --- a/catalyst/exchange/utils/ccxt_utils.py +++ b/catalyst/exchange/utils/ccxt_utils.py @@ -104,7 +104,14 @@ def get_exchange_config(exchange_name, path=None, environ=None, now = pd.Timestamp.utcnow() limit = pd.Timedelta(expiry) if pd.Timedelta(now - last_modified_time(filename)) > limit: - request.urlretrieve(url=url, filename=filename) + try: + request.urlretrieve(url=url, filename=filename) + except Exception as e: + log.warn( + 'unable to update config {} => {}: {}'.format( + url, filename, e + ) + ) else: request.urlretrieve(url=url, filename=filename) From 62d21f1aca1bcb288c030a27455c66e8df324ec6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 29 Jan 2018 23:17:09 -0500 Subject: [PATCH 6/9] BLD: updated CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 24d0bfa1..80b0dbc9 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.774 + - ccxt==1.10.909 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index 8138f6c3..f014ed33 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.837 +ccxt==1.10.909 boto3==1.4.8 redo==1.6 From 1aed7c71f6d9787aa0fbcb0f3d5a9e6f922ff1cc Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 12 Feb 2018 12:46:09 -0500 Subject: [PATCH 7/9] working on trades collector --- catalyst/exchange/exchange.py | 2 ++ etc/requirements.txt | 2 +- tests/exchange/test_suites/test_suite_exchange.py | 7 +++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 84cac446..bab87d3c 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -5,6 +5,7 @@ from time import sleep import numpy as np import pandas as pd + from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -20,6 +21,7 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ resample_history_df, has_bundle from logbook import Logger + log = Logger('Exchange', level=LOG_LEVEL) diff --git a/etc/requirements.txt b/etc/requirements.txt index f014ed33..9f48e0e2 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.909 +ccxt==1.10.1069 boto3==1.4.8 redo==1.6 diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index ca4357a5..3015cf1e 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -5,17 +5,16 @@ from logging import Logger, WARNING from time import sleep import pandas as pd -from catalyst.assets._assets import TradingPair from logbook import TestHandler -from catalyst.exchange.exchange_errors import ExchangeRequestError +from catalyst.assets._assets import TradingPair from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import get_exchange_folder +from catalyst.exchange.utils.factory import get_exchanges from catalyst.exchange.utils.test_utils import select_random_exchanges, \ - handle_exchange_error, select_random_assets + select_random_assets from catalyst.testing import ZiplineTestCase from catalyst.testing.fixtures import WithLogger -from exchange.utils.factory import get_exchanges log = Logger('TestSuiteExchange') From 69153295f0c0f260c92e04b8434d372016d21c68 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 20 Feb 2018 15:47:08 -0500 Subject: [PATCH 8/9] BUG: for issue #237, checking considering open orders when verifying the exchange balance for each positions --- catalyst/exchange/exchange.py | 12 ++++++++---- catalyst/exchange/exchange_algorithm.py | 3 +-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index bab87d3c..2f3ce3ed 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -21,7 +21,6 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ resample_history_df, has_bundle from logbook import Logger - log = Logger('Exchange', level=LOG_LEVEL) @@ -657,16 +656,21 @@ class Exchange: return df - def _check_low_balance(self, currency, balances, amount): + def _check_low_balance(self, currency, balances, amount, open_orders=None): free = balances[currency]['free'] if currency in balances else 0.0 + if open_orders: + # TODO: make sure that this works + free += sum([order.amount for order in open_orders]) + if free < amount: return free, True else: return free, False - def sync_positions(self, positions, cash=None, check_balances=False): + def sync_positions(self, positions, open_orders=None, cash=None, + check_balances=False): """ Update the portfolio cash and position balances based on the latest ticker prices. @@ -695,7 +699,7 @@ class Exchange: balances=balances, amount=cash, ) - if is_lower: + if is_lower and not open_orders: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 5c83a74a..dc1f18e1 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -591,8 +591,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if base_currency is None: base_currency = exchange.base_currency - # Don't check the cash if there are open orders. This could - # results in false positives. orders = [] for asset in self.blotter.open_orders: asset_orders = self.blotter.open_orders[asset] @@ -602,6 +600,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): required_cash = self.portfolio.cash if not orders else None cash, positions_value = exchange.sync_positions( positions=exchange_positions, + open_orders=orders, check_balances=check_balances, cash=required_cash, ) From 271a51a3932581f288c66c9c422562f9333a363e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 21 Feb 2018 12:52:31 -0500 Subject: [PATCH 9/9] BLD: adjusting candle computation --- catalyst/examples/mean_reversion_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 6909dbc4..d2652f8c 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -248,7 +248,7 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=100, + capital_base=0.03, initialize=initialize, handle_data=handle_data, analyze=analyze, @@ -257,7 +257,7 @@ if __name__ == '__main__': algo_namespace=NAMESPACE, base_currency='btc', live_graph=False, - simulate_orders=True, + simulate_orders=False, stats_output=None, # auth_aliases=dict(poloniex='auth2') )