BLD: replacing symbols.json and fetching markets with a single config

This commit is contained in:
Frederic Fortier
2018-01-06 17:28:32 -05:00
parent fa60457a0f
commit 6b3f59ff76
6 changed files with 254 additions and 248 deletions
+56 -10
View File
@@ -433,7 +433,7 @@ cdef class TradingPair(Asset):
'taker',
'trading_state',
'data_source',
'decimals'
'decimals',
})
def __init__(self,
object symbol,
@@ -455,7 +455,7 @@ cdef class TradingPair(Asset):
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=0,
int trading_state=1,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
@@ -600,14 +600,51 @@ cdef class TradingPair(Asset):
cpdef to_dict(self):
"""
Convert to a python dict.
Repeat constructor params:
object symbol,
object exchange,
object start_date=None,
object asset_name=None,
int sid=0,
float leverage=1.0,
object end_daily=None,
object end_minute=None,
object end_date=None,
object exchange_symbol=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
object data_source='catalyst',
"""
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
super_dict['leverage'] = self.leverage
super_dict['min_trade_size'] = self.min_trade_size
return super_dict
trading_pair_dict = dict(
symbol=self.symbol,
exchange=self.exchange,
start_date=self.start_date,
asset_name=self.asset_name,
leverage=self.leverage,
end_daily=self.end_daily,
end_minute=self.end_minute,
end_date=self.end_date,
exchange_symbol=self.exchange_symbol,
exchange_full=self.exchange_full,
min_trade_size=self.min_trade_size,
max_trade_size=self.max_trade_size,
maker=self.maker,
taker=self.taker,
lot=self.lot,
decimals=self.decimals,
trading_state=self.trading_state,
data_source=self.data_source,
)
return trading_pair_dict
def is_exchange_open(self, dt_minute):
"""
@@ -623,6 +660,13 @@ cdef class TradingPair(Asset):
#TODO: make more dymanic to catch holds
return True
def set_end_date(self, dt, data_frequency):
if data_frequency == 'minute':
self.end_minute = dt
else:
self.end_daily = dt
cpdef __reduce__(self):
"""
Function used by pickle to determine how to serialize/deserialize this
@@ -646,7 +690,9 @@ cdef class TradingPair(Asset):
self.lot,
self.decimals,
self.taker,
self.maker))
self.maker,
self.trading_state,
self.data_source))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
+1 -1
View File
@@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json'
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
+148 -203
View File
@@ -1,5 +1,3 @@
import json
import os
import re
from collections import defaultdict
@@ -20,6 +18,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
UnsupportedHistoryFrequencyError
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
MarketsNotFoundError, InvalidMarketError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
get_exchange_folder, get_catalyst_symbol, \
@@ -27,8 +27,16 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
get_epoch, \
get_periods_range
from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \
get_epoch, get_catalyst_symbol, \
get_exchange_auth, get_exchange_config
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from redo import retry
from six import string_types
log = Logger('CCXT', level=LOG_LEVEL)
@@ -62,6 +70,8 @@ class CCXT(Exchange):
'secret': secret,
})
self.api.enableRateLimit = True
self.has = self.api.has
self.fees = self.api.fees
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
@@ -69,6 +79,7 @@ class CCXT(Exchange):
self._symbol_maps = [None, None]
self.name = exchange_name
self.assets = []
self.base_currency = base_currency
self.transactions = defaultdict(list)
@@ -80,50 +91,115 @@ class CCXT(Exchange):
self._common_symbols = dict()
self.bundle = ExchangeBundle(self.name)
self.markets = None
self._is_init = False
self._config = None
def init(self):
if self._is_init:
return
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
if self._config is None:
self._config = get_exchange_config(self.name)
log.debug(
'got exchange config {}:\n{}'.format(
self.name, self._config
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
)
self.load_assets()
self._is_init = True
def load_assets(self):
if self._config is None:
raise ValueError('Exchange config not available.')
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
def _fetch_markets(self):
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
try:
markets = self.api.fetch_markets()
except NetworkError as e:
raise ExchangeRequestError(error=e)
if not markets:
raise MarketsNotFoundError(
exchange=self.name,
)
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=self.name,
market=market,
)
return markets
def create_exchange_config(self):
config = dict(
name=self.name,
features=[feature for feature in self.has if self.has[feature]]
)
markets = retry(
action=self._fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(self.name)
),
)
config['assets'] = []
for market in markets:
asset = self.create_trading_pair(market=market)
config['assets'].append(asset)
return config
def create_trading_pair(self, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=self.name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
self.apply_conditional_market_params(params, market)
return TradingPair(**params)
@staticmethod
def find_exchanges(features=None, is_authenticated=False):
ccxt_features = []
@@ -202,42 +278,6 @@ class CCXT(Exchange):
return frequencies
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def substitute_currency_code(self, currency, source='catalyst'):
if source == 'catalyst':
currency = currency.upper()
key = self.api.common_currency_code(currency)
self._common_symbols[key] = currency.lower()
return key
else:
if currency in self._common_symbols:
return self._common_symbols[currency]
else:
return currency.lower()
def get_symbol(self, asset_or_symbol, source='catalyst'):
"""
The CCXT symbol.
@@ -485,144 +525,53 @@ class CCXT(Exchange):
except ExchangeSymbolsNotFound:
return None
def get_asset_defs(self, market):
def apply_conditional_market_params(self, params, market):
"""
The local and Catalyst definitions of the specified market.
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
market: dict[str, Object]
The CCXT market dicts.
params: dict[Object]
market: dict[Object]
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
else:
return None
params['trading_state'] = 1
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
if self.name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
Returns
-------
"""
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
params['symbol'] = get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
return TradingPair(**params)
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
def load_assets(self):
log.debug('loading assets for {}'.format(self.name))
self.assets = []
for market in self.markets:
if 'id' not in market:
log.warn('invalid market: {}'.format(market))
continue
asset_defs = self.get_asset_defs(market)
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def get_balances(self):
try:
@@ -760,18 +709,14 @@ class CCXT(Exchange):
side = 'buy' if amount > 0 else 'sell'
if hasattr(self.api, 'amount_to_lots'):
# TODO: is this right?
if self.api.markets is None:
self.api.load_markets()
# https://github.com/ccxt/ccxt/issues/1483
adj_amount = abs(amount)
market = self.api.markets[symbol]
if 'lots' in market and market['lots'] > amount:
raise CreateOrderError(
exchange=self.name,
e='order amount lower than the smallest lot: {}'.format(
amount
adj_amount = self.api.amount_to_lots(
symbol=symbol,
amount=abs(amount),
)
if adj_amount != abs(amount):
log.info(
'adjusted order amount {} to {} based on lot size'.format(
abs(amount), adj_amount,
)
)
+27 -34
View File
@@ -5,6 +5,7 @@ from functools import partial
from itertools import chain
from operator import is_not
import copy
import numpy as np
import pandas as pd
import pytz
@@ -25,7 +26,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
get_period_label, get_month_start_end, get_year_start_end
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
from logbook import Logger
@@ -700,42 +701,36 @@ class ExchangeBundle:
for symbol in symbols:
start_dt = df.index.get_level_values(1).min()
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
try:
asset = self.exchange.get_asset(symbol, is_local=True)
except:
asset = copy.deepcopy(self.exchange.get_asset(symbol))
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
if asset.data_source == 'local':
asset.start_date = asset.start_date \
if asset.start_date < start_dt else start_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
if data_frequency == 'daily':
asset.end_date = asset.end_daily = asset.end_daily \
if asset.end_daily > end_dt else end_dt
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
asset.end_date = asset.end_minute = asset.end_minute \
if asset.end_minute > end_dt else end_dt
else:
params['symbol'] = get_catalyst_symbol(market)
asset.data_source = 'local'
asset.start_date = start_dt
asset.end_dt = end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if data_frequency == 'daily':
asset.end_daily = end_dt
asset.end_minute = None
else:
asset.end_daily = None
asset.end_minute = end_dt
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -743,11 +738,9 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
assets[symbol] = asset
# TODO: update config.json
writer = self.get_writer(
start_dt=min_start_dt.replace(hour=00, minute=00),
end_dt=max_end_dt.replace(hour=23, minute=59),
+14
View File
@@ -322,3 +322,17 @@ class BalanceTooLowError(ZiplineError):
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class MarketsNotFoundError(ZiplineError):
msg = (
'Exchange {exchange} contains no valid market so it is unusable in '
'Catalyst.'
).strip()
class InvalidMarketError(ZiplineError):
msg = (
'Exchange {exchange} contains at least one incorrectly structured '
'market: {market}, so it is unusable in Catalyst.'
).strip()
+8
View File
@@ -0,0 +1,8 @@
from catalyst.exchange.utils.factory import get_exchange
class TestConfig:
def test_create_config(self):
exchange = get_exchange('binance', skip_init=True)
config = exchange.create_exchange_config()
pass