wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-18 12:20:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

BLD: replacing symbols.json and fetching markets with a single config

Browse Source
This commit is contained in:
Frederic Fortier
2018-01-06 17:28:32 -05:00
parent 13de3e69ef
commit e872b1fc82
10 changed files with 323 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/assets/_assets.pyx
+56 -10
View File
@@ -433,7 +433,7 @@ cdef class TradingPair(Asset):
'taker',
'trading_state',
'data_source',
'decimals'
'decimals',
})
def __init__(self,
object symbol,
@@ -455,7 +455,7 @@ cdef class TradingPair(Asset):
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=0,
int trading_state=1,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
@@ -600,14 +600,51 @@ cdef class TradingPair(Asset):
cpdef to_dict(self):
"""
Convert to a python dict.
Repeat constructor params:
object symbol,
object exchange,
object start_date=None,
object asset_name=None,
int sid=0,
float leverage=1.0,
object end_daily=None,
object end_minute=None,
object end_date=None,
object exchange_symbol=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
object data_source='catalyst',
"""
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
super_dict['leverage'] = self.leverage
super_dict['min_trade_size'] = self.min_trade_size
return super_dict
trading_pair_dict = dict(
symbol=self.symbol,
exchange=self.exchange,
start_date=self.start_date,
asset_name=self.asset_name,
leverage=self.leverage,
end_daily=self.end_daily,
end_minute=self.end_minute,
end_date=self.end_date,
exchange_symbol=self.exchange_symbol,
exchange_full=self.exchange_full,
min_trade_size=self.min_trade_size,
max_trade_size=self.max_trade_size,
maker=self.maker,
taker=self.taker,
lot=self.lot,
decimals=self.decimals,
trading_state=self.trading_state,
data_source=self.data_source,
)
return trading_pair_dict
def is_exchange_open(self, dt_minute):
"""
@@ -623,6 +660,13 @@ cdef class TradingPair(Asset):
#TODO: make more dymanic to catch holds
return True
def set_end_date(self, dt, data_frequency):
if data_frequency == 'minute':
self.end_minute = dt
else:
self.end_daily = dt
cpdef __reduce__(self):
"""
Function used by pickle to determine how to serialize/deserialize this
@@ -646,7 +690,9 @@ cdef class TradingPair(Asset):
self.lot,
self.decimals,
self.taker,
self.maker))
self.maker,
self.trading_state,
self.data_source))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/constants.py
+1 -1
View File
@@ -11,7 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
EXCHANGE_CONFIG_URL = 'http://127.0.0.1:8080/exchanges/{exchange}/config.json'
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
catalyst/exchange/ccxt/ccxt_exchange.py
+136 -180
View File
@@ -1,5 +1,3 @@
import json
import os
import re
from collections import defaultdict
@@ -13,15 +11,17 @@ from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
MarketsNotFoundError, InvalidMarketError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
get_exchange_auth
from catalyst.exchange.utils.exchange_utils import from_ms_timestamp, \
get_epoch, get_catalyst_symbol, \
get_exchange_auth, get_exchange_config
from catalyst.finance.order import Order, ORDER_STATUS
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from redo import retry
from six import string_types
log = Logger('CCXT', level=LOG_LEVEL)
@@ -55,6 +55,8 @@ class CCXT(Exchange):
'apiKey': key,
'secret': secret,
})
self.has = self.api.has
self.fees = self.api.fees
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
@@ -62,6 +64,7 @@ class CCXT(Exchange):
self._symbol_maps = [None, None]
self.name = exchange_name
self.assets = []
self.base_currency = base_currency
self.transactions = defaultdict(list)
@@ -72,50 +75,115 @@ class CCXT(Exchange):
self.request_cpt = dict()
self.bundle = ExchangeBundle(self.name)
self.markets = None
self._is_init = False
self._config = None
def init(self):
if self._is_init:
return
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
if self._config is None:
self._config = get_exchange_config(self.name)
log.debug(
'got exchange config {}:\n{}'.format(
self.name, self._config
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
)
self.load_assets()
self._is_init = True
def load_assets(self):
if self._config is None:
raise ValueError('Exchange config not available.')
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
def _fetch_markets(self):
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
try:
markets = self.api.fetch_markets()
except NetworkError as e:
raise ExchangeRequestError(error=e)
if not markets:
raise MarketsNotFoundError(
exchange=self.name,
)
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=self.name,
market=market,
)
return markets
def create_exchange_config(self):
config = dict(
name=self.name,
features=[feature for feature in self.has if self.has[feature]]
)
markets = retry(
action=self._fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(self.name)
),
)
config['assets'] = []
for market in markets:
asset = self.create_trading_pair(market=market)
config['assets'].append(asset)
return config
def create_trading_pair(self, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=self.name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
self.apply_conditional_market_params(params, market)
return TradingPair(**params)
@staticmethod
def find_exchanges(features=None, is_authenticated=False):
ccxt_features = []
@@ -194,27 +262,6 @@ class CCXT(Exchange):
return frequencies
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def get_symbol(self, asset_or_symbol, source='catalyst'):
"""
The CCXT symbol.
@@ -417,144 +464,53 @@ class CCXT(Exchange):
except ExchangeSymbolsNotFound:
return None
def get_asset_defs(self, market):
def apply_conditional_market_params(self, params, market):
"""
The local and Catalyst definitions of the specified market.
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
market: dict[str, Object]
The CCXT market dicts.
params: dict[Object]
market: dict[Object]
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
else:
return None
params['trading_state'] = 1
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
if self.name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
Returns
-------
"""
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
params['symbol'] = get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
return TradingPair(**params)
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
def load_assets(self):
log.debug('loading assets for {}'.format(self.name))
self.assets = []
for market in self.markets:
if 'id' not in market:
log.warn('invalid market: {}'.format(market))
continue
asset_defs = self.get_asset_defs(market)
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def get_balances(self):
try:
catalyst/exchange/exchange.py
+6 -28
View File
@@ -5,8 +5,6 @@ from time import sleep
import numpy as np
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -17,8 +15,9 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
TickerNotFoundError, NotEnoughCashError
from catalyst.exchange.utils.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
from catalyst.exchange.utils.exchange_utils import \
get_frequency, resample_history_df, has_bundle
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL)
@@ -284,16 +283,6 @@ class Exchange:
log.debug('found asset: {}'.format(asset))
return asset
def fetch_symbol_map(self, is_local=False):
index = 1 if is_local else 0
if self._symbol_maps[index] is not None:
return self._symbol_maps[index]
else:
symbol_map = get_exchange_symbols(self.name, is_local)
self._symbol_maps[index] = symbol_map
return symbol_map
@abstractmethod
def init(self):
"""
@@ -305,24 +294,13 @@ class Exchange:
"""
@abstractmethod
def load_assets(self, is_local=False):
def create_exchange_config(self):
"""
Populate the 'assets' attribute with a dictionary of Assets.
The key of the resulting dictionary is the exchange specific
currency pair symbol. The universal symbol is contained in the
'symbol' attribute of each asset.
Notes
-----
The sid of each asset is calculated based on a numeric hash of the
universal symbol. This simple approach avoids maintaining a mapping
of sids.
This method can be omerridden if an exchange offers equivalent data
via its api.
Fetch the exchange market data and generate a config object
Returns
-------
"""
pass
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
"""
catalyst/exchange/exchange_bundle.py
+41 -48
View File
@@ -5,6 +5,7 @@ from functools import partial
from itertools import chain
from operator import is_not
import copy
import numpy as np
import pandas as pd
import pytz
@@ -29,7 +30,7 @@ from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
get_delta, get_assets
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
@@ -233,12 +234,12 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has empty ' \
'periods: {dates}'.format(
name=asset.symbol,
start_dt=asset.start_date.strftime(
DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
dates=[date.strftime(
DATE_TIME_FORMAT) for date in dates])
name=asset.symbol,
start_dt=asset.start_date.strftime(
DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
dates=[date.strftime(
DATE_TIME_FORMAT) for date in dates])
if empty_rows_behavior == 'warn':
log.warn(problem)
@@ -287,12 +288,12 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
'identical close values on: {dates}'.format(
name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
threshold=threshold,
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
for date in dates])
name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
threshold=threshold,
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
for date in dates])
problems.append(problem)
@@ -630,8 +631,8 @@ class ExchangeBundle:
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
for chunk in it:
problems += self.ingest_ctable(
@@ -701,42 +702,36 @@ class ExchangeBundle:
for symbol in symbols:
start_dt = df.index.get_level_values(1).min()
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
try:
asset = self.exchange.get_asset(symbol, is_local=True)
except:
asset = copy.deepcopy(self.exchange.get_asset(symbol))
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
if asset.data_source == 'local':
asset.start_date = asset.start_date \
if asset.start_date < start_dt else start_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
if data_frequency == 'daily':
asset.end_date = asset.end_daily = asset.end_daily \
if asset.end_daily > end_dt else end_dt
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
asset.end_date = asset.end_minute = asset.end_minute \
if asset.end_minute > end_dt else end_dt
else:
params['symbol'] = get_catalyst_symbol(market)
asset.data_source = 'local'
asset.start_date = start_dt
asset.end_dt = end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if data_frequency == 'daily':
asset.end_daily = end_dt
asset.end_minute = None
else:
asset.end_daily = None
asset.end_minute = end_dt
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -744,11 +739,9 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
assets[symbol] = asset
# TODO: update config.json
writer = self.get_writer(
start_dt=min_start_dt.replace(hour=00, minute=00),
end_dt=max_end_dt.replace(hour=23, minute=59),
catalyst/exchange/exchange_errors.py
+14
View File
@@ -315,3 +315,17 @@ class BalanceTooLowError(ZiplineError):
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class MarketsNotFoundError(ZiplineError):
msg = (
'Exchange {exchange} contains no valid market so it is unusable in '
'Catalyst.'
).strip()
class InvalidMarketError(ZiplineError):
msg = (
'Exchange {exchange} contains at least one incorrectly structured '
'market: {market}, so it is unusable in Catalyst.'
).strip()
catalyst/exchange/utils/exchange_utils.py
+36 -104
View File
@@ -8,16 +8,15 @@ from datetime import date, datetime
import pandas as pd
from catalyst.assets._assets import TradingPair
from six import string_types
from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
from catalyst.constants import EXCHANGE_CONFIG_URL
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
InvalidHistoryFrequencyAlias
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
ExchangeJSONDecoder
ExchangeJSONDecoder, ConfigJSONEncoder
from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time
from six import string_types
from six.moves.urllib import request
def get_sid(symbol):
@@ -71,7 +70,7 @@ def is_blacklist(exchange_name, environ=None):
return os.path.exists(filename)
def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
def get_exchange_config_filename(exchange_name, environ=None):
"""
The absolute path of the exchange's symbol.json file.
@@ -85,12 +84,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
str
"""
name = 'symbols.json' if not is_local else 'symbols_local.json'
name = 'config.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
return os.path.join(exchange_folder, name)
def download_exchange_symbols(exchange_name, environ=None):
def download_exchange_config(exchange_name, filename, environ=None):
"""
Downloads the exchange's symbols.json from the repository.
@@ -104,15 +103,13 @@ def download_exchange_symbols(exchange_name, environ=None):
str
"""
filename = get_exchange_symbols_filename(exchange_name)
url = SYMBOLS_URL.format(exchange=exchange_name)
response = request.urlretrieve(url=url, filename=filename)
return response
url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
request.urlretrieve(url=url, filename=filename)
def get_exchange_symbols(exchange_name, is_local=False, environ=None):
def get_exchange_config(exchange_name, filename=None, environ=None):
"""
The de-serialized content of the exchange's symbols.json.
The de-serialized content of the exchange's config.json.
Parameters
----------
@@ -125,52 +122,47 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
Object
"""
filename = get_exchange_symbols_filename(exchange_name, is_local)
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
download_exchange_symbols(exchange_name, environ)
if filename is None:
filename = get_exchange_config_filename(exchange_name)
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
now = pd.Timestamp.utcnow()
limit = pd.Timedelta('2H')
if pd.Timedelta(now - last_modified_time(filename)) > limit:
download_exchange_config(exchange_name, filename, environ)
except ValueError:
return dict()
else:
raise ExchangeSymbolsNotFound(
exchange=exchange_name,
filename=filename
)
download_exchange_config(exchange_name, filename, environ)
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
except ValueError:
return dict()
def save_exchange_config(exchange_name, config, filename=None, environ=None):
"""
Save assets into an exchange_symbols file.
Save assets into an exchange_config file.
Parameters
----------
exchange_name: str
assets: list[dict[str, object]]
is_local: bool
config
environ
Returns
-------
"""
asset_dicts = dict()
for symbol in assets:
asset_dicts[symbol] = assets[symbol].to_dict()
if filename is None:
name = 'config.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, name)
filename = get_exchange_symbols_filename(
exchange_name, is_local, environ
)
with open(filename, 'wt') as handle:
json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
with open(filename, 'w+') as handle:
json.dump(config, handle, indent=4, cls=ConfigJSONEncoder)
def get_symbols_string(assets):
@@ -443,25 +435,6 @@ def has_bundle(exchange_name, data_frequency, environ=None):
return os.path.isdir(folder)
def symbols_serial(obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
raise TypeError("Type %s not serializable" % type(obj))
def perf_serial(obj):
"""
JSON serializer for objects not serializable by default json code
@@ -608,47 +581,6 @@ def resample_history_df(df, freq, field):
return resampled_df
def mixin_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
Returns
-------
"""
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
catalyst/exchange/utils/serialization_utils.py
+24 -1
View File
@@ -5,7 +5,30 @@ from json import JSONEncoder
import pandas as pd
from six import string_types
from catalyst.constants import DATE_TIME_FORMAT
from datetime import date, datetime
from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT
from catalyst.assets._assets import TradingPair
class ConfigJSONEncoder(json.JSONEncoder):
def default(self, obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
elif isinstance(obj, TradingPair):
return obj.to_dict()
class ExchangeJSONEncoder(json.JSONEncoder):
etc/requirements.txt
+1 -1
View File
@@ -82,5 +82,5 @@ tables==3.3.0
#Catalyst dependencies
ccxt==1.10.565
boto3==1.4.8
boto3==1.5.7
redo==1.6
tests/exchange/test_config.py
+8
View File
@@ -0,0 +1,8 @@
from catalyst.exchange.utils.factory import get_exchange
class TestConfig:
def test_create_config(self):
exchange = get_exchange('binance', skip_init=True)
config = exchange.create_exchange_config()
pass