diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index e78e3af1..feb41710 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -396,7 +396,7 @@ cdef class Future(Asset): cdef class TradingPair(Asset): cdef readonly float leverage - cdef readonly object market_currency + cdef readonly object quote_currency cdef readonly object base_currency cdef readonly object end_daily cdef readonly object end_minute @@ -417,7 +417,7 @@ cdef class TradingPair(Asset): 'exchange', 'exchange_full', 'leverage', - 'market_currency', + 'quote_currency', 'base_currency', 'end_daily', 'end_minute', @@ -442,7 +442,7 @@ cdef class TradingPair(Asset): object first_traded=None, object auto_close_date=None, object exchange_full=None, - float min_trade_size=0.000001, + float min_trade_size=0.0001, float maker=0.0015, float taker=0.0025, int trading_state=0, @@ -516,7 +516,7 @@ cdef class TradingPair(Asset): symbol = symbol.lower() try: - self.market_currency, self.base_currency = symbol.split('_') + self.base_currency,self.quote_currency = symbol.split('_') except Exception as e: raise InvalidSymbolError(symbol=symbol, error=e) @@ -530,7 +530,7 @@ cdef class TradingPair(Asset): asset_name = ' / '.join(symbol.split('_')).upper() if start_date is None: - start_date = pd.Timestamp.utcnow() + start_date = pd.to_datetime('2009-1-1', utc=True) if end_date is None: end_date = pd.Timestamp.utcnow() + timedelta(days=365) @@ -560,8 +560,8 @@ cdef class TradingPair(Asset): def __repr__(self): return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \ 'Introduced On: {start_date}, ' \ - 'Market Currency: {market_currency}, ' \ 'Base Currency: {base_currency}, ' \ + 'Quote Currency: {quote_currency}, ' \ 'Exchange Leverage: {leverage}, ' \ 'Minimum Trade Size: {min_trade_size} ' \ 'Last daily ingestion: {end_daily} ' \ @@ -570,7 +570,7 @@ cdef class TradingPair(Asset): sid=self.sid, exchange=self.exchange, start_date=self.start_date, - market_currency=self.market_currency, + quote_currency=self.quote_currency, base_currency=self.base_currency, leverage=self.leverage, min_trade_size=self.min_trade_size, diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 00dd2970..58be745c 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -11,7 +11,8 @@ from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound -from catalyst.exchange.exchange_utils import mixin_market_params +from catalyst.exchange.exchange_utils import mixin_market_params, \ + from_ms_timestamp log = Logger('CCXT', level=LOG_LEVEL) @@ -19,17 +20,28 @@ log = Logger('CCXT', level=LOG_LEVEL) class CCXT(Exchange): def __init__(self, exchange_name, key, secret, base_currency, portfolio=None): - log.debug('available exchanges:\n{}'.format(ccxt.exchanges)) - self.api = ccxt.poloniex({ - 'apiKey': key, - 'secret': secret, - }) + log.debug( + 'finding {} in CCXT exchanges:\n{}'.format( + exchange_name, ccxt.exchanges + ) + ) + try: + exchange_attr = getattr(ccxt, exchange_name) + self.api = exchange_attr({ + 'apiKey': key, + 'secret': secret, + }) + except Exception: + raise ValueError('exchange not in CCXT') + markets = self.api.load_markets() log.debug('the markets:\n{}'.format(markets)) self.name = exchange_name - self.assets = {} + + self.assets = dict() self.load_assets() + self.base_currency = base_currency self._portfolio = portfolio self.transactions = defaultdict(list) @@ -50,6 +62,10 @@ class CCXT(Exchange): parts = asset.symbol.split('_') return '{}/{}'.format(parts[0].upper(), parts[1].upper()) + def get_catalyst_symbol(self, market): + parts = market['symbol'].split('/') + return '{}_{}'.format(parts[0].lower(), parts[1].lower()) + def get_timeframe(self, freq): freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I) if freq_match: @@ -99,22 +115,49 @@ class CCXT(Exchange): )) return candles - def load_assets(self, is_local=False): - markets = self.api.fetch_markets() + def _fetch_symbol_map(self, is_local): try: - symbol_map = self.fetch_symbol_map(is_local) + return self.fetch_symbol_map(is_local) except ExchangeSymbolsNotFound: return None - data_source = 'local' if is_local else 'catalyst' + def _fetch_asset(self, market_id, is_local=False): + symbol_map = self._fetch_symbol_map(is_local) + if symbol_map is not None: + assets_lower = {k.lower(): v for k, v in symbol_map.items()} + key = market_id.lower() + + asset = assets_lower[key] if key in assets_lower else None + if asset is not None: + return asset, is_local + + elif not is_local: + return self._fetch_asset(market_id, True) + + else: + return None, is_local + + elif not is_local: + return self._fetch_asset(market_id, True) + + else: + return None, is_local + + def load_assets(self): + markets = self.api.fetch_markets() + for market in markets: - asset = symbol_map[market['id']] \ - if market['id'] in markets else None + asset, is_local = self._fetch_asset(market['id']) + data_source = 'local' if is_local else 'catalyst' - params = dict(exchange=self.name, data_source=data_source) - mixin_market_params(params, market) + params = dict( + exchange=self.name, + data_source=data_source, + exchange_symbol=market['id'], + ) + mixin_market_params(self.name, params, market) - if asset: + if asset is not None: params['symbol'] = asset['symbol'] params['start_date'] = pd.to_datetime( @@ -142,13 +185,10 @@ class CCXT(Exchange): else None else: - params['symbol'] = market['id'] + params['symbol'] = self.get_catalyst_symbol(market) trading_pair = TradingPair(**params) - if is_local: - self.local_assets[market['id']] = trading_pair - else: - self.assets[market['id']] = trading_pair + self.assets[market['id']] = trading_pair def get_balances(self): return None @@ -166,10 +206,56 @@ class CCXT(Exchange): return None def tickers(self, assets): - return None + """ + Retrieve current tick data for the given assets + + Parameters + ---------- + assets: list[TradingPair] + + Returns + ------- + list[dict[str, float] + + """ + tickers = dict() + for asset in assets: + ccxt_symbol = self.get_symbol(asset) + ticker = self.api.fetch_ticker(ccxt_symbol) + + ticker['last_traded'] = from_ms_timestamp(ticker['timestamp']) + + # Using the volume represented in the base currency + ticker['volume'] = ticker['baseVolume'] \ + if 'baseVolume' in ticker else 0 + + tickers[asset] = ticker + + return tickers def get_account(self): return None - def get_orderbook(self, asset, order_type, limit): - return None + def get_orderbook(self, asset, order_type='all', limit=None): + ccxt_symbol = self.get_symbol(asset) + + params = dict() + if limit is not None: + params['depth'] = limit + + order_book = self.api.fetch_order_book(ccxt_symbol, params) + + order_types = ['bids', 'asks'] if order_type == 'all' else [order_type] + result = dict(last_traded=from_ms_timestamp(order_book['timestamp'])) + for index, order_type in enumerate(order_types): + if limit is not None and index > limit - 1: + break + + result[order_type] = [] + for entry in order_book[order_type]: + result[order_type].append(dict( + rate=float(entry[0]), + quantity=float(entry[1]) + )) + + return result diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index f43424aa..8a69d3fb 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -16,7 +16,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ PricingDataNotLoadedError, \ - NoDataAvailableOnExchange, ExchangeSymbolsNotFound + NoDataAvailableOnExchange, ExchangeSymbolsNotFound, NoValueForField from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \ ExchangeLimitOrder, ExchangeStopOrder from catalyst.exchange.exchange_portfolio import ExchangePortfolio @@ -412,12 +412,15 @@ class Exchange: if field not in BASE_FIELDS: raise KeyError('Invalid column: {}'.format(field)) - values = [] - for asset in assets: - value = self.get_single_spot_value(asset, field, data_frequency) - values.append(value) + tickers = self.tickers(assets) + if field == 'close' or field == 'price': + return [t['last'] for t in tickers] - return values + elif field == 'volume': + return [t['volume'] for t in tickers] + + else: + raise NoValueForField(field=field) def get_single_spot_value(self, asset, field, data_frequency): """ diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index cb4f4d32..5530ccb2 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -240,3 +240,7 @@ class NoDataAvailableOnExchange(ZiplineError): 'Requested data for trading pair {symbol} is not available on exchange {exchange} ' 'in `{data_frequency}` frequency at this time. ' 'Check `http://enigma.co/catalyst/status` for market coverage.').strip() + + +class NoValueForField(ZiplineError): + msg = ('Value not found for field: {field}.').strip() diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 83dfae41..ee0ce44e 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -573,7 +573,7 @@ def resample_history_df(df, freq, field): return resampled_df -def mixin_market_params(params, market): +def mixin_market_params(exchange_name, params, market): """ Applies a CCXT market dict to parameters of TradingPair init. @@ -586,7 +586,26 @@ def mixin_market_params(params, market): ------- """ - params['min_trade_size'] = market['lot'] - params['maker'] = market['maker'] - params['taker'] = market['taker'] - params['trading_state'] = 1 if int(market['info']['isFrozen']) == 0 else 0 + # TODO: make this more externalized / configurable + if 'lot' in market: + params['min_trade_size'] = market['lot'] + + if exchange_name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 + + else: + if 'maker' in market: + params['maker'] = market['maker'] + + if 'taker' in market: + params['taker'] = market['taker'] + + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) + + +def from_ms_timestamp(ms): + return pd.to_datetime(ms, unit='ms', utc=True) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index f920fa39..f00ea049 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -15,7 +15,7 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'poloniex' + exchange_name = 'binance' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, @@ -97,7 +97,7 @@ class TestCCXT(BaseExchangeTestCase): def test_orderbook(self): log.info('testing order book for bittrex') asset = self.exchange.get_asset('eth_btc') - orderbook = self.exchange.get_orderbook(asset) + orderbook = self.exchange.get_orderbook(asset, 'all', limit=10) pass def test_get_fees(self):