BLD: tested all public APIs with CCXT

catalyst/assets/_assets.pyx

@@ -396,7 +396,7 @@ cdef class Future(Asset):
 
 cdef class TradingPair(Asset):
     cdef readonly float leverage
-    cdef readonly object market_currency
+    cdef readonly object quote_currency
     cdef readonly object base_currency
     cdef readonly object end_daily
     cdef readonly object end_minute
@@ -417,7 +417,7 @@ cdef class TradingPair(Asset):
         'exchange',
         'exchange_full',
         'leverage',
-        'market_currency',
+        'quote_currency',
         'base_currency',
         'end_daily',
         'end_minute',
@@ -442,7 +442,7 @@ cdef class TradingPair(Asset):
                  object first_traded=None,
                  object auto_close_date=None,
                  object exchange_full=None,
-                 float min_trade_size=0.000001,
+                 float min_trade_size=0.0001,
                  float maker=0.0015,
                  float taker=0.0025,
                  int trading_state=0,
@@ -516,7 +516,7 @@ cdef class TradingPair(Asset):
 
         symbol = symbol.lower()
         try:
-            self.market_currency, self.base_currency = symbol.split('_')
+            self.base_currency,self.quote_currency  = symbol.split('_')
         except Exception as e:
             raise InvalidSymbolError(symbol=symbol, error=e)
 
@@ -530,7 +530,7 @@ cdef class TradingPair(Asset):
             asset_name = ' / '.join(symbol.split('_')).upper()
 
         if start_date is None:
-            start_date = pd.Timestamp.utcnow()
+            start_date = pd.to_datetime('2009-1-1', utc=True)
 
         if end_date is None:
             end_date = pd.Timestamp.utcnow() + timedelta(days=365)
@@ -560,8 +560,8 @@ cdef class TradingPair(Asset):
     def __repr__(self):
         return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
                'Introduced On: {start_date}, ' \
-               'Market Currency: {market_currency}, ' \
                'Base Currency: {base_currency}, ' \
+               'Quote Currency: {quote_currency}, ' \
                'Exchange Leverage: {leverage}, ' \
                'Minimum Trade Size: {min_trade_size} ' \
                'Last daily ingestion: {end_daily} ' \
@@ -570,7 +570,7 @@ cdef class TradingPair(Asset):
             sid=self.sid,
             exchange=self.exchange,
             start_date=self.start_date,
-            market_currency=self.market_currency,
+            quote_currency=self.quote_currency,
             base_currency=self.base_currency,
             leverage=self.leverage,
             min_trade_size=self.min_trade_size,

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -11,7 +11,8 @@ from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound
-from catalyst.exchange.exchange_utils import mixin_market_params
+from catalyst.exchange.exchange_utils import mixin_market_params, \
+    from_ms_timestamp
 
 log = Logger('CCXT', level=LOG_LEVEL)
 
@@ -19,17 +20,28 @@ log = Logger('CCXT', level=LOG_LEVEL)
 class CCXT(Exchange):
     def __init__(self, exchange_name, key, secret, base_currency,
                  portfolio=None):
-        log.debug('available exchanges:\n{}'.format(ccxt.exchanges))
-        self.api = ccxt.poloniex({
-            'apiKey': key,
-            'secret': secret,
-        })
+        log.debug(
+            'finding {} in CCXT exchanges:\n{}'.format(
+                exchange_name, ccxt.exchanges
+            )
+        )
+        try:
+            exchange_attr = getattr(ccxt, exchange_name)
+            self.api = exchange_attr({
+                'apiKey': key,
+                'secret': secret,
+            })
+        except Exception:
+            raise ValueError('exchange not in CCXT')
+
         markets = self.api.load_markets()
         log.debug('the markets:\n{}'.format(markets))
 
         self.name = exchange_name
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.transactions = defaultdict(list)
@@ -50,6 +62,10 @@ class CCXT(Exchange):
         parts = asset.symbol.split('_')
         return '{}/{}'.format(parts[0].upper(), parts[1].upper())
 
+    def get_catalyst_symbol(self, market):
+        parts = market['symbol'].split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
     def get_timeframe(self, freq):
         freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
         if freq_match:
@@ -99,22 +115,49 @@ class CCXT(Exchange):
             ))
         return candles
 
-    def load_assets(self, is_local=False):
-        markets = self.api.fetch_markets()
+    def _fetch_symbol_map(self, is_local):
         try:
-            symbol_map = self.fetch_symbol_map(is_local)
+            return self.fetch_symbol_map(is_local)
         except ExchangeSymbolsNotFound:
             return None
 
-        data_source = 'local' if is_local else 'catalyst'
+    def _fetch_asset(self, market_id, is_local=False):
+        symbol_map = self._fetch_symbol_map(is_local)
+        if symbol_map is not None:
+            assets_lower = {k.lower(): v for k, v in symbol_map.items()}
+            key = market_id.lower()
+
+            asset = assets_lower[key] if key in assets_lower else None
+            if asset is not None:
+                return asset, is_local
+
+            elif not is_local:
+                return self._fetch_asset(market_id, True)
+
+            else:
+                return None, is_local
+
+        elif not is_local:
+            return self._fetch_asset(market_id, True)
+
+        else:
+            return None, is_local
+
+    def load_assets(self):
+        markets = self.api.fetch_markets()
+
         for market in markets:
-            asset = symbol_map[market['id']] \
-                if market['id'] in markets else None
+            asset, is_local = self._fetch_asset(market['id'])
+            data_source = 'local' if is_local else 'catalyst'
 
-            params = dict(exchange=self.name, data_source=data_source)
-            mixin_market_params(params, market)
+            params = dict(
+                exchange=self.name,
+                data_source=data_source,
+                exchange_symbol=market['id'],
+            )
+            mixin_market_params(self.name, params, market)
 
-            if asset:
+            if asset is not None:
                 params['symbol'] = asset['symbol']
 
                 params['start_date'] = pd.to_datetime(
@@ -142,13 +185,10 @@ class CCXT(Exchange):
                     else None
 
             else:
-                params['symbol'] = market['id']
+                params['symbol'] = self.get_catalyst_symbol(market)
 
             trading_pair = TradingPair(**params)
-            if is_local:
-                self.local_assets[market['id']] = trading_pair
-            else:
-                self.assets[market['id']] = trading_pair
+            self.assets[market['id']] = trading_pair
 
     def get_balances(self):
         return None
@@ -166,10 +206,56 @@ class CCXT(Exchange):
         return None
 
     def tickers(self, assets):
-        return None
+        """
+        Retrieve current tick data for the given assets
+
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[dict[str, float]
+
+        """
+        tickers = dict()
+        for asset in assets:
+            ccxt_symbol = self.get_symbol(asset)
+            ticker = self.api.fetch_ticker(ccxt_symbol)
+
+            ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
+
+            # Using the volume represented in the base currency
+            ticker['volume'] = ticker['baseVolume'] \
+                if 'baseVolume' in ticker else 0
+
+            tickers[asset] = ticker
+
+        return tickers
 
     def get_account(self):
         return None
 
-    def get_orderbook(self, asset, order_type, limit):
-        return None
+    def get_orderbook(self, asset, order_type='all', limit=None):
+        ccxt_symbol = self.get_symbol(asset)
+
+        params = dict()
+        if limit is not None:
+            params['depth'] = limit
+
+        order_book = self.api.fetch_order_book(ccxt_symbol, params)
+
+        order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
+        result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
+        for index, order_type in enumerate(order_types):
+            if limit is not None and index > limit - 1:
+                break
+
+            result[order_type] = []
+            for entry in order_book[order_type]:
+                result[order_type].append(dict(
+                    rate=float(entry[0]),
+                    quantity=float(entry[1])
+                ))
+
+        return result

catalyst/exchange/exchange.py

@@ -16,7 +16,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, ExchangeSymbolsNotFound
+    NoDataAvailableOnExchange, ExchangeSymbolsNotFound, NoValueForField
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -412,12 +412,15 @@ class Exchange:
         if field not in BASE_FIELDS:
             raise KeyError('Invalid column: {}'.format(field))
 
-        values = []
-        for asset in assets:
-            value = self.get_single_spot_value(asset, field, data_frequency)
-            values.append(value)
+        tickers = self.tickers(assets)
+        if field == 'close' or field == 'price':
+            return [t['last'] for t in tickers]
 
-        return values
+        elif field == 'volume':
+            return [t['volume'] for t in tickers]
+
+        else:
+            raise NoValueForField(field=field)
 
     def get_single_spot_value(self, asset, field, data_frequency):
         """

catalyst/exchange/exchange_errors.py

@@ -240,3 +240,7 @@ class NoDataAvailableOnExchange(ZiplineError):
         'Requested data for trading pair {symbol} is not available on exchange {exchange} '
         'in `{data_frequency}` frequency at this time. '
         'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+
+
+class NoValueForField(ZiplineError):
+    msg = ('Value not found for field: {field}.').strip()

catalyst/exchange/exchange_utils.py

@@ -573,7 +573,7 @@ def resample_history_df(df, freq, field):
     return resampled_df
 
 
-def mixin_market_params(params, market):
+def mixin_market_params(exchange_name, params, market):
     """
     Applies a CCXT market dict to parameters of TradingPair init.
 
@@ -586,7 +586,26 @@ def mixin_market_params(params, market):
     -------
 
     """
-    params['min_trade_size'] = market['lot']
-    params['maker'] = market['maker']
-    params['taker'] = market['taker']
-    params['trading_state'] = 1 if int(market['info']['isFrozen']) == 0 else 0
+    # TODO: make this more externalized / configurable
+    if 'lot' in market:
+        params['min_trade_size'] = market['lot']
+
+    if exchange_name == 'bitfinex':
+        params['maker'] = 0.001
+        params['taker'] = 0.002
+
+    else:
+        if 'maker' in market:
+            params['maker'] = market['maker']
+
+        if 'taker' in market:
+            params['taker'] = market['taker']
+
+    info = market['info'] if 'info' in market else None
+    if info:
+        if 'minimum_order_size' in info:
+            params['min_trade_size'] = float(info['minimum_order_size'])
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)

tests/exchange/test_ccxt.py

@@ -15,7 +15,7 @@ log = Logger('test_ccxt')
 class TestCCXT(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        exchange_name = 'poloniex'
+        exchange_name = 'binance'
         auth = get_exchange_auth(exchange_name)
         self.exchange = CCXT(
             exchange_name=exchange_name,
@@ -97,7 +97,7 @@ class TestCCXT(BaseExchangeTestCase):
     def test_orderbook(self):
         log.info('testing order book for bittrex')
         asset = self.exchange.get_asset('eth_btc')
-        orderbook = self.exchange.get_orderbook(asset)
+        orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
         pass
 
     def test_get_fees(self):