BLD: working on unit tests and data ingestion

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -20,7 +20,7 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeNotFoundError, CreateOrderError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.exchange_utils import mixin_market_params, \
-    from_ms_timestamp, get_epoch, get_exchange_folder
+    from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol
 from catalyst.finance.order import Order, ORDER_STATUS
 
 log = Logger('CCXT', level=LOG_LEVEL)
@@ -188,28 +188,6 @@ class CCXT(Exchange):
         parts = symbol.split('_')
         return '{}/{}'.format(parts[0].upper(), parts[1].upper())
 
-    def get_catalyst_symbol(self, market_or_symbol):
-        """
-        The Catalyst symbol.
-
-        Parameters
-        ----------
-        market_or_symbol
-
-        Returns
-        -------
-
-        """
-        if isinstance(market_or_symbol, string_types):
-            parts = market_or_symbol.split('/')
-            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
-
-        else:
-            return '{}_{}'.format(
-                market_or_symbol['base'].lower(),
-                market_or_symbol['quote'].lower(),
-            )
-
     def get_timeframe(self, freq):
         """
         The CCXT timeframe from the Catalyst frequency.
@@ -402,7 +380,7 @@ class CCXT(Exchange):
                    and asset_def['end_minute'] != 'N/A' else None
 
         else:
-            params['symbol'] = self.get_catalyst_symbol(market)
+            params['symbol'] = get_catalyst_symbol(market)
             # TODO: add as an optional column
             params['leverage'] = 1.0
 
@@ -656,30 +634,46 @@ class CCXT(Exchange):
 
         """
         tickers = dict()
-        for asset in assets:
-            try:
-                ccxt_symbol = self.get_symbol(asset)
-                ticker = self.api.fetch_ticker(ccxt_symbol)
+        try:
+            symbols = [self.get_symbol(asset) for asset in assets]
+            ccxt_tickers = self.api.fetch_tickers(symbols)
 
+            for asset in assets:
+                symbol = self.get_symbol(asset)
+                if symbol not in ccxt_tickers:
+                    log.warn('ticker not found for {} {}'.format(
+                        self.name, symbol
+                    ))
+                    continue
+
+                ticker = ccxt_tickers[symbol]
                 ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
 
                 if 'last_price' not in ticker:
                     # TODO: any more exceptions?
                     ticker['last_price'] = ticker['last']
 
-                # Using the volume represented in the base currency
-                ticker['volume'] = ticker['baseVolume'] \
-                    if 'baseVolume' in ticker else 0
+                if 'baseVolume' in ticker and ticker['baseVolume'] is not None:
+                    # Using the volume represented in the base currency
+                    ticker['volume'] = ticker['baseVolume']
+
+                elif 'info' in ticker and 'bidQty' in ticker['info'] \
+                        and 'askQty' in ticker['info']:
+                    ticker['volume'] = float(ticker['info']['bidQty']) + \
+                                       float(ticker['info']['askQty'])
+
+                else:
+                    ticker['volume'] = 0
 
                 tickers[asset] = ticker
 
-            except ExchangeNotAvailable as e:
-                log.warn(
-                    'unable to fetch ticker: {} {}'.format(
-                        self.name, asset.symbol
-                    )
+        except ExchangeNotAvailable as e:
+            log.warn(
+                'unable to fetch ticker: {} {}'.format(
+                    self.name, asset.symbol
                 )
-                raise ExchangeRequestError(error=e)
+            )
+            raise ExchangeRequestError(error=e)
 
         return tickers
 

catalyst/exchange/exchange_bundle.py

@@ -29,7 +29,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.exchange_utils import get_exchange_folder, \
-    save_exchange_symbols, mixin_market_params
+    save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -730,7 +730,7 @@ class ExchangeBundle:
                     if data_frequency == 'minute' else asset_def['end_minute']
 
             else:
-                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+                params['symbol'] = get_catalyst_symbol(market)
 
                 params['end_daily'] = end_dt \
                     if data_frequency == 'daily' else 'N/A'

catalyst/exchange/exchange_utils.py

@@ -62,6 +62,13 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
+def is_blacklist(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'blacklist.txt')
+
+    return os.path.exists(filename)
+
+
 def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
@@ -133,8 +140,8 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     filename = get_exchange_symbols_filename(exchange_name, is_local)
 
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-                pd.Timestamp('now', tz='UTC') - last_modified_time(
-                    filename)).days > 1):
+            pd.Timestamp('now', tz='UTC') - last_modified_time(
+                filename)).days > 1):
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
@@ -646,3 +653,25 @@ def group_assets_by_exchange(assets):
         exchange_assets[asset.exchange].append(asset)
 
     return exchange_assets
+
+def get_catalyst_symbol(market_or_symbol):
+    """
+    The Catalyst symbol.
+
+    Parameters
+    ----------
+    market_or_symbol
+
+    Returns
+    -------
+
+    """
+    if isinstance(market_or_symbol, string_types):
+        parts = market_or_symbol.split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+    else:
+        return '{}_{}'.format(
+            market_or_symbol['base'].lower(),
+            market_or_symbol['quote'].lower(),
+        )

catalyst/exchange/factory.py

@@ -8,7 +8,7 @@ from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
 from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_exchange_folder
+    get_exchange_folder, is_blacklist
 
 log = Logger('factory', level=LOG_LEVEL)
 
@@ -47,7 +47,7 @@ def get_exchanges(exchange_names):
     return exchanges
 
 
-def find_exchanges(features=None):
+def find_exchanges(features=None, skip_blacklist=True):
     """
     Find exchanges filtered by a list of feature.
 
@@ -65,6 +65,9 @@ def find_exchanges(features=None):
 
     exchanges = []
     for exchange_name in exchange_names:
+        if skip_blacklist and is_blacklist(exchange_name):
+            continue
+
         exchanges.append(get_exchange(exchange_name, skip_init=True))
 
     return exchanges

tests/exchange/test_suite_exchange.py

@@ -30,24 +30,17 @@ def handle_exchange_error(exchange, e):
         is_blacklist = True
 
     if is_blacklist:
-        root = data_root()
-        filename = os.path.join(root, 'exchanges', 'blacklist.json')
+        try:
+            message = '{}: {}'.format(
+                e.__class__, e.message.decode('ascii', 'ignore')
+            )
+        except Exception:
+            message = 'unexpected error'
 
-        if os.path.isfile(filename):
-            with open(filename) as handle:
-                try:
-                    bl_data = json.load(handle)
-
-                except ValueError:
-                    bl_data = dict()
-
-        else:
-            bl_data = dict()
-
-        if exchange.name not in bl_data:
-            bl_data[exchange.name] = '{}: {}'.format(e.__class__, e.message)
-            with open(filename, 'wt') as handle:
-                json.dump(bl_data, handle, indent=4)
+        folder = get_exchange_folder(exchange.name)
+        filename = os.path.join(folder, 'blacklist.txt')
+        with open(filename, 'wt') as handle:
+            handle.write(message)
 
 
 def select_random_exchanges(population=3, features=None):
@@ -62,8 +55,7 @@ def select_random_exchanges(population=3, features=None):
     return exchanges
 
 
-def select_random_assets(exchange, population=3):
-    all_assets = exchange.assets
+def select_random_assets(all_assets, population=3):
     assets = random.sample(all_assets, population)
     return assets
 
@@ -93,6 +85,11 @@ class TestSuiteExchange(unittest.TestCase):
                 handle_exchange_error(exchange, e)
 
             else:
+                print(
+                    're-trying an exchange request {} {}'.format(
+                        exchange.name, attempts
+                    )
+                )
                 self._test_markets_exchange(exchange, attempts + 1)
 
         except Exception as e:
@@ -101,17 +98,18 @@ class TestSuiteExchange(unittest.TestCase):
         return assets
 
     def test_markets(self):
-        population = None
+        population = 3
         results = dict()
 
         exchanges = select_random_exchanges(population)  # Type: list[Exchange]
         for exchange in exchanges:
             assets = self._test_markets_exchange(exchange)
+
             if assets is not None:
                 results[exchange.name] = len(assets)
 
                 folder = get_exchange_folder(exchange.name)
-                filename = os.path.join(folder, 'supported_assets.json')
+                filename = os.path.join(folder, 'whitelist.json')
 
                 symbols = [asset.symbol for asset in assets]
                 with open(filename, 'wt') as handle:
@@ -125,13 +123,29 @@ class TestSuiteExchange(unittest.TestCase):
 
         pass
 
-    def test_ticker(self):
-        exchanges = select_random_exchanges(3)  # Type: list[Exchange]
+    def test_tickers(self):
+        exchange_population = 3
+        asset_population = 3
+
+        exchanges = select_random_exchanges(
+            exchange_population
+        )  # Type: list[Exchange]
         for exchange in exchanges:
             exchange.init()
 
-            assets = select_random_assets(exchange, 3)
-            exchange.tickers()
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+                tickers = exchange.tickers(assets)
+
+                assert len(tickers) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
         pass
 
     def test_candles(self):