BLD: replacing symbols.json and fetching markets with a single config

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -12,7 +12,8 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
     UnsupportedHistoryFrequencyError, \
-    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError
+    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
+    MarketsNotFoundError, InvalidMarketError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.utils.ccxt_utils import get_exchange_config
 from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
@@ -23,6 +24,7 @@ from catalyst.finance.transaction import Transaction
 from ccxt import InvalidOrder, NetworkError, \
     ExchangeError
 from logbook import Logger
+from redo import retry
 from six import string_types
 
 log = Logger('CCXT', level=LOG_LEVEL)
@@ -107,6 +109,97 @@ class CCXT(Exchange):
             asset = TradingPair(**asset_dict)
             self.assets.append(asset)
 
+    def _fetch_markets(self):
+        markets_symbols = self.api.load_markets()
+        log.debug(
+            'fetching {} markets:\n{}'.format(
+                self.name, markets_symbols
+            )
+        )
+        try:
+            markets = self.api.fetch_markets()
+
+        except NetworkError as e:
+            raise ExchangeRequestError(error=e)
+
+        if not markets:
+            raise MarketsNotFoundError(
+                exchange=self.name,
+            )
+
+        for market in markets:
+            if 'id' not in market:
+                raise InvalidMarketError(
+                    exchange=self.name,
+                    market=market,
+                )
+        return markets
+
+    def create_exchange_config(self):
+        config = dict(
+            name=self.name,
+            features=[feature for feature in self.has if self.has[feature]]
+        )
+        markets = retry(
+            action=self._fetch_markets,
+            attempts=5,
+            sleeptime=5,
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn(
+                'fetching markets again for {}'.format(self.name)
+            ),
+        )
+
+        config['assets'] = []
+        for market in markets:
+            asset = self.create_trading_pair(market=market)
+            config['assets'].append(asset)
+
+        return config
+
+    def create_trading_pair(self, market, start_dt=None, end_dt=None,
+                            leverage=1, end_daily=None, end_minute=None):
+        """
+        Creating a TradingPair from market and asset data.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+        start_dt
+        end_dt
+        leverage
+        end_daily
+        end_minute
+
+        Returns
+        -------
+
+        """
+        params = dict(
+            exchange=self.name,
+            data_source='catalyst',
+            exchange_symbol=market['id'],
+            symbol=get_catalyst_symbol(market),
+            start_date=start_dt,
+            end_date=end_dt,
+            leverage=leverage,
+            asset_name=market['symbol'],
+            end_daily=end_daily,
+            end_minute=end_minute,
+        )
+        self.apply_conditional_market_params(params, market)
+
+        return TradingPair(**params)
+
+    def load_assets(self):
+        if self._config is None:
+            raise ValueError('Exchange config not available.')
+
+        self.assets = []
+        for asset_dict in self._config['assets']:
+            asset = TradingPair(**asset_dict)
+            self.assets.append(asset)
+
     def account(self):
         return None
 
@@ -365,6 +458,54 @@ class CCXT(Exchange):
         except ExchangeSymbolsNotFound:
             return None
 
+    def apply_conditional_market_params(self, params, market):
+        """
+        Applies a CCXT market dict to parameters of TradingPair init.
+
+        Parameters
+        ----------
+        params: dict[Object]
+        market: dict[Object]
+
+        Returns
+        -------
+
+        """
+        # TODO: make this more externalized / configurable
+        # Consider representing in some type of JSON structure
+        if 'active' in market:
+            params['trading_state'] = 1 if market['active'] else 0
+
+        else:
+            params['trading_state'] = 1
+
+        if 'lot' in market:
+            params['min_trade_size'] = market['lot']
+            params['lot'] = market['lot']
+
+        if self.name == 'bitfinex':
+            params['maker'] = 0.001
+            params['taker'] = 0.002
+
+        elif 'maker' in market and 'taker' in market \
+                and market['maker'] is not None \
+                and market['taker'] is not None:
+            params['maker'] = market['maker']
+            params['taker'] = market['taker']
+
+        else:
+            # TODO: default commission, make configurable
+            params['maker'] = 0.0015
+            params['taker'] = 0.0025
+
+        info = market['info'] if 'info' in market else None
+        if info:
+            if 'minimum_order_size' in info:
+                params['min_trade_size'] = float(info['minimum_order_size'])
+
+                if 'lot' not in params:
+                    params['lot'] = params['min_trade_size']
+
     def get_balances(self):
         try:
             log.debug('retrieving wallets balances')

catalyst/exchange/exchange.py

@@ -16,7 +16,7 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
     get_periods, get_start_dt, get_frequency
-from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
+from catalyst.exchange.utils.exchange_utils import \
     resample_history_df, has_bundle
 from logbook import Logger
 
@@ -290,16 +290,6 @@ class Exchange:
         log.debug('found asset: {}'.format(asset))
         return asset
 
-    def fetch_symbol_map(self, is_local=False):
-        index = 1 if is_local else 0
-        if self._symbol_maps[index] is not None:
-            return self._symbol_maps[index]
-
-        else:
-            symbol_map = get_exchange_symbols(self.name, is_local)
-            self._symbol_maps[index] = symbol_map
-            return symbol_map
-
     @abstractmethod
     def init(self):
         """
@@ -311,24 +301,13 @@ class Exchange:
         """
 
     @abstractmethod
-    def load_assets(self, is_local=False):
+    def create_exchange_config(self):
         """
-        Populate the 'assets' attribute with a dictionary of Assets.
-        The key of the resulting dictionary is the exchange specific
-        currency pair symbol. The universal symbol is contained in the
-        'symbol' attribute of each asset.
-
-        Notes
-        -----
-        The sid of each asset is calculated based on a numeric hash of the
-        universal symbol. This simple approach avoids maintaining a mapping
-        of sids.
-
-        This method can be omerridden if an exchange offers equivalent data
-        via its api.
+        Fetch the exchange market data and generate a config object
+        Returns
+        -------
 
         """
-        pass
 
     def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
         """

catalyst/exchange/exchange_bundle.py

@@ -626,13 +626,13 @@ class ExchangeBundle:
                 key=lambda chunk: pd.to_datetime(chunk['period'])
             )
             with maybe_show_progress(
-                all_chunks,
-                show_progress,
-                label='Ingesting {frequency} price data on '
-                      '{exchange}'.format(
-                    exchange=self.exchange_name,
-                    frequency=data_frequency,
-                )) as it:
+                    all_chunks,
+                    show_progress,
+                    label='Ingesting {frequency} price data on '
+                          '{exchange}'.format(
+                            exchange=self.exchange_name,
+                            frequency=data_frequency,
+                    )) as it:
                 for chunk in it:
                     problems += self.ingest_ctable(
                         asset=chunk['asset'],

catalyst/exchange/utils/exchange_utils.py

@@ -11,11 +11,14 @@ from six import string_types
 from six.moves.urllib import request
 
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
+    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
 from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
-    ExchangeJSONDecoder
+    ExchangeJSONDecoder, ConfigJSONEncoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
+from six import string_types
+from six.moves.urllib import request
 
 
 def get_sid(symbol):
@@ -69,7 +72,7 @@ def is_blacklist(exchange_name, environ=None):
     return os.path.exists(filename)
 
 
-def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
+def get_exchange_config_filename(exchange_name, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
 
@@ -83,12 +86,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     str
 
     """
-    name = 'symbols.json' if not is_local else 'symbols_local.json'
+    name = 'config.json'
     exchange_folder = get_exchange_folder(exchange_name, environ)
     return os.path.join(exchange_folder, name)
 
 
-def download_exchange_symbols(exchange_name, environ=None):
+def download_exchange_config(exchange_name, filename, environ=None):
     """
     Downloads the exchange's symbols.json from the repository.
 
@@ -102,15 +105,13 @@ def download_exchange_symbols(exchange_name, environ=None):
     str
 
     """
-    filename = get_exchange_symbols_filename(exchange_name)
-    url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = request.urlretrieve(url=url, filename=filename)
-    return response
+    url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
+    request.urlretrieve(url=url, filename=filename)
 
 
-def get_exchange_symbols(exchange_name, is_local=False, environ=None):
+def get_exchange_config(exchange_name, filename=None, environ=None):
     """
-    The de-serialized content of the exchange's symbols.json.
+    The de-serialized content of the exchange's config.json.
 
     Parameters
     ----------
@@ -123,55 +124,47 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     Object
 
     """
-    filename = get_exchange_symbols_filename(exchange_name, is_local)
-
-    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-            pd.Timestamp('now', tz='UTC') - last_modified_time(
-            filename)).days > 1):
-        try:
-            download_exchange_symbols(exchange_name, environ)
-        except Exception:
-            pass
+    if filename is None:
+        filename = get_exchange_config_filename(exchange_name)
 
     if os.path.isfile(filename):
-        with open(filename) as data_file:
-            try:
-                data = json.load(data_file, cls=ExchangeJSONDecoder)
-                return data
+        now = pd.Timestamp.utcnow()
+        limit = pd.Timedelta('2H')
+        if pd.Timedelta(now - last_modified_time(filename)) > limit:
+            download_exchange_config(exchange_name, filename, environ)
 
-            except ValueError:
-                return dict()
     else:
-        raise ExchangeSymbolsNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
+        download_exchange_config(exchange_name, filename, environ)
 
+    with open(filename) as data_file:
+        try:
+            data = json.load(data_file, cls=ExchangeJSONDecoder)
+            return data
 
-def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
+        except ValueError:
+            return dict()
+
+def save_exchange_config(exchange_name, config, filename=None, environ=None):
     """
-    Save assets into an exchange_symbols file.
+    Save assets into an exchange_config file.
 
     Parameters
     ----------
     exchange_name: str
-    assets: list[dict[str, object]]
-    is_local: bool
+    config
     environ
 
     Returns
     -------
 
     """
-    asset_dicts = dict()
-    for symbol in assets:
-        asset_dicts[symbol] = assets[symbol].to_dict()
+    if filename is None:
+        name = 'config.json'
+        exchange_folder = get_exchange_folder(exchange_name, environ)
+        filename = os.path.join(exchange_folder, name)
 
-    filename = get_exchange_symbols_filename(
-        exchange_name, is_local, environ
-    )
-    with open(filename, 'wt') as handle:
-        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
+    with open(filename, 'w+') as handle:
+        json.dump(config, handle, indent=4, cls=ConfigJSONEncoder)
 
 
 def get_symbols_string(assets):
@@ -508,25 +501,6 @@ def has_bundle(exchange_name, data_frequency, environ=None):
     return os.path.isdir(folder)
 
 
-def symbols_serial(obj):
-    """
-    JSON serializer for objects not serializable by default json code
-
-    Parameters
-    ----------
-    obj: Object
-
-    Returns
-    -------
-    str
-
-    """
-    if isinstance(obj, (datetime, date)):
-        return obj.floor('1D').strftime(DATE_FORMAT)
-
-    raise TypeError("Type %s not serializable" % type(obj))
-
-
 def perf_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -616,46 +590,12 @@ def resample_history_df(df, freq, field, start_dt=None):
     return resampled_df
 
 
-def mixin_market_params(exchange_name, params, market):
-    """
-    Applies a CCXT market dict to parameters of TradingPair init.
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
 
-    Parameters
-    ----------
-    params: dict[Object]
-    market: dict[Object]
 
-    Returns
-    -------
-
-    """
-    # TODO: make this more externalized / configurable
-    if 'lot' in market:
-        params['min_trade_size'] = market['lot']
-        params['lot'] = market['lot']
-
-    if exchange_name == 'bitfinex':
-        params['maker'] = 0.001
-        params['taker'] = 0.002
-
-    elif 'maker' in market and 'taker' in market and \
-            market['maker'] is not None and market['taker'] is not None:
-
-        params['maker'] = market['maker']
-        params['taker'] = market['taker']
-
-    else:
-        # TODO: default commission, make configurable
-        params['maker'] = 0.0015
-        params['taker'] = 0.0025
-
-    info = market['info'] if 'info' in market else None
-    if info:
-        if 'minimum_order_size' in info:
-            params['min_trade_size'] = float(info['minimum_order_size'])
-
-            if 'lot' not in params:
-                params['lot'] = params['min_trade_size']
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)
 
 
 def group_assets_by_exchange(assets):

catalyst/exchange/utils/serialization_utils.py

@@ -3,10 +3,33 @@ import re
 from json import JSONEncoder
 
 import pandas as pd
-from catalyst.assets._assets import TradingPair
-from catalyst.constants import DATE_TIME_FORMAT
 from six import string_types
 
+from datetime import date, datetime
+from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT
+from catalyst.assets._assets import TradingPair
+
+
+class ConfigJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        """
+        JSON serializer for objects not serializable by default json code
+
+        Parameters
+        ----------
+        obj: Object
+
+        Returns
+        -------
+        str
+
+        """
+        if isinstance(obj, (datetime, date)):
+            return obj.floor('1D').strftime(DATE_FORMAT)
+
+        elif isinstance(obj, TradingPair):
+            return obj.to_dict()
+
 
 class ExchangeJSONEncoder(json.JSONEncoder):
     def default(self, obj):