Merge branch 'develop'

Dockerfile

@@ -11,13 +11,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

Dockerfile-dev

@@ -5,7 +5,7 @@
 #
 # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile
+#    docker build -t quantopian/catalyst -f Dockerfile .
 #
 # To run the container:
 #
@@ -15,13 +15,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

catalyst/__main__.py

@@ -9,7 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_utils import delete_algo_folder
+from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 

catalyst/assets/_assets.pyx

@@ -17,6 +17,7 @@
 """
 Cythonized Asset object.
 """
+
 import hashlib
 
 cimport cython
@@ -38,7 +39,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
-from catalyst.exchange.exchange_utils import get_sid
+from catalyst.exchange.utils.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 

catalyst/constants.py

@@ -7,7 +7,7 @@ import logbook
     For example, if you want to see the DEBUG messages, run:
     $ export CATALYST_LOG_LEVEL=10
 '''
-LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
+LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.DEBUG))
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'

catalyst/curate/poloniex.py

@@ -1,16 +1,16 @@
-import os
-import time
-import shutil
-import json
 import csv
+import json
+import os
+import shutil
+import time
 from datetime import datetime
 
+import logbook
 import pandas as pd
 import requests
-import logbook
-
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
+from catalyst.exchange.utils.exchange_utils import \
+    get_exchange_symbols_filename
 
 DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
 DT_END = pd.to_datetime('today').value // 10 ** 9
@@ -193,7 +193,8 @@ class PoloniexCurator(object):
                    for this currencyPair
         '''
         try:
-            if('end_file' in locals() and end_file + 3600 < end):
+            if(temp is not None
+                    or ('end_file' in locals() and end_file + 3600 < end)):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
@@ -261,7 +262,7 @@ class PoloniexCurator(object):
         vol = df['total'].to_frame('volume')           # set Vol aside
         df.drop('total', axis=1, inplace=True)         # Drop volume data
         ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
-        ohlc.cols = ohlc.cols.map(lambda t: t[1])      # Raname cols
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Rename cols
         closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
         ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
         vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin

catalyst/data/loader.py

@@ -22,6 +22,7 @@ from pandas_datareader.data import DataReader
 from six import iteritems
 from six.moves.urllib_error import HTTPError
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.utils.calendars import get_calendar
 from . import treasuries, treasuries_can
 from .benchmarks import get_benchmark_returns
@@ -31,8 +32,6 @@ from ..utils.paths import (
     data_root,
 )
 
-from catalyst.constants import LOG_LEVEL
-
 logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
@@ -143,7 +142,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.factory import get_exchange
+        from catalyst.exchange.utils.factory import get_exchange
         exchange = get_exchange(
             exchange_name='poloniex', base_currency='usdt'
         )

catalyst/examples/arbitrage_with_interface.py

@@ -6,7 +6,7 @@ from catalyst.api import (
     symbol,
     get_open_orders
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'arbitrage_eth_btc'

catalyst/examples/buy_low_sell_high.py

@@ -1,19 +1,7 @@
-'''
-This algorithm requires an additional library (ta-lib) beyond those
-required by catalyst. Install it first by running:
-$ pip install TA-Lib
-
-If you get build errors like:
-    "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and it
-needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
-instructions on how to install the required dependencies.
-'''
-
 import talib
+import pandas as pd
 from logbook import Logger
 
-from catalyst import run_algorithm
 from catalyst.api import (
     order,
     order_target_percent,
@@ -21,59 +9,52 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
-import pandas as pd
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+from catalyst.utils.run_algo import run_algorithm
 
-algo_namespace = 'buy_low_sell_high_xrp'
-log = Logger(algo_namespace)
+algo_namespace = 'buy_the_dip_live'
+log = Logger('buy low sell high')
 
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USDT'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 5000
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.05
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.swallow_errors = True
+    context.SLIPPAGE_ALLOWED = 0.02
 
     context.errors = []
     pass
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1D'
     )
-
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
     elif rsi <= 70:
-        buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -141,11 +122,11 @@ def _handle_data(context, data):
 
 def handle_data(context, data):
     log.info('handling bar {}'.format(data.current_dt))
-    try:
-        _handle_data(context, data)
-    except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
+    # try:
+    _handle_data(context, data)
+    # except Exception as e:
+    #     log.warn('aborting the bar on error {}'.format(e))
+    #     context.errors.append(e)
 
     log.info('completed bar {}, total execution errors {}'.format(
         data.current_dt,
@@ -162,15 +143,29 @@ def analyze(context, stats):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        algo_namespace='buy_and_hodl',
-        base_currency='usd',
-        start=pd.to_datetime('2015-03-01', utc=True),
-        end=pd.to_datetime('2017-10-31', utc=True),
-    )
+    live = False
+    if live:
+        run_algorithm(
+            capital_base=0.001,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='binance',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='buy_and_hodl',
+            base_currency='usdt',
+            start=pd.to_datetime('2015-03-01', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )

catalyst/examples/buy_low_sell_high_live.py

@@ -1,160 +0,0 @@
-import talib
-from logbook import Logger
-
-import pandas as pd
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_the_dip_live'
-log = Logger('buy low sell high')
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.ASSET_NAME = 'btc_usdt'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.TARGET_POSITIONS = 30
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='1D'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.2
-    else:
-        buy_increment = 0.1
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(
-        price=price,
-        rsi=rsi,
-    )
-
-    orders = get_open_orders(context.asset)
-    if orders:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif (position.amount > 0
-              and price > cost_basis * (1 + context.PROFIT_TARGET)):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            log.info('the rsi is too high to consider buying {}'.format(rsi))
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
-        )
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-    pass
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=0.001,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='binance',
-        live=True,
-        algo_namespace=algo_namespace,
-        base_currency='btc',
-        simulate_orders=True,
-    )

catalyst/examples/dual_moving_average.py

@@ -1,12 +1,12 @@
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from logbook import Logger
-import matplotlib.pyplot as plt
 
 from catalyst import run_algorithm
 from catalyst.api import (record, symbol, order_target_percent,
                           get_open_orders)
-from catalyst.exchange.stats_utils import extract_transactions
+from catalyst.exchange.utils.stats_utils import extract_transactions
 
 NAMESPACE = 'dual_moving_average'
 log = Logger(NAMESPACE)

catalyst/examples/mean_reversion_simple.py

@@ -12,8 +12,7 @@ from logbook import Logger
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.stats_utils import extract_transactions
-
+from catalyst.exchange.utils.stats_utils import extract_transactions
 # We give a name to the algorithm which Catalyst will use to persist its state.
 # In this example, Catalyst will create the `.catalyst/data/live_algos`
 # directory. If we stop and start the algorithm, Catalyst will resume its
@@ -34,12 +33,12 @@ def initialize(context):
     # parameters or values you're going to use.
 
     # In our example, we're looking at Neo in Ether.
-    context.market = symbol('neo_eth')
+    context.market = symbol('eth_btc')
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 30
-    context.RSI_OVERBOUGHT = 80
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 65
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
@@ -245,9 +244,24 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    MODE = 'backtest'
+    live = True
 
-    if MODE == 'backtest':
+    if live:
+        run_algorithm(
+            capital_base=0.03,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
         folder = os.path.join(
             tempfile.gettempdir(), 'catalyst', NAMESPACE
         )
@@ -272,18 +286,3 @@ if __name__ == '__main__':
             output=out
         )
         log.info('saved perf stats: {}'.format(out))
-
-    elif MODE == 'live':
-        run_algorithm(
-            capital_base=0.05,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='binance',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            live_graph=False,
-            simulate_orders=True,
-            stats_output=None
-        )

catalyst/examples/simple_loop.py

@@ -1,23 +1,26 @@
-import talib
 import pandas as pd
+import talib
+from logbook import Logger, INFO
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record
-from catalyst.exchange.stats_utils import get_pretty_stats, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
     extract_transactions
 
+log = Logger('simple_loop', level=INFO)
+
 
 def initialize(context):
-    print('initializing')
+    log.info('initializing')
     context.asset = symbol('eth_btc')
     context.base_price = None
 
 
 def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
+    log.info('handling bar: {}'.format(data.current_dt))
 
     price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
+    log.info('got price {price}'.format(price=price))
 
     prices = data.history(
         context.asset,
@@ -26,10 +29,10 @@ def handle_data(context, data):
         frequency='30T'
     )
     last_traded = prices.index[-1]
-    print('last candle date: {}'.format(last_traded))
+    log.info('last candle date: {}'.format(last_traded))
 
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    print('got rsi: {}'.format(rsi))
+    log.info('got rsi: {}'.format(rsi))
 
     # If base_price is not set, we use the current value. This is the
     # price at the first bar which we reference to calculate price_change.
@@ -51,7 +54,7 @@ def handle_data(context, data):
 
 def analyze(context, perf):
     import matplotlib.pyplot as plt
-    print('the stats: {}'.format(get_pretty_stats(perf)))
+    log.info('the stats: {}'.format(get_pretty_stats(perf)))
 
     # The base currency of the algo exchange
     base_currency = context.exchanges.values()[0].base_currency.upper()
@@ -111,15 +114,31 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=1,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=None,
-        exchange_name='poloniex',
-        live=True,
-        algo_namespace='simple_loop',
-        base_currency='eth',
-        live_graph=False,
-        simulate_orders=True
-    )
+    mode = 'backtest'
+
+    if mode == 'backtest':
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='poloniex',
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            data_frequency='minute',
+            start=pd.to_datetime('2017-9-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='binance',
+            live=True,
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True
+        )

catalyst/examples/simple_universe.py

@@ -35,8 +35,8 @@ import numpy as np
 import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.api import (symbols, )
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
 
 
 def initialize(context):

catalyst/examples/talib_simple.py

@@ -23,7 +23,7 @@ from catalyst.api import (
     order_target_percent,
     symbol,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 
 algo_namespace = 'talib_sample'
 log = Logger(algo_namespace)

catalyst/exchange/asset_finder_exchange.py

@@ -1,99 +0,0 @@
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-
-log = Logger('AssetFinderExchange', level=LOG_LEVEL)
-
-
-class AssetFinderExchange(object):
-    def __init__(self):
-        self._asset_cache = {}
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        return list()
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        # for sid in sids:
-        #     if sid in self._asset_cache:
-        #         log.debug('got asset from cache: {}'.format(sid))
-        #     else:
-        #         log.debug('fetching asset: {}'.format(sid))
-        return list()
-
-    def lookup_symbol(self, symbol, exchange, data_frequency=None,
-                      as_of_date=None, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
-
-        if data_frequency is not None:
-            key = ','.join([exchange.name, symbol, data_frequency])
-
-        else:
-            key = ','.join([exchange.name, symbol])
-
-        if key in self._asset_cache:
-            return self._asset_cache[key]
-        else:
-            asset = exchange.get_asset(symbol, data_frequency)
-            self._asset_cache[key] = asset
-            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,709 +0,0 @@
-import base64
-import datetime
-import hashlib
-import hmac
-import json
-import re
-import time
-
-import numpy as np
-import pandas as pd
-import pytz
-import requests
-import six
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
-
-# Trying to account for REST api instability
-# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-from catalyst.utils.deprecate import deprecated
-
-requests.adapters.DEFAULT_RETRIES = 20
-
-BITFINEX_URL = 'https://api.bitfinex.com'
-
-log = Logger('Bitfinex', level=LOG_LEVEL)
-warning_logger = Logger('AlgoWarning')
-
-
-@deprecated
-class Bitfinex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.url = BITFINEX_URL
-        self.key = key
-        self.secret = secret.encode('UTF-8')
-        self.name = 'bitfinex'
-        self.color = 'green'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-
-        # The candle limit for each request
-        self.num_candles_limit = 1000
-
-        # Max is 90 but playing it safe
-        # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 80
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def _request(self, operation, data, version='v1'):
-        payload_object = {
-            'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 1000000),
-            # convert to string
-            'options': {}
-        }
-
-        if data is None:
-            payload_dict = payload_object
-        else:
-            payload_dict = payload_object.copy()
-            payload_dict.update(data)
-
-        payload_json = json.dumps(payload_dict)
-        if six.PY3:
-            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
-        else:
-            payload = base64.b64encode(payload_json)
-
-        m = hmac.new(self.secret, payload, hashlib.sha384)
-        m = m.hexdigest()
-
-        # headers
-        headers = {
-            'X-BFX-APIKEY': self.key,
-            'X-BFX-PAYLOAD': payload,
-            'X-BFX-SIGNATURE': m
-        }
-
-        if data is None:
-            request = requests.get(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ), data={},
-                headers=headers)
-        else:
-            request = requests.post(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ),
-                headers=headers)
-
-        return request
-
-    def _get_v2_symbol(self, asset):
-        pair = asset.symbol.split('_')
-        symbol = 't' + pair[0].upper() + pair[1].upper()
-        return symbol
-
-    def _get_v2_symbols(self, assets):
-        """
-        Workaround to support Bitfinex v2
-        TODO: Might require a separate asset dictionary
-
-        :param assets:
-        :return:
-        """
-
-        v2_symbols = []
-        for asset in assets:
-            v2_symbols.append(self._get_v2_symbol(asset))
-
-        return v2_symbols
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from a Bitfinex order dictionary
-        :param order_status:
-        :return: Order
-        """
-        if order_status['is_cancelled']:
-            status = ORDER_STATUS.CANCELLED
-        elif not order_status['is_live']:
-            log.info('found executed order {}'.format(order_status))
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['original_amount'])
-        filled = float(order_status['executed_amount'])
-
-        if order_status['side'] == 'sell':
-            amount = -amount
-            filled = -filled
-
-        price = float(order_status['price'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        if order_type.endswith('limit'):
-            limit_price = price
-        elif order_type.endswith('stop'):
-            stop_price = price
-
-        executed_price = float(order_status['avg_execution_price'])
-
-        # TODO: bitfinex does not specify comission.
-        # I could calculate it but not sure if it's worth it.
-        commission = None
-
-        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        date = pytz.utc.localize(date)
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['id']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        log.debug('retrieving wallets balances')
-        try:
-            self.ask_request()
-            response = self._request('balances', None)
-            balances = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['message'])
-            )
-
-        std_balances = dict()
-        for balance in balances:
-            currency = balance['currency'].lower()
-            std_balances[currency] = float(balance['available'])
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Bitfinex
-
-        :param data_frequency:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
-         '1M'
-        """
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
-                               '360T', '720T', '1D', '7D', '14D', '30D']
-        if freq not in allowed_frequencies:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
-        if freq_match:
-            number = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-            if unit == 'T':
-                if number in [60, 180, 360, 720]:
-                    number = number / 60
-                    converted_unit = 'h'
-                else:
-                    converted_unit = 'm'
-            else:
-                converted_unit = unit
-
-            frequency = '{}{}'.format(number, converted_unit)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-        for asset in asset_list:
-            symbol = self._get_v2_symbol(asset)
-            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
-                url=self.url,
-                frequency=frequency,
-                symbol=symbol
-            )
-
-            if bar_count:
-                is_list = True
-                url += '/hist?limit={}'.format(int(bar_count))
-
-                def get_ms(date):
-                    epoch = datetime.datetime.utcfromtimestamp(0)
-                    epoch = epoch.replace(tzinfo=pytz.UTC)
-
-                    return (date - epoch).total_seconds() * 1000.0
-
-                if start_dt is not None:
-                    start_ms = get_ms(start_dt)
-                    url += '&start={0:f}'.format(start_ms)
-
-                if end_dt is not None:
-                    end_ms = get_ms(end_dt)
-                    url += '&end={0:f}'.format(end_ms)
-
-            else:
-                is_list = False
-                url += '/last'
-
-            try:
-                self.ask_request()
-                response = requests.get(url)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response.content:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            candles = response.json()
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(
-                    candle[0] / 1000.0)
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-                ohlc = dict(
-                    open=np.float64(candle[1]),
-                    high=np.float64(candle[3]),
-                    low=np.float64(candle[4]),
-                    close=np.float64(candle[2]),
-                    volume=np.float64(candle[5]),
-                    price=np.float64(candle[2]),
-                    last_traded=last_traded
-                )
-                return ohlc
-
-            if is_list:
-                ohlc_bars = []
-                # We can to list candles from old to new
-                for candle in reversed(candles):
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-            else:
-                ohlc = ohlc_from_candle(candles)
-                ohlc_map[asset] = ohlc
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-        if isinstance(style, ExchangeLimitOrder) \
-                or isinstance(style, ExchangeStopLimitOrder):
-            price = style.get_limit_price(is_buy)
-            order_type = 'limit'
-
-        elif isinstance(style, ExchangeStopOrder):
-            price = style.get_stop_price(is_buy)
-            order_type = 'stop'
-
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-        req = dict(
-            symbol=exchange_symbol,
-            amount=str(float(abs(amount))),
-            price="{:.20f}".format(float(price)),
-            side='buy' if is_buy else 'sell',
-            type='exchange ' + order_type,  # TODO: support margin trades
-            exchange=self.name,
-            is_hidden=False,
-            is_postonly=False,
-            use_all_available=0,
-            ocoorder=False,
-            buy_price_oco=0,
-            sell_price_oco=0
-        )
-
-        date = pd.Timestamp.utcnow()
-        try:
-            self.ask_request()
-            response = self._request('order/new', req)
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='unable to create Bitfinex order {}'.format(
-                    order_status['message'])
-            )
-
-        order_id = str(order_status['id'])
-        order = Order(
-            dt=date,
-            asset=asset,
-            amount=amount,
-            stop=style.get_stop_price(is_buy),
-            limit=style.get_limit_price(is_buy),
-            id=order_id
-        )
-
-        return order
-
-    def get_open_orders(self, asset=None):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        try:
-            self.ask_request()
-            response = self._request('orders', None)
-            order_statuses = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_statuses:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        orders = []
-        for order_status in order_statuses:
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-        try:
-            self.ask_request()
-            response = self._request(
-                'order/status', {'order_id': int(order_id)})
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='Unable to retrieve order status: {}'.format(
-                    order_status['message'])
-            )
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-
-        try:
-            self.ask_request()
-            response = self._request('order/cancel', {'order_id': order_id})
-            status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self._get_v2_symbols(assets)
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            self.ask_request()
-            response = requests.get(
-                '{url}/v2/tickers?symbols={symbols}'.format(
-                    url=self.url,
-                    symbols=','.join(symbols),
-                )
-            )
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response.content:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response.content)
-            )
-
-        try:
-            tickers = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        ticks = dict()
-        for index, ticker in enumerate(tickers):
-            if not len(ticker) == 11:
-                raise ExchangeRequestError(
-                    error='Invalid ticker in response: {}'.format(ticker)
-                )
-
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker[1],
-                ask=ticker[3],
-                last_price=ticker[7],
-                low=ticker[10],
-                high=ticker[9],
-                volume=ticker[8],
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self._request('symbols', None)
-
-        for symbol in response.json():
-            if (source_dates):
-                start_date = self.get_symbol_start_date(symbol)
-            else:
-                try:
-                    start_date = cached_symbols[symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[symbol] = dict(
-                symbol=symbol[:-3] + '_' + symbol[-3:],
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-
-        print(symbol)
-        symbol_v2 = 't' + symbol.upper()
-
-        """
-            For each symbol we retrieve candles with Monhtly resolution
-            We get the first month, and query again with daily resolution
-            around that date, and we get the first date
-        """
-        url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
-            url=self.url,
-            symbol=symbol_v2
-        )
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        """
-            If we don't get any data back for our monthly-resolution query
-            it means that symbol started trading less than a month ago, so
-            arbitrarily set the ref. date to 15 days ago to be safe with
-            +/- 31 days
-        """
-        if (len(response.json())):
-            startmonth = response.json()[-1][0]
-        else:
-            startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
-
-        """
-            Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than
-            now: time.time()
-        """
-        url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
-               '&end={end}').format(
-            url=self.url,
-            symbol=symbol_v2,
-            start=startmonth - 3600 * 24 * 31 * 1000,
-            end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000)))
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d',
-                             time.gmtime(int(response.json()[-1][0] / 1000)))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        exchange_symbol = asset.exchange_symbol
-        try:
-            self.ask_request()
-            # TODO: implement limit
-            response = self._request(
-                'book/{}'.format(exchange_symbol), None)
-            data = response.json()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        # TODO: filter by type
-        result = dict()
-        for order_type in data:
-            result[order_type] = []
-
-            for entry in data[order_type]:
-                result[order_type].append(dict(
-                    rate=float(entry['price']),
-                    quantity=float(entry['amount'])
-                ))
-
-        return result

catalyst/exchange/bitfinex/symbols.json

@@ -1,127 +0,0 @@
-{
-  "neobtc": {
-    "symbol": "neo_btc",
-    "start_date": "2017-09-07",
-    "precision": 5
-  },
-  "neousd": {
-    "symbol": "neo_usd",
-    "start_date": "2017-09-07"
-  },
-  "neoeth": {
-    "symbol": "neo_eth",
-    "start_date": "2017-09-07"
-  },
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bchusd": {
-    "symbol": "bch_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2017-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcbtc": {
-    "symbol": "etc_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcusd": {
-    "symbol": "etc_usd",
-    "start_date": "2017-01-01"
-  },
-  "rrtusd": {
-    "symbol": "rrt_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtbtc": {
-    "symbol": "rrt_btc",
-    "start_date": "2010-01-01"
-  },
-  "zecusd": {
-    "symbol": "zec_usd",
-    "start_date": "2010-01-01"
-  },
-  "zecbtc": {
-    "symbol": "zec_btc",
-    "start_date": "2010-01-01"
-  },
-  "xmrusd": {
-    "symbol": "xmr_usd",
-    "start_date": "2010-01-01"
-  },
-  "xmrbtc": {
-    "symbol": "xmr_btc",
-    "start_date": "2010-01-01"
-  },
-  "dshusd": {
-    "symbol": "dsh_usd",
-    "start_date": "2010-01-01"
-  },
-  "dshbtc": {
-    "symbol": "dsh_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccbtc": {
-    "symbol": "bcc_btc",
-    "start_date": "2010-01-01"
-  },
-  "bcubtc": {
-    "symbol": "bcu_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccusd": {
-    "symbol": "bcc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bcuusd": {
-    "symbol": "bcu_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpusd": {
-    "symbol": "xrp_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpbtc": {
-    "symbol": "xrp_btc",
-    "start_date": "2010-01-01"
-  },
-  "iotusd": {
-    "symbol": "iot_usd",
-    "start_date": "2010-01-01"
-  },
-  "iotbtc": {
-    "symbol": "iot_btc",
-    "start_date": "2010-01-01"
-  },
-  "ioteth": {
-    "symbol": "iot_eth",
-    "start_date": "2010-01-01"
-  },
-  "eosusd": {
-    "symbol": "eos_usd",
-    "start_date": "2010-01-01"
-  },
-  "eosbtc": {
-    "symbol": "eos_btc",
-    "start_date": "2010-01-01"
-  },
-  "eoseth": {
-    "symbol": "eos_eth",
-    "start_date": "2010-01-01"
-  }
-}

catalyst/exchange/bittrex/bittrex.py

@@ -1,417 +0,0 @@
-import json
-import time
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from six.moves import urllib
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-    ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
-    CreateOrderError
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-
-# TODO: consider using this: https://github.com/mondeja/bittrex_v2
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Bittrex', level=LOG_LEVEL)
-
-URL2 = 'https://bittrex.com/Api/v2.0'
-
-
-@deprecated
-class Bittrex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret)
-        self.name = 'bittrex'
-        self.color = 'blue'
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-
-        self.num_candles_limit = 2000
-
-        # Not sure what the rate limit is but trying to play it safe
-        # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.minute_writer = None
-        self.minute_reader = None
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.bundle = ExchangeBundle(self.name)
-
-    @property
-    def account(self):
-        pass
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def get_balances(self):
-        balances = self.api.getbalances()
-        try:
-            log.debug('retrieving wallet balances')
-            self.ask_request()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        std_balances = dict()
-        try:
-            for balance in balances:
-                currency = balance['Currency'].lower()
-                std_balances[currency] = balance['Available']
-
-        except TypeError:
-            raise ExchangeRequestError(error=balances)
-
-        return std_balances
-
-    def create_order(self, asset, amount, is_buy, style):
-        log.info('creating {} order'.format('buy' if is_buy else 'sell'))
-        exchange_symbol = self.get_symbol(asset)
-
-        if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
-            if isinstance(style, StopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-            try:
-                self.ask_request()
-                if is_buy:
-                    order_status = self.api.buylimit(exchange_symbol, amount,
-                                                     price)
-                else:
-                    order_status = self.api.selllimit(exchange_symbol,
-                                                      abs(amount), price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'uuid' in order_status:
-                order_id = order_status['uuid']
-                order = Order(
-                    dt=pd.Timestamp.utcnow(),
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                if order_status == 'INSUFFICIENT_FUNDS':
-                    log.warn('not enough funds to create order')
-                    return None
-                elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
-                    log.warn('Your order is too small, order at least 50K'
-                             ' Satoshi')
-                    return None
-                else:
-                    raise CreateOrderError(
-                        exchange=self.name,
-                        error=order_status
-                    )
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset):
-        symbol = self.get_symbol(asset)
-        try:
-            self.ask_request()
-            open_orders = self.api.getopenorders(symbol)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        orders = list()
-        for order_status in open_orders:
-            order = self._create_order(order_status)
-            orders.append(order)
-
-        return orders
-
-    def _create_order(self, order_status):
-        log.info(
-            'creating catalyst order from Bittrex {}'.format(order_status))
-        if order_status['CancelInitiated']:
-            status = ORDER_STATUS.CANCELLED
-        elif order_status['Closed'] is not None:
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        date = pd.to_datetime(order_status['Opened'], utc=True)
-        amount = order_status['Quantity']
-        filled = amount - order_status['QuantityRemaining']
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['Exchange']],
-            amount=amount,
-            stop=None,  # Not yet supported by Bittrex
-            limit=order_status['Limit'],
-            filled=filled,
-            id=order_status['OrderUuid'],
-            commission=order_status['CommissionPaid']
-        )
-        order.status = status
-
-        executed_price = order_status['PricePerUnit']
-
-        return order, executed_price
-
-    def get_order(self, order_id):
-        log.info('retrieving order {}'.format(order_id))
-        try:
-            self.ask_request()
-            order_status = self.api.getorder(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if order_status is None:
-            raise OrderNotFound(order_id=order_id, exchange=self.name)
-
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-        log.info('cancelling order {}'.format(order_id))
-
-        try:
-            self.ask_request()
-            status = self.api.cancel(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Supported Intervals
-        -------------------
-        day, oneMin, fiveMin, thirtyMin, hour
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :param start_dt
-        :param end_dt
-        :return:
-        """
-
-        # TODO: this has no effect at the moment
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T':
-            frequency = 'oneMin'
-        elif freq == '5T':
-            frequency = 'fiveMin'
-        elif freq == '30T':
-            frequency = 'thirtyMin'
-        elif freq == '60T':
-            frequency = 'hour'
-        elif freq == '1D':
-            frequency = 'day'
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        for asset in asset_list:
-            end = int(time.mktime(end_dt.timetuple()))
-            url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_={end}'.format(
-                    url=URL2,
-                    symbol=self.get_symbol(asset),
-                    frequency=frequency,
-                    end=end, )
-
-            try:
-                data = json.loads(urllib.request.urlopen(url).read().decode())
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if data['message']:
-                raise ExchangeRequestError(
-                    error='Unable to fetch candles {}'.format(data['message'])
-                )
-
-            candles = data['result']
-
-            def ohlc_from_candle(candle):
-                ohlc = dict(
-                    open=candle['O'],
-                    high=candle['H'],
-                    low=candle['L'],
-                    close=candle['C'],
-                    volume=candle['V'],
-                    price=candle['C'],
-                    last_traded=pd.to_datetime(candle['T'], utc=True)
-                )
-                return ohlc
-
-            ordered_candles = list(reversed(candles))
-            ohlc_map = dict()
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
-            else:
-                # TODO: optimize
-                ohlc_bars = []
-                for candle in ordered_candles[:bar_count]:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def tickers(self, assets):
-        """
-        As of v1.1, Bittrex only allows one ticker at the time.
-        So we have to make multiple calls to fetch multiple assets.
-
-        :param assets:
-        :return:
-        """
-        log.info('retrieving tickers')
-
-        ticks = dict()
-        for asset in assets:
-            symbol = self.get_symbol(asset)
-            try:
-                self.ask_request()
-                ticker = self.api.getticker(symbol)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            # TODO: catch invalid ticker
-            ticks[asset] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker['Bid'],
-                ask=ticker['Ask'],
-                last_price=ticker['Last']
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def generate_symbols_json(self, filename=None):
-        symbol_map = {}
-
-        fn, r = download_exchange_symbols(self.name)
-        with open(fn) as data_file:
-            cached_symbols = json.load(data_file)
-
-        markets = self.api.getmarkets()
-        for market in markets:
-            exchange_symbol = market['MarketName']
-            symbol = '{market}_{base}'.format(
-                market=self.sanitize_curency_symbol(market['MarketCurrency']),
-                base=self.sanitize_curency_symbol(market['BaseCurrency'])
-            )
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=pd.to_datetime(market['Created'],
-                                          utc=True).strftime("%Y-%m-%d"),
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        if order_type == 'all':
-            order_type = 'both'
-        elif order_type == 'bid':
-            order_type = 'buy'
-        elif order_type == 'ask':
-            order_type = 'sell'
-        else:
-            raise ValueError('invalid type')
-
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(
-            market=exchange_symbol,
-            type=order_type,
-            depth=100
-        )
-
-        result = dict()
-        for exchange_type in data:
-            if exchange_type == 'buy':
-                order_type = 'bids'
-            elif exchange_type == 'sell':
-                order_type = 'asks'
-
-            result[order_type] = []
-            for entry in data[exchange_type]:
-                result[order_type].append(dict(
-                    rate=entry['Rate'],
-                    quantity=entry['Quantity']
-                ))
-
-        return result

catalyst/exchange/bittrex/bittrex_api.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-from six.moves import urllib
-
-urlopen = urllib.request.urlopen
-
-
-class Bittrex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-        self.public = ['getmarkets', 'getcurrencies', 'getticker',
-                       'getmarketsummaries', 'getmarketsummary',
-                       'getorderbook', 'getmarkethistory']
-        self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
-                       'cancel', 'getopenorders']
-        self.account = ['getbalances', 'getbalance', 'getdepositaddress',
-                        'withdraw', 'getorder', 'getorderhistory',
-                        'getwithdrawalhistory', 'getdeposithistory']
-
-    def query(self, method, values={}):
-        if method in self.public:
-            url = 'https://bittrex.com/api/v1.1/public/'
-        elif method in self.market:
-            url = 'https://bittrex.com/api/v1.1/market/'
-        elif method in self.account:
-            url = 'https://bittrex.com/api/v1.1/account/'
-        else:
-            return 'Something went wrong, sorry.'
-
-        url += method + '?' + urllib.parse.urlencode(values)
-
-        if method not in self.public:
-            url += '&apikey=' + self.key
-            url += '&nonce=' + str(int(time.time()))
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 url.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'apisign': signature}
-        else:
-            headers = {}
-
-        req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(
-            req, context=ssl._create_unverified_context()).read())
-
-        if response["result"]:
-            return response["result"]
-        else:
-            return response["message"]
-
-    def getmarkets(self):
-        return self.query('getmarkets')
-
-    def getcurrencies(self):
-        return self.query('getcurrencies')
-
-    def getticker(self, market):
-        return self.query('getticker', {'market': market})
-
-    def getmarketsummaries(self):
-        return self.query('getmarketsummaries')
-
-    def getmarketsummary(self, market):
-        return self.query('getmarketsummary', {'market': market})
-
-    def getorderbook(self, market, type, depth=20):
-        return self.query('getorderbook',
-                          {'market': market, 'type': type, 'depth': depth})
-
-    def getmarkethistory(self, market, count=20):
-        return self.query('getmarkethistory',
-                          {'market': market, 'count': count})
-
-    def buylimit(self, market, quantity, rate):
-        return self.query('buylimit', {'market': market, 'quantity': quantity,
-                                       'rate': rate})
-
-    def buymarket(self, market, quantity):
-        return self.query('buymarket',
-                          {'market': market, 'quantity': quantity})
-
-    def selllimit(self, market, quantity, rate):
-        return self.query('selllimit', {'market': market, 'quantity': quantity,
-                                        'rate': rate})
-
-    def sellmarket(self, market, quantity):
-        return self.query('sellmarket',
-                          {'market': market, 'quantity': quantity})
-
-    def cancel(self, uuid):
-        return self.query('cancel', {'uuid': uuid})
-
-    def getopenorders(self, market):
-        return self.query('getopenorders', {'market': market})
-
-    def getbalances(self):
-        return self.query('getbalances')
-
-    def getbalance(self, currency):
-        return self.query('getbalance', {'currency': currency})
-
-    def getdepositaddress(self, currency):
-        return self.query('getdepositaddress', {'currency': currency})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'quantity': quantity,
-                           'address': address})
-
-    def getorder(self, uuid):
-        return self.query('getorder', {'uuid': uuid})
-
-    def getorderhistory(self, market, count):
-        return self.query('getorderhistory',
-                          {'market': market, 'count': count})
-
-    def getwithdrawalhistory(self, currency, count):
-        return self.query('getwithdrawalhistory',
-                          {'currency': currency, 'count': count})
-
-    def getdeposithistory(self, currency, count):
-        return self.query('getdeposithistory',
-                          {'currency': currency, 'count': count})

catalyst/exchange/bittrex/extensions-example.py

@@ -1,7 +0,0 @@
-from catalyst.data.bundles import register
-from catalyst.exchange.exchange_bundle import exchange_bundle
-
-symbols = (
-    'neo_btc',
-)
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -1,24 +1,27 @@
+import json
+import os
 import re
 from collections import defaultdict
 
 import ccxt
 import pandas as pd
 import six
+from catalyst.assets._assets import TradingPair
 from ccxt import ExchangeNotAvailable, InvalidOrder
 from logbook import Logger
 from six import string_types
 
 from catalyst.algorithm import MarketOrder
-from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
-    ExchangeNotFoundError, CreateOrderError
+    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-from catalyst.exchange.exchange_utils import mixin_market_params, \
-    from_ms_timestamp, get_epoch
+from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
+    from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
+    get_exchange_auth
 from catalyst.finance.order import Order, ORDER_STATUS
 
 log = Logger('CCXT', level=LOG_LEVEL)
@@ -58,26 +61,103 @@ class CCXT(Exchange):
 
         self._symbol_maps = [None, None]
 
-        try:
-            markets_symbols = self.api.load_markets()
-            log.debug('the markets:\n{}'.format(markets_symbols))
-
-        except ExchangeNotAvailable as e:
-            raise ExchangeRequestError(error=e)
-
         self.name = exchange_name
 
-        self.markets = self.api.fetch_markets()
-        self.load_assets()
-
         self.base_currency = base_currency
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
         self.max_requests_per_minute = 60
+        self.low_balance_threshold = 0.1
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self.name)
+        self.markets = None
+        self._is_init = False
+
+    def init(self):
+        if self._is_init:
+            return
+
+        exchange_folder = get_exchange_folder(self.name)
+        filename = os.path.join(exchange_folder, 'cctx_markets.json')
+
+        if os.path.exists(filename):
+            timestamp = os.path.getmtime(filename)
+            dt = pd.to_datetime(timestamp, unit='s', utc=True)
+
+            if dt >= pd.Timestamp.utcnow().floor('1D'):
+                with open(filename) as f:
+                    self.markets = json.load(f)
+
+                log.debug('loaded markets for {}'.format(self.name))
+
+        if self.markets is None:
+            try:
+                markets_symbols = self.api.load_markets()
+                log.debug(
+                    'fetching {} markets:\n{}'.format(
+                        self.name, markets_symbols
+                    )
+                )
+
+                self.markets = self.api.fetch_markets()
+                with open(filename, 'w+') as f:
+                    json.dump(self.markets, f, indent=4)
+
+            except ExchangeNotAvailable as e:
+                raise ExchangeRequestError(error=e)
+
+        self.load_assets()
+        self._is_init = True
+
+    @staticmethod
+    def find_exchanges(features=None, is_authenticated=False):
+        ccxt_features = []
+        if features is not None:
+            for feature in features:
+                if not feature.endswith('Bundle'):
+                    ccxt_features.append(feature)
+
+        exchange_names = []
+        for exchange_name in ccxt.exchanges:
+            if is_authenticated:
+                exchange_auth = get_exchange_auth(exchange_name)
+
+                has_auth = (exchange_auth['key'] != ''
+                            and exchange_auth['secret'] != '')
+
+                if not has_auth:
+                    continue
+
+            log.debug('loading exchange: {}'.format(exchange_name))
+            exchange = getattr(ccxt, exchange_name)()
+
+            if ccxt_features is None:
+                has_feature = True
+
+            else:
+                try:
+                    has_feature = all(
+                        [exchange.has[feature] for feature in ccxt_features]
+                    )
+
+                except Exception:
+                    has_feature = False
+
+            if has_feature:
+                try:
+                    log.info('initializing {}'.format(exchange_name))
+                    exchange_names.append(exchange_name)
+
+                except Exception as e:
+                    log.warn(
+                        'unable to initialize exchange {}: {}'.format(
+                            exchange_name, e
+                        )
+                    )
+
+        return exchange_names
 
     def account(self):
         return None
@@ -85,6 +165,30 @@ class CCXT(Exchange):
     def time_skew(self):
         return None
 
+    def get_candle_frequencies(self, data_frequency=None):
+        frequencies = []
+        try:
+            for timeframe in self.api.timeframes:
+                freq = CCXT.get_frequency(timeframe, raise_error=False)
+
+                # TODO: support all frequencies
+                if data_frequency == 'minute' and not freq.endswith('T'):
+                    continue
+
+                elif data_frequency == 'daily' and not freq.endswith('D'):
+                    continue
+
+                frequencies.append(freq)
+
+        except Exception as e:
+            log.warn(
+                'candle frequencies not available for exchange {}'.format(
+                    self.name
+                )
+            )
+
+        return frequencies
+
     def get_market(self, symbol):
         """
         The CCXT market.
@@ -106,7 +210,7 @@ class CCXT(Exchange):
         )
         return market
 
-    def get_symbol(self, asset_or_symbol):
+    def get_symbol(self, asset_or_symbol, source='catalyst'):
         """
         The CCXT symbol.
 
@@ -118,36 +222,109 @@ class CCXT(Exchange):
         -------
 
         """
-        symbol = asset_or_symbol if isinstance(
-            asset_or_symbol, string_types
-        ) else asset_or_symbol.symbol
 
-        parts = symbol.split('_')
-        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+        if source == 'ccxt':
+            if isinstance(asset_or_symbol, string_types):
+                parts = asset_or_symbol.split('/')
+                return '{}_{}'.format(parts[0].lower(), parts[1].lower())
 
-    def get_catalyst_symbol(self, market_or_symbol):
+            else:
+                return asset_or_symbol.symbol
+
+        else:
+            symbol = asset_or_symbol if isinstance(
+                asset_or_symbol, string_types
+            ) else asset_or_symbol.symbol
+
+            parts = symbol.split('_')
+            return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+
+    @staticmethod
+    def map_frequency(value, source='ccxt', raise_error=True):
         """
-        The Catalyst symbol.
+        Map a frequency value between CCXT and Catalyst
 
         Parameters
         ----------
-        market_or_symbol
+        value: str
+        source: str
+        raise_error: bool
 
         Returns
         -------
 
+        Notes
+        -----
+        The Pandas offset aliases supported by Catalyst:
+        Alias	Description
+        W	weekly frequency
+        M	month end frequency
+        D	calendar day frequency
+        H	hourly frequency
+        T, min	minutely frequency
+
+        The CCXT timeframes:
+        '1m': '1minute',
+        '1h': '1hour',
+        '1d': '1day',
+        '1w': '1week',
+        '1M': '1month',
+        '1y': '1year',
         """
-        if isinstance(market_or_symbol, string_types):
-            parts = market_or_symbol.split('/')
-            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+        match = re.match(
+            r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value, re.M | re.I
+        )
+        if match:
+            candle_size = int(match.group(1)) \
+                if match.group(1) else 1
+
+            unit = match.group(2)
 
         else:
-            return '{}_{}'.format(
-                market_or_symbol['base'].lower(),
-                market_or_symbol['quote'].lower(),
-            )
+            raise ValueError('Unable to parse frequency or timeframe')
 
-    def get_timeframe(self, freq):
+        if source == 'ccxt':
+            if unit == 'd':
+                result = '{}D'.format(candle_size)
+
+            elif unit == 'm':
+                result = '{}T'.format(candle_size)
+
+            elif unit == 'h':
+                result = '{}H'.format(candle_size)
+
+            elif unit == 'w':
+                result = '{}W'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryTimeframeError(timeframe=value)
+
+        else:
+            if unit == 'D':
+                result = '{}d'.format(candle_size)
+
+            elif unit == 'min' or unit == 'T':
+                result = '{}m'.format(candle_size)
+
+            elif unit == 'H':
+                result = '{}h'.format(candle_size)
+
+            elif unit == 'W':
+                result = '{}w'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryFrequencyError(frequency=value)
+
+        return result
+
+    @staticmethod
+    def get_timeframe(freq, raise_error=True):
         """
         The CCXT timeframe from the Catalyst frequency.
 
@@ -161,26 +338,29 @@ class CCXT(Exchange):
         str
 
         """
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) \
-                if freq_match.group(1) else 1
+        return CCXT.map_frequency(
+            freq, source='catalyst', raise_error=raise_error
+        )
 
-            unit = freq_match.group(2)
+    @staticmethod
+    def get_frequency(timeframe, raise_error=True):
+        """
+        Test Catalyst frequency from the CCXT timeframe
 
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
+        Catalyst uses the Pandas offset alias convention:
+        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
 
-        if unit.lower() == 'd':
-            timeframe = '{}d'.format(candle_size)
+        Parameters
+        ----------
+        timeframe
 
-        elif unit.lower() == 'm' or unit == 'T':
-            timeframe = '{}m'.format(candle_size)
+        Returns
+        -------
 
-        elif unit.lower() == 'h' or unit == 'T':
-            timeframe = '{}h'.format(candle_size)
-
-        return timeframe
+        """
+        return CCXT.map_frequency(
+            timeframe, source='ccxt', raise_error=raise_error
+        )
 
     def get_candles(self, freq, assets, bar_count=None, start_dt=None,
                     end_dt=None):
@@ -189,7 +369,7 @@ class CCXT(Exchange):
             assets = [assets]
 
         symbols = self.get_symbols(assets)
-        timeframe = self.get_timeframe(freq)
+        timeframe = CCXT.get_timeframe(freq)
 
         ms = None
         if start_dt is not None:
@@ -198,30 +378,26 @@ class CCXT(Exchange):
 
         candles = dict()
         for asset in assets:
-            try:
-                ohlcvs = self.api.fetch_ohlcv(
-                    symbol=symbols[0],
-                    timeframe=timeframe,
-                    since=ms,
-                    limit=bar_count,
-                    params={}
-                )
+            ohlcvs = self.api.fetch_ohlcv(
+                symbol=symbols[0],
+                timeframe=timeframe,
+                since=ms,
+                limit=bar_count,
+                params={}
+            )
 
-                candles[asset] = []
-                for ohlcv in ohlcvs:
-                    candles[asset].append(dict(
-                        last_traded=pd.to_datetime(
-                            ohlcv[0], unit='ms', utc=True
-                        ),
-                        open=ohlcv[1],
-                        high=ohlcv[2],
-                        low=ohlcv[3],
-                        close=ohlcv[4],
-                        volume=ohlcv[5]
-                    ))
-
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
+            candles[asset] = []
+            for ohlcv in ohlcvs:
+                candles[asset].append(dict(
+                    last_traded=pd.to_datetime(
+                        ohlcv[0], unit='ms', utc=True
+                    ),
+                    open=ohlcv[1],
+                    high=ohlcv[2],
+                    low=ohlcv[3],
+                    close=ohlcv[4],
+                    volume=ohlcv[5]
+                ))
 
         if is_single:
             return six.next(six.itervalues(candles))
@@ -339,16 +515,21 @@ class CCXT(Exchange):
                    and asset_def['end_minute'] != 'N/A' else None
 
         else:
-            params['symbol'] = self.get_catalyst_symbol(market)
+            params['symbol'] = get_catalyst_symbol(market)
             # TODO: add as an optional column
             params['leverage'] = 1.0
 
         return TradingPair(**params)
 
     def load_assets(self):
+        log.debug('loading assets for {}'.format(self.name))
         self.assets = []
 
         for market in self.markets:
+            if 'id' not in market:
+                log.warn('invalid market: {}'.format(market))
+                continue
+
             asset_defs = self.get_asset_defs(market)
 
             asset = None
@@ -399,21 +580,61 @@ class CCXT(Exchange):
             The Catalyst order object
 
         """
-        if order_status['status'] == 'canceled':
+        order_id = order_status['id']
+        symbol = self.get_symbol(order_status['symbol'], source='ccxt')
+        asset = self.get_asset(symbol)
+
+        s = order_status['status']
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if s == 'canceled' or (s == 'closed' and filled == 0):
             status = ORDER_STATUS.CANCELLED
 
-        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
-            log.debug('found executed order {}'.format(order_status))
+        elif s == 'closed' and filled > 0:
+            if filled < amount:
+                log.warn(
+                    'order {id} is executed but only partially filled:'
+                    ' {filled} {symbol} out of {amount}'.format(
+                        id=order_status['status'],
+                        filled=order_status['filled'],
+                        symbol=asset.symbol,
+                        amount=order_status['amount'],
+                    )
+                )
+            else:
+                log.info(
+                    'order {id} executed in full: {filled} {symbol}'.format(
+                        id=order_id,
+                        filled=filled,
+                        symbol=asset.symbol,
+                    )
+                )
+
             status = ORDER_STATUS.FILLED
 
-        elif order_status['status'] == 'open':
+        elif s == 'open':
+            status = ORDER_STATUS.OPEN
+
+        elif filled > 0:
+            log.info(
+                'order {id} partially filled: {filled} {symbol} out of '
+                '{amount}, waiting for complete execution'.format(
+                    id=order_id,
+                    filled=filled,
+                    symbol=asset.symbol,
+                    amount=amount,
+                )
+            )
             status = ORDER_STATUS.OPEN
 
         else:
-            raise ValueError('invalid state for order')
-
-        amount = order_status['amount']
-        filled = order_status['filled']
+            log.warn(
+                'invalid state {} for order {}'.format(
+                    s, order_id
+                )
+            )
+            status = ORDER_STATUS.OPEN
 
         if order_status['side'] == 'sell':
             amount = -amount
@@ -423,25 +644,16 @@ class CCXT(Exchange):
         order_type = order_status['type']
 
         limit_price = price if order_type == 'limit' else None
-        stop_price = None  # TODO: add support
 
         executed_price = order_status['cost'] / order_status['amount']
         commission = order_status['fee']
         date = from_ms_timestamp(order_status['timestamp'])
 
-        # order_id = str(order_status['info']['clientOrderId'])
-        order_id = order_status['id']
-
-        # TODO: this won't work, redo the packages with a different key.
-        symbol = order_status['info']['symbol'] \
-            if 'symbol' in order_status['info'] \
-            else order_status['info']['Exchange']
-
         order = Order(
             dt=date,
-            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            asset=asset,
             amount=amount,
-            stop=stop_price,
+            stop=None,
             limit=limit_price,
             filled=filled,
             id=order_id,
@@ -469,7 +681,6 @@ class CCXT(Exchange):
             )
 
         side = 'buy' if amount > 0 else 'sell'
-
         if hasattr(self.api, 'amount_to_lots'):
             adj_amount = self.api.amount_to_lots(
                 symbol=symbol,
@@ -583,10 +794,19 @@ class CCXT(Exchange):
 
         """
         tickers = dict()
-        for asset in assets:
-            try:
-                ccxt_symbol = self.get_symbol(asset)
-                ticker = self.api.fetch_ticker(ccxt_symbol)
+        try:
+            for asset in assets:
+                symbol = self.get_symbol(asset)
+                # TODO: use fetch_tickers() for efficiency
+                # I tried using fetch_tickers() but noticed some
+                # inconsistencies, see issue:
+                # https://github.com/ccxt/ccxt/issues/870
+                ticker = self.api.fetch_ticker(symbol=symbol)
+                if not ticker:
+                    log.warn('ticker not found for {} {}'.format(
+                        self.name, symbol
+                    ))
+                    continue
 
                 ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
 
@@ -594,19 +814,27 @@ class CCXT(Exchange):
                     # TODO: any more exceptions?
                     ticker['last_price'] = ticker['last']
 
-                # Using the volume represented in the base currency
-                ticker['volume'] = ticker['baseVolume'] \
-                    if 'baseVolume' in ticker else 0
+                if 'baseVolume' in ticker and ticker['baseVolume'] is not None:
+                    # Using the volume represented in the base currency
+                    ticker['volume'] = ticker['baseVolume']
+
+                elif 'info' in ticker and 'bidQty' in ticker['info'] \
+                        and 'askQty' in ticker['info']:
+                    ticker['volume'] = float(ticker['info']['bidQty']) + \
+                                       float(ticker['info']['askQty'])
+
+                else:
+                    ticker['volume'] = 0
 
                 tickers[asset] = ticker
 
-            except ExchangeNotAvailable as e:
-                log.warn(
-                    'unable to fetch ticker: {} {}'.format(
-                        self.name, asset.symbol
-                    )
+        except ExchangeNotAvailable as e:
+            log.warn(
+                'unable to fetch ticker: {} {}'.format(
+                    self.name, asset.symbol
                 )
-                raise ExchangeRequestError(error=e)
+            )
+            raise ExchangeRequestError(error=e)
 
         return tickers
 

catalyst/exchange/exchange.py

@@ -9,15 +9,16 @@ from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
-from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError
-from catalyst.exchange.exchange_utils import get_exchange_symbols, \
-    get_frequency, resample_history_df
+    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
+    TickerNotFoundError, NotEnoughCashError
+from catalyst.exchange.utils.bundle_utils import get_start_dt, \
+    get_delta, get_periods, get_periods_range
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
+    get_frequency, resample_history_df, has_bundle
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -38,6 +39,8 @@ class Exchange:
         self.request_cpt = None
         self.bundle = ExchangeBundle(self.name)
 
+        self.low_balance_threshold = None
+
     @abstractproperty
     def account(self):
         pass
@@ -46,6 +49,9 @@ class Exchange:
     def time_skew(self):
         pass
 
+    def has_bundle(self, data_frequency):
+        return has_bundle(self.name, data_frequency)
+
     def is_open(self, dt):
         """
         Is the exchange open
@@ -148,7 +154,7 @@ class Exchange:
 
     def get_assets(self, symbols=None, data_frequency=None,
                    is_exchange_symbol=False,
-                   is_local=None):
+                   is_local=None, quote_currency=None):
         """
         The list of markets for the specified symbols.
 
@@ -172,6 +178,14 @@ class Exchange:
         if symbols is None:
             # Make a distinct list of all symbols
             symbols = list(set([asset.symbol for asset in self.assets]))
+
+            if quote_currency is not None:
+                for symbol in symbols[:]:
+                    suffix = '_{}'.format(quote_currency.lower())
+
+                    if not symbol.endswith(suffix):
+                        symbols.remove(symbol)
+
             is_exchange_symbol = False
 
         assets = []
@@ -235,10 +249,10 @@ class Exchange:
 
             elif data_frequency is not None:
                 applies = (
-                        (
-                                data_frequency == 'minute' and a.end_minute is not None)
-                        or (
-                                data_frequency == 'daily' and a.end_daily is not None)
+                    (
+                        data_frequency == 'minute' and a.end_minute is not None)
+                    or (
+                        data_frequency == 'daily' and a.end_daily is not None)
                 )
 
             else:
@@ -247,8 +261,16 @@ class Exchange:
             # The symbol provided may use the Catalyst or the exchange
             # convention
             key = a.exchange_symbol if is_exchange_symbol else a.symbol
-            if not asset and key.lower() == symbol.lower() and applies:
-                asset = a
+            if not asset and key.lower() == symbol.lower():
+                if applies:
+                    asset = a
+
+                else:
+                    raise NoDataAvailableOnExchange(
+                        symbol=key,
+                        exchange=self.name,
+                        data_frequency=data_frequency,
+                    )
 
         if asset is None:
             supported_symbols = sorted([a.symbol for a in self.assets])
@@ -272,6 +294,16 @@ class Exchange:
             self._symbol_maps[index] = symbol_map
             return symbol_map
 
+    @abstractmethod
+    def init(self):
+        """
+        Load the asset list from the network.
+
+        Returns
+        -------
+
+        """
+
     @abstractmethod
     def load_assets(self, is_local=False):
         """
@@ -377,6 +409,7 @@ class Exchange:
 
         return value
 
+    # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 data_frequency, field, previous_value=None):
         """
@@ -619,46 +652,98 @@ class Exchange:
 
         return df
 
-    def calculate_totals(self, check_cash=False, positions=None):
+    def _check_low_balance(self, currency, balances, amount):
+        free = balances[currency]['free'] if currency in balances else 0.0
+
+        if free < amount:
+            return free, True
+
+        else:
+            return free, False
+
+    def sync_positions(self, positions, cash=None, check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
+        Parameters
+        ----------
+        positions:
+            The positions to synchronize.
+
+        check_balances:
+            Check balances amounts against the exchange.
+
         """
-        log.debug('synchronizing portfolio with exchange {}'.format(self.name))
-
-        cash = None
-        if check_cash:
+        free_cash = 0.0
+        if check_balances:
+            log.debug('fetching {} balances'.format(self.name))
             balances = self.get_balances()
-
-            cash = balances[self.base_currency]['free'] \
-                if self.base_currency in balances else None
-
-            if cash is None:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=self.base_currency,
-                    exchange=self.name
+            log.debug(
+                'got free balances for {} currencies'.format(
+                    len(balances)
                 )
-            log.debug('found base currency balance: {}'.format(cash))
+            )
+            if cash is not None:
+                free_cash, is_lower = self._check_low_balance(
+                    currency=self.base_currency,
+                    balances=balances,
+                    amount=cash,
+                )
+                if is_lower:
+                    raise NotEnoughCashError(
+                        currency=self.base_currency,
+                        exchange=self.name,
+                        free=free_cash,
+                        cash=cash,
+                    )
 
         positions_value = 0.0
-        if positions:
+        if positions is not None:
             assets = set([position.asset for position in positions])
             tickers = self.tickers(assets)
-            log.debug('got tickers for positions: {}'.format(tickers))
 
-            for asset in tickers:
+            for position in positions:
+                asset = position.asset
+                if asset not in tickers:
+                    raise TickerNotFoundError(
+                        symbol=asset.symbol,
+                        exchange=self.name,
+                    )
+
                 ticker = tickers[asset]
-                positions = [p for p in positions if p.asset == asset]
+                log.debug(
+                    'updating {symbol} position, last traded on {dt} for '
+                    '{price}{currency}'.format(
+                        symbol=asset.symbol,
+                        dt=ticker['last_traded'],
+                        price=ticker['last_price'],
+                        currency=asset.quote_currency,
+                    )
+                )
+                position.last_sale_price = ticker['last_price']
+                position.last_sale_date = ticker['last_traded']
 
-                for position in positions:
-                    position.last_sale_price = ticker['last_price']
-                    position.last_sale_date = ticker['last_traded']
+                positions_value += \
+                    position.amount * position.last_sale_price
 
-                    positions_value += \
-                        position.amount * position.last_sale_price
+                if check_balances:
+                    free, is_lower = self._check_low_balance(
+                        currency=asset.base_currency,
+                        balances=balances,
+                        amount=position.amount,
+                    )
 
-        return cash, positions_value
+                    if is_lower:
+                        log.warn(
+                            'detected lower balance for {} on {}: {} < {}, '
+                            'updating position amount'.format(
+                                asset.symbol, self.name, free, position.amount
+                            )
+                        )
+                        position.amount = free
+
+        return free_cash, positions_value
 
     def order(self, asset, amount, style):
         """Place an order.

catalyst/exchange/exchange_algorithm.py

@@ -10,16 +10,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import pickle
 import signal
 import sys
 from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
-from time import sleep
 
 import logbook
 import pandas as pd
+from redo import retry
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
@@ -27,21 +28,21 @@ from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangePortfolioDataError,
-    OrderTypeNotSupported, )
+    OrderTypeNotSupported)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-from catalyst.exchange.exchange_utils import (
+from catalyst.exchange.live_graph_clock import LiveGraphClock
+from catalyst.exchange.simple_clock import SimpleClock
+from catalyst.exchange.utils.exchange_utils import (
     save_algo_object,
     get_algo_object,
     get_algo_folder,
     get_algo_df,
     save_algo_df,
     group_assets_by_exchange, )
-from catalyst.exchange.live_graph_clock import LiveGraphClock
-from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
     stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
+from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
 from catalyst.utils.api_support import api_method
@@ -70,12 +71,26 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                 and self.sim_params.arena == 'backtest':
             self.simulate_orders = True
 
+        # Operations with retry features
+        self.attempts = dict(
+            get_transactions_attempts=5,
+            order_attempts=5,
+            synchronize_portfolio_attempts=5,
+            get_order_attempts=5,
+            get_open_orders_attempts=5,
+            cancel_order_attempts=5,
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
+
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
             cancel_policy=self.cancel_policy,
             simulate_orders=self.simulate_orders,
-            exchanges=self.exchanges
+            exchanges=self.exchanges,
+            attempts=self.attempts,
         )
 
     @staticmethod
@@ -218,28 +233,28 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         """
         tracker = self.perf_tracker
-        period = tracker.todays_performance
+        cum = tracker.cumulative_performance
 
-        pos_stats = period.position_tracker.stats()
-        period_stats = calc_period_stats(pos_stats, period.ending_cash)
+        pos_stats = cum.position_tracker.stats()
+        period_stats = calc_period_stats(pos_stats, cum.ending_cash)
 
         stats = dict(
             period_start=tracker.period_start,
             period_end=tracker.period_end,
             capital_base=tracker.capital_base,
             progress=tracker.progress,
-            ending_value=period.ending_value,
-            ending_exposure=period.ending_exposure,
-            capital_used=period.cash_flow,
-            starting_value=period.starting_value,
-            starting_exposure=period.starting_exposure,
-            starting_cash=period.starting_cash,
-            ending_cash=period.ending_cash,
-            portfolio_value=period.ending_cash + period.ending_value,
-            pnl=period.pnl,
-            returns=period.returns,
-            period_open=period.period_open,
-            period_close=period.period_close,
+            ending_value=cum.ending_value,
+            ending_exposure=cum.ending_exposure,
+            capital_used=cum.cash_flow,
+            starting_value=cum.starting_value,
+            starting_exposure=cum.starting_exposure,
+            starting_cash=cum.starting_cash,
+            ending_cash=cum.ending_cash,
+            portfolio_value=cum.ending_cash + cum.ending_value,
+            pnl=cum.pnl,
+            returns=cum.returns,
+            period_open=start_dt,
+            period_close=end_dt,
             gross_leverage=period_stats.gross_leverage,
             net_leverage=period_stats.net_leverage,
             short_exposure=pos_stats.short_exposure,
@@ -256,8 +271,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
-        stats['positions'] = period.position_tracker.get_positions_list()
+        stats['positions'] = cum.position_tracker.get_positions_list()
 
+        period = tracker.todays_performance
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
         stats['transactions'] = []
@@ -276,6 +292,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         return stats
 
+    def run(self, data=None, overwrite_sim_params=True):
+        data.attempts = self.attempts
+        return super(ExchangeTradingAlgorithmBase, self).run(
+            data, overwrite_sim_params
+        )
+
 
 class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
@@ -328,6 +350,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
         self.stats_output = kwargs.pop('stats_output', None)
+        self._analyze_live = kwargs.pop('analyze_live', None)
 
         self._clock = None
         self.frame_stats = list()
@@ -342,42 +365,30 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         self.is_running = True
 
-        self.retry_check_open_orders = 5
-        self.retry_synchronize_portfolio = 5
-        self.retry_get_open_orders = 5
-        self.retry_order = 2
-        self.retry_delay = 5
+        self.stats_minutes = 1
 
-        self.stats_minutes = 10
+        self._last_orders = []
+        self.trading_client = None
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
-        signal.signal(signal.SIGINT, self.signal_handler)
+        try:
+            signal.signal(signal.SIGINT, self.signal_handler)
+        except ValueError:
+            log.warn("Can't initialize signal handler inside another thread."
+                     "Exit should be handled by the user.")
 
         log.info('initialized trading algorithm in live mode')
 
-    def signal_handler(self, signal, frame):
-        """
-        Handles the keyboard interruption signal.
-
-        Parameters
-        ----------
-        signal
-        frame
-
-        Returns
-        -------
-
-        """
+    def interrupt_algorithm(self):
         self.is_running = False
 
         if self._analyze is None:
-            log.info('Interruption signal detected {}, exiting the '
-                     'algorithm'.format(signal))
+            log.info('Exiting the algorithm.')
 
         else:
-            log.info('Interruption signal detected {}, calling `analyze()` '
-                     'before exiting the algorithm'.format(signal))
+            log.info('Exiting the algorithm. Calling `analyze()` '
+                     'before exiting the algorithm.')
 
             algo_folder = get_algo_folder(self.algo_namespace)
             folder = join(algo_folder, 'daily_perf')
@@ -395,6 +406,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         sys.exit(0)
 
+    def signal_handler(self, signal, frame):
+        """
+        Handles the keyboard interruption signal.
+
+        Parameters
+        ----------
+        signal
+        frame
+
+        Returns
+        -------
+
+        """
+        log.info('Interruption signal detected {}, exiting the '
+                 'algorithm'.format(signal))
+        self.interrupt_algorithm()
+
     @property
     def clock(self):
         if self._clock is None:
@@ -419,10 +447,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
-        if self.live_graph:
+        if self.live_graph or self._analyze_live is not None:
             self._clock = LiveGraphClock(
                 self.sim_params.sessions,
-                context=self
+                context=self,
+                callback=self._analyze_live,
             )
         else:
             self._clock = SimpleClock(
@@ -431,26 +460,52 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def _create_generator(self, sim_params):
+    def get_generator(self):
+        if self.trading_client is not None:
+            return self.trading_client.transform()
+
+        perf = None
         if self.perf_tracker is None:
-            self.perf_tracker = get_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker'
+            tracker = self.perf_tracker = PerformanceTracker(
+                sim_params=self.sim_params,
+                trading_calendar=self.trading_calendar,
+                env=self.trading_environment,
             )
 
+            # Set the dt initially to the period start by forcing it to change.
+            self.on_dt_changed(self.sim_params.start_session)
+
+            # Unpacking the perf_tracker and positions if available
+            perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key='cumulative_performance',
+            )
+
+        if not self.initialized:
+            self.initialize(*self.initialize_args, **self.initialize_kwargs)
+            self.initialized = True
+
         # Call the simulation trading algorithm for side-effects:
         # it creates the perf tracker
-        TradingAlgorithm._create_generator(self, sim_params)
-        self.trading_client = ExchangeAlgorithmExecutor(
-            self,
-            sim_params,
-            self.data_portal,
-            self.clock,
-            self._create_benchmark_source(),
-            self.restrictions,
-            universe_func=self._calculate_universe
-        )
+        # TradingAlgorithm._create_generator(self, self.sim_params)
+        if perf is not None:
+            tracker.cumulative_performance = perf
 
+            period = self.perf_tracker.todays_performance
+            period.starting_cash = perf.ending_cash
+            period.starting_exposure = perf.ending_exposure
+            period.starting_value = perf.ending_value
+            period.position_tracker = perf.position_tracker
+
+        self.trading_client = ExchangeAlgorithmExecutor(
+            algo=self,
+            sim_params=self.sim_params,
+            data_portal=self.data_portal,
+            clock=self.clock,
+            benchmark_source=self._create_benchmark_source(),
+            restrictions=self.restrictions,
+            universe_func=self._calculate_universe,
+        )
         return self.trading_client.transform()
 
     def updated_portfolio(self):
@@ -459,7 +514,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self):
         """
         Synchronizes the portfolio tracked by the algorithm to refresh
         its current value.
@@ -481,63 +536,51 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The total value of all tracked positions.
 
         """
+        check_balances = (not self.simulate_orders)
+        base_currency = None
         tracker = self.perf_tracker.position_tracker
         total_cash = 0.0
         total_positions_value = 0.0
 
-        try:
-            # Position keys correspond to assets
-            positions = self.portfolio.positions
-            assets = list(positions)
-            exchange_assets = group_assets_by_exchange(assets)
-            for exchange_name in self.exchanges:
-                assets = exchange_assets[exchange_name] \
-                    if exchange_name in exchange_assets else []
+        # Position keys correspond to assets
+        positions = self.portfolio.positions
+        assets = list(positions)
+        exchange_assets = group_assets_by_exchange(assets)
+        for exchange_name in self.exchanges:
+            assets = exchange_assets[exchange_name] \
+                if exchange_name in exchange_assets else []
 
-                exchange_positions = \
-                    [positions[asset] for asset in assets]
-
-                check_cash = (not self.simulate_orders)
-
-                exchange = self.exchanges[exchange_name]  # Type: Exchange
-                cash, positions_value = exchange.calculate_totals(
-                    positions=exchange_positions,
-                    check_cash=check_cash,
-                )
-                total_positions_value += positions_value
-
-                if cash is not None:
-                    total_cash += cash
-
-                for position in exchange_positions:
-                    tracker.update_position(
-                        asset=position.asset,
-                        last_sale_date=position.last_sale_date,
-                        last_sale_price=position.last_sale_price
-                    )
-
-            if cash is None:
-                total_cash = self.portfolio.cash
-
-            elif total_cash < self.portfolio.cash:
-                raise ValueError('Cash on exchanges is lower than the algo.')
-
-            return total_cash, total_positions_value
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'update portfolio attempt {}: {}'.format(attempt_index, e)
+            exchange_positions = copy.deepcopy(
+                [positions[asset] for asset in assets if asset in positions]
             )
-            if attempt_index < self.retry_synchronize_portfolio:
-                sleep(self.retry_delay)
-                return self.synchronize_portfolio(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='update-portfolio',
-                    attempts=attempt_index,
-                    error=e
+
+            exchange = self.exchanges[exchange_name]  # Type: Exchange
+
+            if base_currency is None:
+                base_currency = exchange.base_currency
+
+            cash, positions_value = exchange.sync_positions(
+                positions=exchange_positions,
+                check_balances=check_balances,
+                cash=self.portfolio.cash,
+            )
+            total_cash += cash
+            total_positions_value += positions_value
+
+            # Applying modifications to the original positions
+            for position in exchange_positions:
+                tracker.update_position(
+                    asset=position.asset,
+                    amount=position.amount,
+                    last_sale_date=position.last_sale_date,
+                    last_sale_price=position.last_sale_price,
                 )
 
+        if not check_balances:
+            total_cash = self.portfolio.cash
+
+        return total_cash, total_positions_value
+
     def add_pnl_stats(self, period_stats):
         """
         Save p&l stats.
@@ -632,13 +675,27 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if self.current_day is not None and today > self.current_day:
             self.frame_stats = list()
 
-        new_transactions, new_commissions, closed_orders = \
-            self.blotter.get_transactions(data)
+        self.performance_needs_update = False
+        new_orders = self.perf_tracker.todays_performance.orders_by_id.keys()
+        if new_orders != self._last_orders:
+            self.performance_needs_update = True
 
-        if len(new_transactions) > 0:
+        self._last_orders = new_orders
+
+        if self.performance_needs_update:
             self.perf_tracker.update_performance()
+            self.performance_needs_update = False
+
+        if self.portfolio_needs_update:
+            cash, positions_value = retry(
+                action=self.synchronize_portfolio,
+                attempts=self.attempts['synchronize_portfolio_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.')
+            )
+            self.portfolio_needs_update = False
 
-        cash, positions_value = self.synchronize_portfolio()
         log.info(
             'got totals from exchanges, cash: {} positions: {}'.format(
                 cash, positions_value
@@ -657,15 +714,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
-        # TODO: pickle does not seem to work in python 3
-        try:
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker',
-                obj=self.perf_tracker
-            )
-        except Exception as e:
-            log.warn('unable to save minute perfs to disk: {}'.format(e))
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='cumulative_performance',
+            obj=self.perf_tracker.cumulative_performance,
+        )
 
         self.current_day = data.current_dt.floor('1D')
 
@@ -677,7 +730,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.perf_tracker.update_performance()
 
         frame_stats = self.prepare_period_stats(
-            data.current_dt, data.current_dt + timedelta(minutes=1))
+            data.current_dt, data.current_dt + timedelta(minutes=1)
+        )
 
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
@@ -699,7 +753,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 stats=get_pretty_stats(
                     stats=self.frame_stats,
                     recorded_cols=recorded_cols,
-                    num_rows=self.stats_minutes
+                    num_rows=self.stats_minutes,
                 )
             ))
 
@@ -751,33 +805,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def batch_market_order(self, share_counts):
         raise NotImplementedError()
 
-    def _get_open_orders(self, asset=None, attempt_index=0):
-        try:
-            if asset:
-                exchange = self.exchanges[asset.exchange]
-                return exchange.get_open_orders(asset)
+    def _get_open_orders(self, asset=None):
+        if asset:
+            exchange = self.exchanges[asset.exchange]
+            return exchange.get_open_orders(asset)
 
-            else:
-                open_orders = []
-                for exchange_name in self.exchanges:
-                    exchange = self.exchanges[exchange_name]
-                    exchange_orders = exchange.get_open_orders()
-                    open_orders.append(exchange_orders)
+        else:
+            open_orders = []
+            for exchange_name in self.exchanges:
+                exchange = self.exchanges[exchange_name]
+                exchange_orders = exchange.get_open_orders()
+                open_orders.append(exchange_orders)
 
-                return open_orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_open_orders:
-                sleep(self.retry_delay)
-                return self._get_open_orders(asset, attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='open-orders',
-                    attempts=attempt_index,
-                    error=e
-                )
+            return open_orders
 
     @error_keywords(sid='Keyword argument `sid` is no longer supported for '
                         'get_open_orders. Use `asset` instead.')
@@ -799,7 +839,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             If an asset is passed then this will return a list of the open
             orders for this asset.
         """
-        return self._get_open_orders(asset)
+        return retry(
+            action=self._get_open_orders,
+            attempts=self.attempts['get_open_orders_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching open orders again.'),
+            args=(asset,))
 
     @api_method
     def get_order(self, order_id, exchange_name):
@@ -819,7 +865,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The execution price per share of the order
         """
         exchange = self.exchanges[exchange_name]
-        return exchange.get_order(order_id)
+        return retry(
+            action=exchange.get_order,
+            attempts=self.attempts['get_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching orders again.'),
+            args=(order_id,))
 
     @api_method
     def cancel_order(self, order_param, exchange_name):
@@ -836,4 +888,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if isinstance(order_param, zp.Order):
             order_id = order_param.id
 
-        exchange.cancel_order(order_id)
+        retry(
+            action=exchange.cancel_order,
+            attempts=self.attempts['cancel_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('cancelling order again.'),
+            args=(order_id,))

catalyst/exchange/exchange_asset_finder.py

@@ -0,0 +1,180 @@
+import pandas as pd
+from logbook import Logger
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.factory import find_exchanges
+
+log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
+
+
+class ExchangeAssetFinder(object):
+    def __init__(self, exchanges):
+        self.exchanges = exchanges
+
+    @property
+    def sids(self):
+        """
+        This seems to be used to pre-fetch assets.
+        I don't think that we need this for live-trading.
+        Leaving the list empty.
+        """
+        all_sids = []
+        for exchange_name in self.exchanges:
+            # This is what initializes each exchanges at the beginning
+            # of an algo
+            exchange = self.exchanges[exchange_name]
+            exchange.init()
+
+            all_sids += [asset.sid for asset in exchange.assets]
+
+        sids = list(set(all_sids))
+        return sids
+
+    def retrieve_asset(self, sid, default_none=False):
+        """
+        Retrieve the first Asset found for a given sid.
+        """
+        asset = None
+        for exchange_name in self.exchanges:
+            if asset is not None:
+                break
+
+            exchange = self.exchanges[exchange_name]
+            assets = [asset for asset in exchange.assets if asset.sid == sid]
+            if assets:
+                asset = assets[0]
+
+        return asset
+
+    def retrieve_all(self, sids, default_none=False):
+        """
+        Retrieve all assets in `sids`.
+
+        Parameters
+        ----------
+        sids : iterable of int
+            Assets to retrieve.
+        default_none : bool
+            If True, return None for failed lookups.
+            If False, raise `SidsNotFound`.
+
+        Returns
+        -------
+        assets : list[Asset or None]
+            A list of the same length as `sids` containing Assets (or Nones)
+            corresponding to the requested sids.
+
+        Raises
+        ------
+        SidsNotFound
+            When a requested sid is not found and default_none=False.
+        """
+        assets = []
+        for exchange_name in self.exchanges:
+            exchange = self.exchanges[exchange_name]
+            xas = [asset for asset in exchange.assets if asset.sid in sids]
+            assets += xas
+
+        return assets
+
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
+        """Lookup an asset by symbol.
+
+        Parameters
+        ----------
+        symbol : str
+            The ticker symbol to resolve.
+        as_of_date : datetime or None
+            Look up the last owner of this symbol as of this datetime.
+            If ``as_of_date`` is None, then this can only resolve the equity
+            if exactly one equity has ever owned the ticker.
+        fuzzy : bool, optional
+            Should fuzzy symbol matching be used? Fuzzy symbol matching
+            attempts to resolve differences in representations for
+            shareclasses. For example, some people may represent the ``A``
+            shareclass of ``BRK`` as ``BRK.A``, where others could write
+            ``BRK_A``.
+
+        Returns
+        -------
+        equity : Asset
+            The equity that held ``symbol`` on the given ``as_of_date``, or the
+            only equity to hold ``symbol`` if ``as_of_date`` is None.
+
+        Raises
+        ------
+        SymbolNotFound
+            Raised when no equity has ever held the given symbol.
+        MultipleSymbolsFound
+            Raised when no ``as_of_date`` is given and more than one equity
+            has held ``symbol``. This is also raised when ``fuzzy=True`` and
+            there are multiple candidates for the given ``symbol`` on the
+            ``as_of_date``.
+        """
+        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
+
+        return exchange.get_asset(symbol, data_frequency)
+
+    def lifetimes(self, dates, include_start_date):
+        """
+        Compute a DataFrame representing asset lifetimes for the specified date
+        range.
+
+        Parameters
+        ----------
+        dates : pd.DatetimeIndex
+            The dates for which to compute lifetimes.
+        include_start_date : bool
+            Whether or not to count the asset as alive on its start_date.
+
+            This is useful in a backtesting context where `lifetimes` is being
+            used to signify "do I have data for this asset as of the morning of
+            this date?"  For many financial metrics, (e.g. daily close), data
+            isn't available for an asset until the end of the asset's first
+            day.
+
+        Returns
+        -------
+        lifetimes : pd.DataFrame
+            A frame of dtype bool with `dates` as index and an Int64Index of
+            assets as columns.  The value at `lifetimes.loc[date, asset]` will
+            be True iff `asset` existed on `date`.  If `include_start_date` is
+            False, then lifetimes.loc[date, asset] will be false when date ==
+            asset.start_date.
+
+        See Also
+        --------
+        numpy.putmask
+        catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
+        """
+        exchanges = find_exchanges(features=['minuteBundle'])
+        if not exchanges:
+            raise ValueError('exchange with minute bundles not found')
+
+        # TODO: find a way to support multiple exchanges
+        exchange = exchanges[0]
+        # Using a single exchange for now because are not unique for the
+        # same asset in different exchanges. I'd like to avoid binding
+        # pipeline to a single exchange.
+        exchange.init()
+
+        data = []
+        for dt in dates:
+            exists = []
+
+            for asset in exchange.assets:
+                if include_start_date:
+                    condition = (asset.start_date <= dt < asset.end_minute)
+
+                else:
+                    condition = (asset.start_date < dt < asset.end_minute)
+
+                exists.append(condition)
+
+            data.append(exists)
+
+        sids = [asset.sid for asset in exchange.assets]
+        df = pd.DataFrame(data, index=dates, columns=exchange.assets)
+
+        return df

catalyst/exchange/exchange_blotter.py

@@ -1,15 +1,13 @@
-from time import sleep
-
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+from redo import retry
 
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import ExchangeRequestError, \
-    ExchangePortfolioDataError, ExchangeTransactionError
+from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
-from catalyst.finance.order import ORDER_STATUS, Order
+from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
@@ -60,12 +58,13 @@ class TradingPairFeeSchedule(CommissionModel):
         maker = self.maker if self.maker is not None else asset.maker
         taker = self.taker if self.taker is not None else asset.taker
 
-        multiplier = maker \
-            if ((order.amount > 0 and order.limit < transaction.price)
-                or (order.amount < 0 and order.limit > transaction.price)) \
-               and order.limit_reached else taker
+        multiplier = taker
+        if order.limit is not None:
+            multiplier = maker \
+                if ((order.amount > 0 and order.limit < transaction.price)
+                    or (order.amount < 0 and order.limit > transaction.price)) \
+                   and order.limit_reached else taker
 
-        # Assuming just the taker fee for now
         fee = cost * multiplier
         return fee
 
@@ -132,6 +131,7 @@ class TradingPairFixedSlippage(SlippageModel):
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
         self.simulate_orders = kwargs.pop('simulate_orders', False)
+        self.attempts = kwargs.pop('attempts', False)
 
         self.exchanges = kwargs.pop('exchanges', None)
         if not self.exchanges:
@@ -151,31 +151,11 @@ class ExchangeBlotter(Blotter):
             TradingPair: TradingPairFeeSchedule()
         }
 
-        self.retry_delay = 5
-        self.retry_check_open_orders = 5
-
-    def exchange_order(self, asset, amount, style=None, attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(
-                asset, amount, style
-            )
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-
-                return self.exchange_order(
-                    asset, amount, style, attempt_index + 1
-                )
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
+    def exchange_order(self, asset, amount, style=None):
+        exchange = self.exchanges[asset.exchange]
+        return exchange.order(
+            asset, amount, style
+        )
 
     @expect_types(asset=TradingPair)
     def order(self, asset, amount, style, order_id=None):
@@ -190,8 +170,13 @@ class ExchangeBlotter(Blotter):
             )
 
         else:
-            order = self.exchange_order(
-                asset, amount, style
+            order = retry(
+                action=self.exchange_order,
+                attempts=self.attempts['order_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.'),
+                args=(asset, amount, style),
             )
 
             self.open_orders[order.asset].append(order)
@@ -258,40 +243,32 @@ class ExchangeBlotter(Blotter):
                         )
                     )
 
-    def get_exchange_transactions(self, attempt_index=0):
+    def get_exchange_transactions(self):
         closed_orders = []
         transactions = []
         commissions = []
 
-        try:
-            for order, txn in self.check_open_orders():
-                order.dt = txn.dt
+        for order, txn in self.check_open_orders():
+            order.dt = txn.dt
 
-                transactions.append(txn)
+            transactions.append(txn)
 
-                if not order.open:
-                    closed_orders.append(order)
+            if not order.open:
+                closed_orders.append(order)
 
-            return transactions, commissions, closed_orders
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self.get_exchange_transactions(attempt_index + 1)
-
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
+        return transactions, commissions, closed_orders
 
     def get_transactions(self, bar_data):
         if self.simulate_orders:
             return super(ExchangeBlotter, self).get_transactions(bar_data)
 
         else:
-            return self.get_exchange_transactions()
+            return retry(
+                action=self.get_exchange_transactions,
+                attempts=self.attempts['get_transactions_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn(
+                    'Fetching exchange transactions again.'
+                )
+            )

catalyst/exchange/exchange_bundle.py

@@ -18,18 +18,18 @@ from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
-    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
-from catalyst.exchange.exchange_utils import get_exchange_folder, \
-    save_exchange_symbols, mixin_market_params
+from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
+    get_bcolz_chunk, get_month_start_end, \
+    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
+    get_delta, get_assets
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
+    save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -462,7 +462,7 @@ class ExchangeBundle:
                 (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
-        if end is None or (last_entry is not None and end > last_entry):
+        if last_entry is not None and (end is None or end > last_entry):
             end = last_entry.replace(minute=59, hour=23) \
                 if data_frequency == 'minute' else last_entry
 
@@ -668,7 +668,7 @@ class ExchangeBundle:
 
         if self.exchange is None:
             # Avoid circular dependencies
-            from catalyst.exchange.factory import get_exchange
+            from catalyst.exchange.utils.factory import get_exchange
             self.exchange = get_exchange(self.exchange_name)
 
         problems = []
@@ -681,6 +681,7 @@ class ExchangeBundle:
                 last_traded=np.object_,
                 open=np.float64,
                 high=np.float64,
+                low=np.float64,
                 close=np.float64,
                 volume=np.float64
             ),
@@ -730,7 +731,7 @@ class ExchangeBundle:
                     if data_frequency == 'minute' else asset_def['end_minute']
 
             else:
-                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+                params['symbol'] = get_catalyst_symbol(market)
 
                 params['end_daily'] = end_dt \
                     if data_frequency == 'daily' else 'N/A'
@@ -755,9 +756,10 @@ class ExchangeBundle:
         )
 
         for symbol in assets:
+            # here the symbol is the market['id']
             asset = assets[symbol]
             ohlcv_df = df.loc[
-                (df.index.get_level_values(0) == symbol)
+                (df.index.get_level_values(0) == asset.symbol)
             ]  # type: pd.DataFrame
             ohlcv_df.index = ohlcv_df.index.droplevel(0)
 
@@ -805,7 +807,7 @@ class ExchangeBundle:
         else:
             if self.exchange is None:
                 # Avoid circular dependencies
-                from catalyst.exchange.factory import get_exchange
+                from catalyst.exchange.utils.factory import get_exchange
                 self.exchange = get_exchange(self.exchange_name)
 
             assets = get_assets(

catalyst/exchange/exchange_data_portal.py

@@ -1,19 +1,18 @@
 import abc
-from time import sleep
 
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+from redo import retry
 
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangeBarDataError,
     PricingDataNotLoadedError)
-from catalyst.exchange.exchange_utils import get_frequency, \
+from catalyst.exchange.utils.exchange_utils import get_frequency, \
     resample_history_df, group_assets_by_exchange
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
@@ -21,11 +20,11 @@ log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
-
-        # TODO: put somewhere accessible by each algo
-        self.retry_get_history_window = 5
-        self.retry_get_spot_value = 5
-        self.retry_delay = 5
+        self.attempts = dict(
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
 
         super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
 
@@ -36,33 +35,14 @@ class DataPortalExchangeBase(DataPortal):
                             frequency,
                             field,
                             data_frequency,
-                            ffill=True,
-                            attempt_index=0):
-        try:
-            exchange_assets = group_assets_by_exchange(assets)
-            if len(exchange_assets) > 1:
-                df_list = []
-                for exchange_name in exchange_assets:
-                    assets = exchange_assets[exchange_name]
+                            ffill=True):
+        exchange_assets = group_assets_by_exchange(assets)
+        if len(exchange_assets) > 1:
+            df_list = []
+            for exchange_name in exchange_assets:
+                assets = exchange_assets[exchange_name]
 
-                    df_exchange = self.get_exchange_history_window(
-                        exchange_name,
-                        assets,
-                        end_dt,
-                        bar_count,
-                        frequency,
-                        field,
-                        data_frequency,
-                        ffill)
-
-                    df_list.append(df_exchange)
-
-                # Merging the values values of each exchange
-                return pd.concat(df_list)
-
-            else:
-                exchange_name = list(exchange_assets.keys())[0]
-                return self.get_exchange_history_window(
+                df_exchange = self.get_exchange_history_window(
                     exchange_name,
                     assets,
                     end_dt,
@@ -72,26 +52,22 @@ class DataPortalExchangeBase(DataPortal):
                     data_frequency,
                     ffill)
 
-        except ExchangeRequestError as e:
-            log.warn(
-                'get history attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_history_window:
-                sleep(self.retry_delay)
-                return self._get_history_window(assets,
-                                                end_dt,
-                                                bar_count,
-                                                frequency,
-                                                field,
-                                                data_frequency,
-                                                ffill,
-                                                attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='history',
-                    attempts=attempt_index,
-                    error=e
-                )
+                df_list.append(df_exchange)
+
+            # Merging the values values of each exchange
+            return pd.concat(df_list)
+
+        else:
+            exchange_name = list(exchange_assets.keys())[0]
+            return self.get_exchange_history_window(
+                exchange_name,
+                assets,
+                end_dt,
+                bar_count,
+                frequency,
+                field,
+                data_frequency,
+                ffill)
 
     def get_history_window(self,
                            assets,
@@ -105,13 +81,19 @@ class DataPortalExchangeBase(DataPortal):
         if field == 'price':
             field = 'close'
 
-        return self._get_history_window(assets,
-                                        end_dt,
-                                        bar_count,
-                                        frequency,
-                                        field,
-                                        data_frequency,
-                                        ffill)
+        return retry(
+            action=self._get_history_window,
+            attempts=self.attempts['get_history_window_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching history again.'),
+            args=(assets,
+                  end_dt,
+                  bar_count,
+                  frequency,
+                  field,
+                  data_frequency,
+                  ffill))
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
@@ -125,69 +107,58 @@ class DataPortalExchangeBase(DataPortal):
                                     ffill=True):
         pass
 
-    def _get_spot_value(self, assets, field, dt, data_frequency,
-                        attempt_index=0):
-        try:
-            if isinstance(assets, TradingPair):
-                spot_values = self.get_exchange_spot_value(
-                    assets.exchange, [assets], field, dt, data_frequency)
+    def _get_spot_value(self, assets, field, dt, data_frequency):
+        if isinstance(assets, TradingPair):
+            spot_values = self.get_exchange_spot_value(
+                assets.exchange, [assets], field, dt, data_frequency)
 
-                if not spot_values:
-                    return np.nan
+            if not spot_values:
+                return np.nan
 
-                return spot_values[0]
+            return spot_values[0]
+
+        else:
+            exchange_assets = dict()
+            for asset in assets:
+                if asset.exchange not in exchange_assets:
+                    exchange_assets[asset.exchange] = list()
+
+                exchange_assets[asset.exchange].append(asset)
+
+            if len(list(exchange_assets.keys())) == 1:
+                exchange_name = list(exchange_assets.keys())[0]
+                return self.get_exchange_spot_value(
+                    exchange_name, assets, field, dt, data_frequency)
 
             else:
-                exchange_assets = dict()
-                for asset in assets:
-                    if asset.exchange not in exchange_assets:
-                        exchange_assets[asset.exchange] = list()
+                spot_values = []
+                for exchange_name in exchange_assets:
+                    assets = exchange_assets[exchange_name]
+                    exchange_spot_values = self.get_exchange_spot_value(
+                        exchange_name,
+                        assets,
+                        field,
+                        dt,
+                        data_frequency
+                    )
+                    if len(assets) == 1:
+                        spot_values.append(exchange_spot_values)
+                    else:
+                        spot_values += exchange_spot_values
 
-                    exchange_assets[asset.exchange].append(asset)
-
-                if len(list(exchange_assets.keys())) == 1:
-                    exchange_name = list(exchange_assets.keys())[0]
-                    return self.get_exchange_spot_value(
-                        exchange_name, assets, field, dt, data_frequency)
-
-                else:
-                    spot_values = []
-                    for exchange_name in exchange_assets:
-                        assets = exchange_assets[exchange_name]
-                        exchange_spot_values = self.get_exchange_spot_value(
-                            exchange_name,
-                            assets,
-                            field,
-                            dt,
-                            data_frequency
-                        )
-                        if len(assets) == 1:
-                            spot_values.append(exchange_spot_values)
-                        else:
-                            spot_values += exchange_spot_values
-
-                    return spot_values
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'get spot value attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_spot_value:
-                sleep(self.retry_delay)
-                return self._get_spot_value(assets, field, dt, data_frequency,
-                                            attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='spot',
-                    attempts=attempt_index,
-                    error=e
-                )
+                return spot_values
 
     def get_spot_value(self, assets, field, dt, data_frequency):
         if field == 'price':
             field = 'close'
 
-        return self._get_spot_value(assets, field, dt, data_frequency)
+        return retry(
+            action=self._get_spot_value,
+            attempts=self.attempts['get_spot_value_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching spot value again.'),
+            args=(assets, field, dt, data_frequency))
 
     @abc.abstractmethod
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -339,7 +310,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
-            trailing_bar_count=trailing_bar_count
+            trailing_bar_count=trailing_bar_count,
         )
 
         df = resample_history_df(pd.DataFrame(series), freq, field)

catalyst/exchange/exchange_errors.py

@@ -100,6 +100,12 @@ class InvalidHistoryFrequencyError(ZiplineError):
     ).strip()
 
 
+class InvalidHistoryTimeframeError(ZiplineError):
+    msg = (
+        'CCXT timeframe {timeframe} not supported by the exchange.'
+    ).strip()
+
+
 class MismatchingFrequencyError(ZiplineError):
     msg = (
         'Bar aggregate frequency {frequency} not compatible with '
@@ -162,8 +168,8 @@ class SidHashError(ZiplineError):
 
 class BaseCurrencyNotFoundError(ZiplineError):
     msg = (
-        'Algorithm base currency {base_currency} not found in exchange '
-        '{exchange}.'
+        'Algorithm base currency {base_currency} not found in account '
+        'balances on {exchange}: {balances}'
     ).strip()
 
 
@@ -226,16 +232,20 @@ class PricingDataValueError(ZiplineError):
 
 
 class DataCorruptionError(ZiplineError):
-    msg = ('Unable to validate data for {exchange} {symbols} in date range '
-           '[{start_dt} - {end_dt}]. The data is either corrupted or '
-           'unavailable. Please try deleting this bundle:'
-           '\n`catalyst clean-exchange -x {exchange}\n'
-           'Then, ingest the data again. Please contact the Catalyst team if '
-           'the issue persists.').strip()
+    msg = (
+        'Unable to validate data for {exchange} {symbols} in date range '
+        '[{start_dt} - {end_dt}]. The data is either corrupted or '
+        'unavailable. Please try deleting this bundle:'
+        '\n`catalyst clean-exchange -x {exchange}\n'
+        'Then, ingest the data again. Please contact the Catalyst team if '
+        'the issue persists.'
+    ).strip()
 
 
 class ApiCandlesError(ZiplineError):
-    msg = ('Unable to fetch candles from the remote API: {error}.').strip()
+    msg = (
+        'Unable to fetch candles from the remote API: {error}.'
+    ).strip()
 
 
 class NoDataAvailableOnExchange(ZiplineError):
@@ -248,13 +258,16 @@ class NoDataAvailableOnExchange(ZiplineError):
 
 
 class NoValueForField(ZiplineError):
-    msg = ('Value not found for field: {field}.').strip()
+    msg = (
+        'Value not found for field: {field}.'
+    ).strip()
 
 
 class OrderTypeNotSupported(ZiplineError):
     msg = (
-        'Order type `{order_type}` not currencly supported by Catalyst. '
-        'Please use `limit` or `market` orders only.').strip()
+        'Order type `{order_type}` not currency supported by Catalyst. '
+        'Please use `limit` or `market` orders only.'
+    ).strip()
 
 
 class NotEnoughCapitalError(ZiplineError):
@@ -262,10 +275,43 @@ class NotEnoughCapitalError(ZiplineError):
         'Not enough capital on exchange {exchange} for trading. Each '
         'exchange should contain at least as much {base_currency} '
         'as the specified `capital_base`. The current balance {balance} is '
-        'lower than the `capital_base`: {capital_base}').strip()
+        'lower than the `capital_base`: {capital_base}'
+    ).strip()
+
+
+class NotEnoughCashError(ZiplineError):
+    msg = (
+        'Total {currency} amount on {exchange} is lower than the cash '
+        'reserved for this algo: {free} < {cash}. While trades can be made on '
+        'the exchange accounts outside of the algo, exchange must have enough '
+        'free {currency} to cover the algo cash.'
+    ).strip()
+
 
 class LastCandleTooEarlyError(ZiplineError):
     msg = (
         'The trade date of the last candle {last_traded} is before the '
         'specified end date minus one candle {end_dt}. Please verify how '
-        '{exchange} calculates the start date of OHLCV candles.').strip()
+        '{exchange} calculates the start date of OHLCV candles.'
+    ).strip()
+
+
+class TickerNotFoundError(ZiplineError):
+    msg = (
+        'Unable to fetch ticker for {symbol} on {exchange}.'
+    ).strip()
+
+
+class BalanceNotFoundError(ZiplineError):
+    msg = (
+        '{currency} not found in account balance on {exchange}: {balances}.'
+    ).strip()
+
+
+class BalanceTooLowError(ZiplineError):
+    msg = (
+        'Balance for {currency} on {exchange} too low: {free} < {amount}. '
+        'Positions have likely been sold outside of this algorithm. Please '
+        'add positions to hold a free amount greater than {amount}, or clean '
+        'the state of this algo and restart.'
+    ).strip()

catalyst/exchange/exchange_pricing_loader.py

@@ -0,0 +1,178 @@
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.data.us_equity_pricing import BcolzDailyBarReader
+from catalyst.errors import NoFurtherDataError
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.lib.adjusted_array import AdjustedArray
+from catalyst.pipeline.data import DataSet, Column
+from catalyst.pipeline.loaders.base import PipelineLoader
+from catalyst.utils.calendars import get_calendar
+from catalyst.utils.numpy_utils import float64_dtype
+
+UINT32_MAX = iinfo(uint32).max
+
+log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
+
+
+class TradingPairPricing(DataSet):
+    """
+    Dataset representing daily trading prices and volumes.
+    """
+    open = Column(float64_dtype)
+    high = Column(float64_dtype)
+    low = Column(float64_dtype)
+    close = Column(float64_dtype)
+    volume = Column(float64_dtype)
+
+
+class ExchangePricingLoader(PipelineLoader):
+    """
+    PipelineLoader for Crypto Pricing data
+
+    Delegates loading of baselines and adjustments.
+    """
+
+    def __init__(self, data_frequency):
+
+        cal = get_calendar('OPEN')
+
+        if data_frequency == 'daily':
+            reader = None
+            all_sessions = cal.all_sessions
+
+        elif data_frequency == 'minute':
+            reader = None
+            all_sessions = cal.all_minutes
+
+        else:
+            raise ValueError(
+                'Invalid data frequency: {}'.format(data_frequency)
+            )
+
+        self.data_frequency = data_frequency
+        self.raw_price_loader = reader
+        self._columns = TradingPairPricing.columns
+        self._all_sessions = all_sessions
+
+    @classmethod
+    def from_files(cls, pricing_path):
+        """
+        Create a loader from a bcolz equity pricing dir and a SQLite
+        adjustments path.
+
+        Parameters
+        ----------
+        pricing_path : str
+            Path to a bcolz directory written by a BcolzDailyBarWriter.
+        """
+        return cls(
+            BcolzDailyBarReader(pricing_path),
+        )
+
+    def load_adjusted_array(self, columns, dates, assets, mask):
+        # load_adjusted_array is called with dates on which the user's algo
+        # will be shown data, which means we need to return the data that would
+        # be known at the start of each date.  We assume that the latest data
+        # known on day N is the data from day (N - 1), so we shift all query
+        # dates back by a day.
+        start_date, end_date = _shift_dates(
+            self._all_sessions, dates[0], dates[-1], shift=1,
+        )
+        colnames = [c.name for c in columns]
+
+        if len(assets) == 0:
+            raise ValueError(
+                'Pipeline cannot load data with eligible assets.'
+            )
+
+        exchange_names = []
+        for asset in assets:
+            if asset.exchange not in exchange_names:
+                exchange_names.append(asset.exchange)
+
+        exchange = get_exchange(exchange_names[0])
+        reader = exchange.bundle.get_reader(self.data_frequency)
+
+        raw_arrays = reader.load_raw_arrays(
+            colnames,
+            start_date,
+            end_date,
+            assets,
+        )
+
+        out = {}
+        for c, c_raw in zip(columns, raw_arrays):
+            out[c] = AdjustedArray(
+                c_raw.astype(c.dtype),
+                mask,
+                {},
+                c.missing_value,
+            )
+        return out
+
+    @property
+    def columns(self):
+        return self._columns
+
+
+def _shift_dates(dates, start_date, end_date, shift):
+    try:
+        start = dates.get_loc(start_date)
+    except KeyError:
+        if start_date < dates[0]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requested data starting on {query_start}, "
+                    "but first known date is {calendar_start}"
+                ).format(
+                    query_start=str(start_date),
+                    calendar_start=str(dates[0]),
+                )
+            )
+        else:
+            raise ValueError("Query start %s not in calendar" % start_date)
+
+    # Make sure that shifting doesn't push us out of the calendar.
+    if start < shift:
+        raise NoFurtherDataError(
+            msg=(
+                "Pipeline Query requested data from {shift}"
+                " days before {query_start}, but first known date is only "
+                "{start} days earlier."
+            ).format(shift=shift, query_start=start_date, start=start),
+        )
+
+    try:
+        end = dates.get_loc(end_date)
+    except KeyError:
+        if end_date > dates[-1]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requesting data up to {query_end}, "
+                    "but last known date is {calendar_end}"
+                ).format(
+                    query_end=end_date,
+                    calendar_end=dates[-1],
+                )
+            )
+        else:
+            raise ValueError("Query end %s not in calendar" % end_date)
+    return dates[start - shift], dates[end - shift]

catalyst/exchange/factory.py

@@ -1,34 +0,0 @@
-import os
-
-from catalyst.exchange.ccxt.ccxt_exchange import CCXT
-from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_exchange_folder
-
-
-def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
-    exchange_auth = get_exchange_auth(exchange_name)
-
-    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
-    if must_authenticate and not has_auth:
-        raise ExchangeAuthEmpty(
-            exchange=exchange_name.title(),
-            filename=os.path.join(
-                get_exchange_folder(exchange_name), 'auth.json'
-            )
-        )
-
-    return CCXT(
-        exchange_name=exchange_name,
-        key=exchange_auth['key'],
-        secret=exchange_auth['secret'],
-        base_currency=base_currency,
-    )
-
-
-def get_exchanges(exchange_names):
-    exchanges = dict()
-    for exchange_name in exchange_names:
-        exchanges[exchange_name] = get_exchange(exchange_name)
-
-    return exchanges

catalyst/exchange/live_graph_clock.py

@@ -6,8 +6,7 @@ from catalyst.gens.sim_engine import (
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
+from catalyst.exchange.utils.stats_utils import prepare_stats
 
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
@@ -38,177 +37,23 @@ class LiveGraphClock(object):
     the exchange and the live trading machine's clock. It's not used currently.
     """
 
-    def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
-
-        global mdates, plt  # TODO: Could be cleaner
-        import matplotlib.dates as mdates
-        from matplotlib import pyplot as plt
-        from matplotlib import style
+    def __init__(self, sessions, context, callback=None,
+                 time_skew=pd.Timedelta('0s')):
 
         self.sessions = sessions
         self.time_skew = time_skew
         self._last_emit = None
         self._before_trading_start_bar_yielded = True
         self.context = context
-        self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
-
-        style.use('dark_background')
-
-        fig = plt.figure()
-        fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
-            self.context.algo_namespace))
-
-        self.ax_pnl = fig.add_subplot(311)
-
-        self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
-
-        self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
-
-        if len(context.minute_stats) > 0:
-            self.draw_pnl()
-            self.draw_custom_signals()
-            self.draw_exposure()
-
-        # rotates and right aligns the x labels, and moves the bottom of the
-        # axes up to make room for them
-        fig.autofmt_xdate()
-        fig.subplots_adjust(hspace=0.5)
-
-        plt.tight_layout()
-        plt.ion()
-        plt.show()
-
-    def format_ax(self, ax):
-        """
-        Trying to assign reasonable parameters to the time axis.
-
-        Parameters
-        ----------
-        ax:
-
-        """
-        # TODO: room for improvement
-        ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-        ax.xaxis.set_major_formatter(self.fmt)
-
-        locator = mdates.HourLocator(interval=4)
-        locator.MAXTICKS = 5000
-        ax.xaxis.set_minor_locator(locator)
-
-        datemin = pd.Timestamp.utcnow()
-        ax.set_xlim(datemin)
-
-        ax.grid(True)
-
-    def set_legend(self, ax):
-        """
-        Set legend on the chart.
-
-        Parameters
-        ----------
-        ax
-
-        """
-        ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-    def draw_pnl(self):
-        """
-        Draw p&l line on the chart.
-
-        """
-        ax = self.ax_pnl
-        df = self.context.pnl_stats
-
-        ax.clear()
-        ax.set_title('Performance')
-        ax.plot(df.index, df['performance'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Performance'
-                )
-
-        def perc(val):
-            return '{:2f}'.format(val)
-
-        ax.format_ydata = perc
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_custom_signals(self):
-        """
-        Draw custom signals on the chart.
-
-        """
-        ax = self.ax_custom_signals
-        df = self.context.custom_signals_stats
-
-        colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-        ax.clear()
-        ax.set_title('Custom Signals')
-        for index, column in enumerate(df.columns.values.tolist()):
-            ax.plot(df.index, df[column], '-',
-                    color=colors[index],
-                    linewidth=1.0,
-                    label=column
-                    )
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_exposure(self):
-        """
-        Draw exposure line on the chart.
-
-        """
-        ax = self.ax_exposure
-        context = self.context
-        df = context.exposure_stats
-
-        # TODO: list exchanges in graph
-        base_currency = None
-        positions = []
-        for exchange_name in context.exchanges:
-            exchange = context.exchanges[exchange_name]
-
-            if not base_currency:
-                base_currency = exchange.base_currency
-            elif base_currency != exchange.base_currency:
-                raise MismatchingBaseCurrenciesExchanges(
-                    base_currency=base_currency,
-                    exchange_name=exchange.name,
-                    exchange_currency=exchange.base_currency
-                )
-
-            positions += exchange.portfolio.positions
-
-        ax.clear()
-        ax.set_title('Exposure')
-        ax.plot(df.index, df['base_currency'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Base Currency: {}'.format(base_currency.upper())
-                )
-
-        symbols = []
-        for position in positions:
-            symbols.append(position.symbol)
-
-        ax.plot(df.index, df['long_exposure'], '-',
-                color='blue',
-                linewidth=1.0,
-                label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-        self.set_legend(ax)
-        self.format_ax(ax)
+        self.callback = callback
 
     def __iter__(self):
+        from matplotlib import pyplot as plt
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:
             current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
+            current_minute = current_time.floor('1T')
 
             if self._last_emit is None or current_minute > self._last_emit:
                 log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -216,14 +61,11 @@ class LiveGraphClock(object):
                 self._last_emit = current_minute
                 yield current_minute, BAR
 
-                try:
-                    self.draw_pnl()
-                    self.draw_custom_signals()
-                    self.draw_exposure()
-
-                    plt.draw()
-                except Exception as e:
-                    log.warn('Unable to update the graph: {}'.format(e))
+                recorded_cols = list(self.context.recorded_vars.keys())
+                df, _ = prepare_stats(
+                    self.context.frame_stats, recorded_cols=recorded_cols
+                )
+                self.callback(self.context, df)
 
             else:
                 # I can't use the "animate" reactive approach here because

catalyst/exchange/poloniex/poloniex.py

@@ -1,661 +0,0 @@
-import json
-import time
-from collections import defaultdict
-
-import numpy as np
-import pandas as pd
-import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-# import six
-from six import iteritems
-
-from catalyst.constants import LOG_LEVEL
-# from websocket import create_connection
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle,
-    OrphanOrderError,
-    OrphanOrderReverseError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.protocol import Account
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Poloniex', level=LOG_LEVEL)
-
-
-@deprecated
-class Poloniex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret)
-        self.name = 'poloniex'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-        self.transactions = defaultdict(list)
-
-        self.num_candles_limit = 2000
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from the Exchange order dictionary
-        :param order_status:
-        :return: Order
-        """
-        # if order_status['is_cancelled']:
-        #    status = ORDER_STATUS.CANCELLED
-        # elif not order_status['is_live']:
-        #    log.info('found executed order {}'.format(order_status))
-        #    status = ORDER_STATUS.FILLED
-        # else:
-        status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['amount'])
-        # filled = float(order_status['executed_amount'])
-        filled = None
-
-        if order_status['type'] == 'sell':
-            amount = -amount
-            # filled = -filled
-
-        price = float(order_status['rate'])
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        # if order_type.endswith('limit'):
-        #    limit_price = price
-        # elif order_type.endswith('stop'):
-        #    stop_price = price
-
-        # executed_price = float(order_status['avg_execution_price'])
-        executed_price = price
-
-        # TODO: Set Poloniex comission
-        commission = None
-
-        # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date=pytz.utc.localize(date)
-        date = None
-
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            # No such field in Poloniex
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['orderNumber']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        balances = self.api.returnbalances()
-        try:
-            log.debug('retrieving wallets balances')
-        except Exception as e:
-            log.debug(e)
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['error'])
-            )
-
-        std_balances = dict()
-        for (key, value) in iteritems(balances):
-            currency = key.lower()
-            std_balances[currency] = float(value)
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Poloniex
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '5m', '15m', '30m', '2h', '4h', '1D'
-        """
-
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T' and (bar_count == 1 or bar_count is None):
-            # TODO: use the order book instead
-            # We use the 5m to fetch the last bar
-            frequency = 300
-        elif freq == '5T':
-            frequency = 300
-        elif freq == '15T':
-            frequency = 900
-        elif freq == '30T':
-            frequency = 1800
-        elif freq == '120T':
-            frequency = 7200
-        elif freq == '240T':
-            frequency = 14400
-        elif freq == '1D':
-            frequency = 86400
-        else:
-            # Poloniex does not offer 1m data candles
-            # It is likely to error out there frequently
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-
-        for asset in asset_list:
-            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
-            end = int(delta.total_seconds())
-
-            if bar_count is None:
-                start = end - 2 * frequency
-            else:
-                start = end - bar_count * frequency
-
-            try:
-                response = self.api.returnchartdata(
-                    self.get_symbol(asset), frequency, start, end
-                )
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-                ohlc = dict(
-                    open=np.float64(candle['open']),
-                    high=np.float64(candle['high']),
-                    low=np.float64(candle['low']),
-                    close=np.float64(candle['close']),
-                    volume=np.float64(candle['volume']),
-                    price=np.float64(candle['close']),
-                    last_traded=last_traded
-                )
-
-                return ohlc
-
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(response[0])
-            else:
-                ohlc_bars = []
-                for candle in response:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-
-        if (isinstance(style, ExchangeLimitOrder)
-                or isinstance(style, ExchangeStopLimitOrder)):
-            if isinstance(style, ExchangeStopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-
-            try:
-                if (is_buy):
-                    response = self.api.buy(exchange_symbol, amount, price)
-                else:
-                    response = self.api.sell(exchange_symbol, -amount, price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            date = pd.Timestamp.utcnow()
-
-            if ('orderNumber' in response):
-                order_id = str(response['orderNumber'])
-                order = Order(
-                    dt=date,
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                log.warn(
-                    '{} order failed: {}'.format('buy' if is_buy else 'sell',
-                                                 response['error']))
-                return None
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset='all'):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not 'all', return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If 'all' is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-
-        return self.portfolio.open_orders
-
-        """
-        TODO: Why going to the exchange if we already have this info locally?
-              And why creating all these Orders if we later discard them?
-        """
-
-        try:
-            if (asset == 'all'):
-                response = self.api.returnopenorders('all')
-            else:
-                response = self.api.returnopenorders(self.get_symbol(asset))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    response['message'])
-            )
-
-        print(self.portfolio.open_orders)
-
-        # TODO: Need to handle openOrders for 'all'
-        orders = list()
-        for order_status in response:
-            # will Throw error b/c Polo doesn't track order['symbol']
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-
-        try:
-            order = self._portfolio.open_orders[order_id]
-        except Exception as e:
-            raise OrphanOrderError(order_id=order_id, exchange=self.name)
-
-        return order
-
-        # TODO: Need to decide whether we fetch orders locally or from exchnage
-        # The code below is ignored
-
-        try:
-            response = self.api.returnopenorders(self.get_symbol(order.sid))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        for o in response:
-            if (int(o['orderNumber']) == int(order_id)):
-                return order
-
-        return None
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-
-        if (isinstance(order_param, Order)):
-            order = order_param
-        else:
-            order = self._portfolio.open_orders[order_param]
-
-        try:
-            response = self.api.cancelorder(order.id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} '
-                '{error}.'.format(
-                    order_id=order.id,
-                    exchange=self.name,
-                    error=response['error']
-                ))
-
-            # raise OrderCancelError(
-            #    order_id=order.id,
-            #    exchange=self.name,
-            #    error=response['error']
-            # )
-
-        self.portfolio.remove_order(order)
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self.get_symbols(assets)
-
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            response = self.api.returnticker()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response['error'])
-            )
-
-        ticks = dict()
-
-        for index, symbol in enumerate(symbols):
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=float(response[symbol]['highestBid']),
-                ask=float(response[symbol]['lowestAsk']),
-                last_price=float(response[symbol]['last']),
-                low=float(response[symbol]['lowestAsk']),
-                # TODO: Polo does not provide low
-                high=float(response[symbol]['highestBid']),
-                # TODO: Polo does not provide high
-                volume=float(response[symbol]['baseVolume']),
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self.api.returnticker()
-
-        for exchange_symbol in response:
-            base, market = self.sanitize_curency_symbol(exchange_symbol).split(
-                '_')
-            symbol = '{market}_{base}'.format(market=market, base=base)
-
-            if (source_dates):
-                start_date = self.get_symbol_start_date(exchange_symbol)
-            else:
-                try:
-                    start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-        try:
-            r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
-                '2010-1-1').value // 10 ** 9)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
-
-    def check_open_orders(self):
-        """
-        Need to override this function for Poloniex:
-
-        Loop through the list of open orders in the Portfolio object.
-        Check if any transactions have been executed:
-            If so, create a transaction and apply to the Portfolio.
-        Check if the order is still open:
-            If not, remove it from open orders
-
-        :return:
-        transactions: Transaction[]
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-
-                order = self._portfolio.open_orders[order_id]
-                log.debug('found open order: {}'.format(order_id))
-
-                try:
-                    order_open = self.get_order(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if (order_open):
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta)
-                    )
-
-                try:
-                    response = self.api.returnordertrades(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if ('error' in response):
-                    if (not order_open):
-                        raise OrphanOrderReverseError(order_id=order_id,
-                                                      exchange=self.name)
-                else:
-                    for tx in response:
-                        """
-                        We maintain a list of dictionaries of transactions that
-                        correspond to partially filled orders, indexed by
-                        order_id. Every time we query executed transactions
-                        from the exchange, we check if we had that transaction
-                        for that order already. If not, we process it.
-
-                        When an order if fully filled, we flush the dict of
-                        transactions associated with that order.
-                        """
-                        if (not filter(
-                                lambda item: item['order_id'] == tx['tradeID'],
-                                self.transactions[order_id])):
-                            log.debug(
-                                'Got new transaction for order {}: amount {}, '
-                                'price {}'.format(
-                                    order_id, tx['amount'], tx['rate']))
-                            tx['amount'] = float(tx['amount'])
-                            if (tx['type'] == 'sell'):
-                                tx['amount'] = -tx['amount']
-                            transaction = Transaction(
-                                asset=order.asset,
-                                amount=tx['amount'],
-                                dt=pd.to_datetime(tx['date'], utc=True),
-                                price=float(tx['rate']),
-                                order_id=tx['tradeID'],
-                                # it's a misnomer, but keep for compatibility
-                                commission=float(tx['fee'])
-                            )
-                            self.transactions[order_id].append(transaction)
-                            self.portfolio.execute_transaction(transaction)
-                            transactions.append(transaction)
-
-                    if (not order_open):
-                        """
-                            Since transactions have been executed individually
-                            the only thing left to do is remove them from list
-                            of open_orders
-                        """
-                        del self.portfolio.open_orders[order_id]
-                        del self.transactions[order_id]
-
-        return transactions
-
-    def get_orderbook(self, asset, order_type='all'):
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.returnOrderBook(market=exchange_symbol)
-
-        result = dict()
-        for order_type in data:
-            # TODO: filter by type
-            if order_type != 'asks' and order_type != 'bids':
-                continue
-
-            result[order_type] = []
-            for entry in data[order_type]:
-                if len(entry) == 2:
-                    result[order_type].append(
-                        dict(
-                            rate=float(entry[0]),
-                            quantity=float(entry[1])
-                        )
-                    )
-        return result

catalyst/exchange/poloniex/poloniex_api.py

@@ -1,212 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-from six.moves import urllib
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-urlopen = urllib.request.urlopen
-
-
-class Poloniex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-
-        self.max_requests_per_second = 6
-        self.request_cpt = dict()
-
-        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                       'returnTradeHistory', 'returnChartData',
-                       'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances', 'returnCompleteBalances',
-                        'returnDepositAddresses',
-                        'generateNewAddress', 'returnDepositsWithdrawals',
-                        'returnOpenOrders',
-                        'returnTradeHistory', 'returnOrderTrades',
-                        'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo',
-                        'returnAvailableAccountBalances',
-                        'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary', 'marginBuy',
-                        'marginSell',
-                        'getMarginPosition', 'closeMarginPosition',
-                        'createLoanOffer',
-                        'cancelLoanOffer', 'returnOpenLoanOffers',
-                        'returnActiveLoans',
-                        'returnLendingHistory', 'toggleAutoRenew']
-
-    def ask_request(self):
-        """
-        Asks permission to issue a request to the exchange.
-        The primary purpose is to avoid hitting rate limits.
-
-        The application will pause if the maximum requests per minute
-        permitted by the exchange is exceeded.
-
-        :return boolean:
-
-        """
-        now = time.time()
-        if not self.request_cpt:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        cpt_date = list(self.request_cpt.keys())[0]
-        cpt = self.request_cpt[cpt_date]
-
-        if now > cpt_date + 1:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        if cpt >= self.max_requests_per_second:
-
-            time.sleep(1)
-
-            now = time.time()
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-        else:
-            self.request_cpt[cpt_date] += 1
-
-    def query(self, method, req={}):
-
-        if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + \
-                  urllib.parse.urlencode(req)
-            headers = {}
-            post_data = None
-        elif method in self.trading:
-            url = 'https://poloniex.com/tradingApi'
-            req['command'] = method
-            req['nonce'] = int(time.time() * 1000)
-            post_data = urllib.parse.urlencode(req)
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 post_data.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'Sign': signature, 'Key': self.key}
-
-            post_data = post_data.encode('utf-8')
-        else:
-            raise ValueError(
-                'Method "' + method + '" not found in neither the Public API '
-                                      'or Trading API endpoints'
-            )
-
-        self.ask_request()
-        req = urllib.request.Request(
-            url,
-            data=post_data,
-            headers=headers,
-        )
-        resource = urlopen(req, context=ssl._create_unverified_context())
-        content = resource.read().decode('utf-8')
-        return json.loads(content)
-
-    def returnticker(self):
-        return self.query('returnTicker', {})
-
-    def return24volume(self):
-        return self.query('return24Volume', {})
-
-    def returnOrderBook(self, market='all'):
-        return self.query('returnOrderBook', {'currencyPair': market})
-
-    def returntradehistory(self, market, start=None, end=None):
-        if (start is not None and end is not None):
-            return self.query('returntradehistory',
-                              {'currencyPair': market, 'start': start,
-                               'end': end})
-        else:
-            return self.query('returntradehistory', {'currencyPair': market})
-
-    def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData',
-                          {'currencyPair': market, 'period': period,
-                           'start': start, 'end': end})
-
-    def returncurrencies(self):
-        return self.query('returnCurrencies', {})
-
-    def returnloadorders(self, market):
-        return self.query('returnLoanOrders', {'currency': market})
-
-    def returnbalances(self):
-        return self.query('returnBalances')
-
-    def returncompletebalances(self, account):
-        if (account):
-            return self.query('returnCompleteBalances', {'account': account})
-        else:
-            return self.query('returnCompleteBalances')
-
-    def returndepositaddresses(self):
-        return self.query('returnDepositAddresses')
-
-    def generatenewaddress(self, currency):
-        return self.query('generateNewAddress', {'currency': currency})
-
-    def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals',
-                          {'start': start, 'end': end})
-
-    def returnopenorders(self, market):
-        return self.query('returnOpenOrders', {'currencyPair': market})
-
-    def returnordertrades(self, ordernumber):
-        return self.query('returnOrderTrades', {'orderNumber': ordernumber})
-
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-            postonly=0):
-        if (fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'postOnly': postonly, })
-        else:
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount, })
-
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-             postonly=0):
-        if (fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'postOnly': postonly, })
-        else:
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount, })
-
-    def cancelorder(self, ordernumber):
-        return self.query('cancelOrder', {'orderNumber': ordernumber})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'amount': quantity,
-                           'address': address})
-
-    def returnfeeinfo(self):
-        return self.query('returnFeeInfo')

catalyst/exchange/utils/bundle_utils.py

@@ -8,7 +8,7 @@ import pandas as pd
 import pytz
 
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 API_URL = 'http://data.enigma.co/api/v1'

catalyst/exchange/utils/exchange_utils.py

@@ -14,6 +14,8 @@ from six.moves.urllib import request
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
 from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
     InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
+from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
+    ExchangeJSONDecoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
@@ -62,6 +64,13 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
+def is_blacklist(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'blacklist.txt')
+
+    return os.path.exists(filename)
+
+
 def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
@@ -101,20 +110,6 @@ def download_exchange_symbols(exchange_name, environ=None):
     return response
 
 
-def symbols_parser(asset_def):
-    for key, value in asset_def.items():
-        match = isinstance(value, string_types) \
-                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
-
-        if match:
-            try:
-                asset_def[key] = pd.to_datetime(value, utc=True)
-            except ValueError:
-                pass
-
-    return asset_def
-
-
 def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     """
     The de-serialized content of the exchange's symbols.json.
@@ -134,13 +129,13 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
 
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
                 pd.Timestamp('now', tz='UTC') - last_modified_time(
-                    filename)).days > 1):
+                filename)).days > 1):
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
             try:
-                data = json.load(data_file, object_hook=symbols_parser)
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
                 return data
 
             except ValueError:
@@ -266,7 +261,7 @@ def get_algo_folder(algo_name, environ=None):
     return algo_folder
 
 
-def get_algo_object(algo_name, key, environ=None, rel_path=None):
+def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     """
     The de-serialized object of the algo name and key.
 
@@ -290,19 +285,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
 
-    filename = os.path.join(folder, key + '.p')
+    name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
+    filename = os.path.join(folder, name)
 
     if os.path.isfile(filename):
-        try:
+        if how == 'pickle':
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception:
-            return None
+
+        else:
+            with open(filename) as data_file:
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
+                return data
+
     else:
         return None
 
 
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
+                     how='pickle'):
     """
     Serialize and save an object by algo name and key.
 
@@ -321,10 +322,15 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
-    filename = os.path.join(folder, key + '.p')
+    if how == 'json':
+        filename = os.path.join(folder, '{}.json'.format(key))
+        with open(filename, 'wt') as handle:
+            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
 
-    with open(filename, 'wb') as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
+    else:
+        filename = os.path.join(folder, '{}.p'.format(key))
+        with open(filename, 'wb') as handle:
+            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
@@ -428,6 +434,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
+def has_bundle(exchange_name, data_frequency, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+
+    folder_name = '{}_bundle'.format(data_frequency.lower())
+    folder = os.path.join(exchange_folder, folder_name)
+
+    return os.path.isdir(folder)
+
+
 def symbols_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -531,6 +546,11 @@ def get_frequency(freq, data_frequency):
         else:
             raise InvalidHistoryFrequencyError(frequency=freq)
 
+    # TODO: some exchanges support H and W frequencies but not bundles
+    # Find a way to pass-through these parameters to exchanges
+    # but resample from minute or daily in backtest mode
+    # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
+    # Pandas offet aliases (used by Catalyst) and the CCXT timeframes
     if unit.lower() == 'd':
         alias = '{}D'.format(candle_size)
 
@@ -646,3 +666,70 @@ def group_assets_by_exchange(assets):
         exchange_assets[asset.exchange].append(asset)
 
     return exchange_assets
+
+
+def get_catalyst_symbol(market_or_symbol):
+    """
+    The Catalyst symbol.
+
+    Parameters
+    ----------
+    market_or_symbol
+
+    Returns
+    -------
+
+    """
+    if isinstance(market_or_symbol, string_types):
+        parts = market_or_symbol.split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+    else:
+        return '{}_{}'.format(
+            market_or_symbol['base'].lower(),
+            market_or_symbol['quote'].lower(),
+        )
+
+
+def save_asset_data(folder, df, decimals=8):
+    symbols = df.index.get_level_values('symbol')
+    for symbol in symbols:
+        symbol_df = df.loc[(symbols == symbol)]  # Type: pd.DataFrame
+
+        filename = os.path.join(folder, '{}.csv'.format(symbol))
+        if os.path.exists(filename):
+            print_headers = False
+
+        else:
+            print_headers = True
+
+        with open(filename, 'a') as f:
+            symbol_df.to_csv(
+                path_or_buf=f,
+                header=print_headers,
+                float_format='%.{}f'.format(decimals),
+            )
+
+
+def get_candles_df(candles, field, freq, bar_count, end_dt,
+                   previous_value=None):
+    all_series = dict()
+    for asset in candles:
+        periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
+
+        dates = [candle['last_traded'] for candle in candles[asset]]
+        values = [candle[field] for candle in candles[asset]]
+        series = pd.Series(values, index=dates)
+
+        series = series.reindex(
+            periods,
+            method='ffill',
+            fill_value=previous_value,
+        )
+        series.sort_index(inplace=True)
+        all_series[asset] = series
+
+    df = pd.DataFrame(all_series)
+    df.dropna(inplace=True)
+
+    return df

catalyst/exchange/utils/factory.py

@@ -0,0 +1,98 @@
+import os
+
+from logbook import Logger
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
+    get_exchange_folder, is_blacklist
+
+log = Logger('factory', level=LOG_LEVEL)
+exchange_cache = dict()
+
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
+                 skip_init=False):
+    key = (exchange_name, base_currency)
+    if key in exchange_cache:
+        return exchange_cache[key]
+
+    exchange_auth = get_exchange_auth(exchange_name)
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
+        )
+
+    exchange = CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+    )
+    exchange_cache[key] = exchange
+
+    if not skip_init:
+        exchange.init()
+
+    return exchange
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges
+
+
+def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
+                   base_currency=None):
+    """
+    Find exchanges filtered by a list of feature.
+
+    Parameters
+    ----------
+    features: str
+        The list of features.
+
+    skip_blacklist: bool
+    is_authenticated: bool
+    base_currency: bool
+
+    Returns
+    -------
+    list[Exchange]
+
+    """
+    exchange_names = CCXT.find_exchanges(features, is_authenticated)
+
+    exchanges = []
+    for exchange_name in exchange_names:
+        if skip_blacklist and is_blacklist(exchange_name):
+            continue
+
+        exchange = get_exchange(
+            exchange_name=exchange_name,
+            skip_init=True,
+            base_currency=base_currency,
+        )
+
+        if features is not None:
+            if 'dailyBundle' in features \
+                    and not exchange.has_bundle('daily'):
+                continue
+
+            elif 'minuteBundle' in features \
+                    and not exchange.has_bundle('minute'):
+                continue
+
+        exchanges.append(exchange)
+
+    return exchanges

catalyst/exchange/utils/live_chart_utils.py

@@ -0,0 +1,131 @@
+import matplotlib.dates as mdates
+import pandas as pd
+
+from catalyst.exchange.exchange_errors import \
+    MismatchingBaseCurrenciesExchanges
+
+fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+
+
+def format_ax(ax):
+    """
+    Trying to assign reasonable parameters to the time axis.
+
+    Parameters
+    ----------
+    ax:
+
+    """
+    # TODO: room for improvement
+    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
+    ax.xaxis.set_major_formatter(fmt)
+
+    locator = mdates.HourLocator(interval=4)
+    locator.MAXTICKS = 5000
+    ax.xaxis.set_minor_locator(locator)
+
+    datemin = pd.Timestamp.utcnow()
+    ax.set_xlim(datemin)
+
+    ax.grid(True)
+
+
+def set_legend(ax):
+    """
+    Set legend on the chart.
+
+    Parameters
+    ----------
+    ax
+
+    """
+    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
+
+
+def draw_pnl(ax, df):
+    """
+    Draw p&l line on the chart.
+
+    """
+    ax.clear()
+    ax.set_title('Performance')
+    index = df.index.unique()
+    dt = index.get_level_values(level=0)
+    pnl = index.get_level_values(level=4)
+    ax.plot(
+        dt, pnl, '-',
+        color='green',
+        linewidth=1.0,
+        label='Performance'
+    )
+
+    def perc(val):
+        return '{:2f}'.format(val)
+
+    ax.format_ydata = perc
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_custom_signals(ax, df):
+    """
+    Draw custom signals on the chart.
+
+    """
+    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
+
+    ax.clear()
+    ax.set_title('Custom Signals')
+    for index, column in enumerate(df.columns.values.tolist()):
+        ax.plot(df.index, df[column], '-',
+                color=colors[index],
+                linewidth=1.0,
+                label=column
+                )
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_exposure(ax, df, context):
+    """
+    Draw exposure line on the chart.
+
+    """
+    # TODO: list exchanges in graph
+    base_currency = None
+    positions = []
+    for exchange_name in context.exchanges:
+        exchange = context.exchanges[exchange_name]
+
+        if not base_currency:
+            base_currency = exchange.base_currency
+        elif base_currency != exchange.base_currency:
+            raise MismatchingBaseCurrenciesExchanges(
+                base_currency=base_currency,
+                exchange_name=exchange.name,
+                exchange_currency=exchange.base_currency
+            )
+
+        positions += exchange.portfolio.positions
+
+    ax.clear()
+    ax.set_title('Exposure')
+    ax.plot(df.index, df['base_currency'], '-',
+            color='green',
+            linewidth=1.0,
+            label='Base Currency: {}'.format(base_currency.upper())
+            )
+
+    symbols = []
+    for position in positions:
+        symbols.append(position.symbol)
+
+    ax.plot(df.index, df['long_exposure'], '-',
+            color='blue',
+            linewidth=1.0,
+            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
+
+    set_legend(ax)
+    format_ax(ax)

catalyst/exchange/utils/serialization_utils.py

@@ -0,0 +1,70 @@
+import json
+import re
+from json import JSONEncoder
+
+import pandas as pd
+from six import string_types
+
+from catalyst.constants import DATE_TIME_FORMAT
+
+
+class ExchangeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pd.Timestamp):
+            return obj.strftime(DATE_TIME_FORMAT)
+
+        # Let the base class default method raise the TypeError
+        return JSONEncoder.default(self, obj)
+
+
+class ExchangeJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        json.JSONDecoder.__init__(
+            self, object_hook=self.object_hook, *args, **kwargs
+        )
+
+    def recursive_iter(self, obj):
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                match = isinstance(value, string_types) and re.search(
+                    r'(\d{4}-\d{2}-\d{2}).*', value
+                )
+                if match:
+                    try:
+                        obj[key] = pd.to_datetime(value, utc=True)
+                    except ValueError:
+                        pass
+
+        elif any(isinstance(obj, t) for t in (list, tuple)):
+            for item in obj:
+                self.recursive_iter(item)
+
+    def object_hook(self, obj):
+        self.recursive_iter(obj)
+        return obj
+
+
+def portfolio_to_dict(portfolio):
+    positions = []
+    for asset in portfolio.positions:
+        p = portfolio.positions[asset]  # Type: Position
+
+        position = dict(
+            symbol=asset.symbol,
+            exchange=asset.exchange,
+            amount=p.amount,
+            cost_basis=p.cost_basis,
+            last_sale_price=p.last_sale_price,
+            last_sale_date=p.last_sale_date,
+        )
+        positions.append(position)
+
+    portfolio_dict = vars(portfolio)
+    portfolio_dict['positions'] = positions
+
+    return portfolio_dict
+
+
+def portfolio_from_dict(self, portfolio_data):
+    from catalyst.protocol import Portfolio
+    return Portfolio()

catalyst/exchange/utils/stats_utils.py

@@ -1,18 +1,19 @@
-import csv
-import numbers
-
 import copy
-import numpy as np
+import csv
+import json
+import numbers
 import os
-import pandas as pd
-import boto3
 import time
 
+import numpy as np
+import pandas as pd
 from catalyst.assets._assets import TradingPair
 
-from catalyst.exchange.exchange_utils import get_algo_folder
+from catalyst.exchange.utils.exchange_utils import get_algo_folder
+from catalyst.utils.paths import data_root, ensure_directory
 
-s3 = boto3.resource('s3')
+s3_conn = []
+mailgun = []
 
 
 def trend_direction(series):
@@ -195,6 +196,9 @@ def prepare_stats(stats, recorded_cols=list()):
         if recorded_cols is not None:
             for column in recorded_cols[:]:
                 value = row_data[column]
+                if isinstance(value, pd.Series):
+                    value = value.to_dict()
+
                 if type(value) is dict:
                     for asset in value:
                         if not isinstance(asset, TradingPair):
@@ -278,21 +282,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     if isinstance(stats, pd.DataFrame):
         stats = stats.T.to_dict().values()
 
-    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+    display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
+    df, columns = prepare_stats(
+        display_stats, recorded_cols=recorded_cols
+    )
 
     pd.set_option('display.expand_frame_repr', False)
     pd.set_option('precision', 8)
     pd.set_option('display.width', 1000)
     pd.set_option('display.max_colwidth', 1000)
 
-    formatters = {
-        'returns': lambda returns: "{0:.4f}".format(returns),
-    }
-
-    return df.tail(num_rows).to_string(
-        columns=columns,
-        formatters=formatters
-    )
+    return df.to_string(columns=columns)
 
 
 def get_csv_stats(stats, recorded_cols=None):
@@ -338,6 +338,12 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     -------
 
     """
+    if not s3_conn:
+        import boto3
+        s3_conn.append(boto3.resource('s3'))
+
+    s3 = s3_conn[0]
+
     if bytes_to_write is None:
         bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
 
@@ -352,6 +358,35 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     obj.put(Body=bytes_to_write)
 
 
+def email_error(algo_name, dt, e, environ=None):
+    import requests
+    import traceback
+
+    if not mailgun:
+        root = data_root(environ)
+        filename = os.path.join(root, 'mailgun.json')
+        if not os.path.exists(filename):
+            raise ValueError(
+                'mailgun.json not found in the catalyst data folder'
+            )
+
+        with open(filename) as data_file:
+            mailgun.append(json.load(data_file))
+
+    mg = mailgun[0]
+
+    return requests.post(
+        mg['url'],
+        auth=("api", mg['api']),
+        data={
+            "from": mg['from'],
+            "to": mg['to'],
+            "subject": 'Error: {}'.format(algo_name),
+            "text": '{}\n\n{}\n{}'.format(
+                dt, e, traceback.format_exc()
+            )})
+
+
 def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     """
     Saves the performance stats to the algo local folder.
@@ -372,7 +407,10 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     timestr = time.strftime('%Y%m%d')
     folder = get_algo_folder(algo_namespace)
 
-    filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
+    stats_folder = os.path.join(folder, 'stats')
+    ensure_directory(stats_folder)
+
+    filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
 
     with open(filename, 'wb') as handle:
         handle.write(bytes_to_write)

catalyst/exchange/utils/test_utils.py

@@ -0,0 +1,83 @@
+import os
+import random
+import tempfile
+
+from catalyst.assets._assets import TradingPair
+
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import find_exchanges
+from catalyst.utils.paths import ensure_directory
+
+
+def handle_exchange_error(exchange, e):
+    try:
+        message = '{}: {}'.format(
+            e.__class__, e.message.decode('ascii', 'ignore')
+        )
+    except Exception:
+        message = 'unexpected error'
+
+    folder = get_exchange_folder(exchange.name)
+    filename = os.path.join(folder, 'blacklist.txt')
+    with open(filename, 'wt') as handle:
+        handle.write(message)
+
+
+def select_random_exchanges(population=3, features=None,
+                            is_authenticated=False, base_currency=None):
+    all_exchanges = find_exchanges(
+        features=features,
+        is_authenticated=is_authenticated,
+        base_currency=base_currency,
+    )
+
+    if population is not None:
+        if len(all_exchanges) < population:
+            population = len(all_exchanges)
+
+        exchanges = random.sample(all_exchanges, population)
+
+    else:
+        exchanges = all_exchanges
+
+    return exchanges
+
+
+def select_random_assets(all_assets, population=3):
+    assets = random.sample(all_assets, population)
+    return assets
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path

catalyst/exchange/validator.py

@@ -1,142 +0,0 @@
-import os
-import tempfile
-
-import pandas as pd
-import six
-from catalyst.assets._assets import TradingPair, get_calendar
-from logbook import Logger
-from pandas.util.testing import assert_frame_equal
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
-from catalyst.exchange.factory import get_exchanges
-from catalyst.utils.paths import ensure_directory
-
-log = Logger('Validator', level=LOG_LEVEL)
-
-
-def output_df(df, assets, name=None):
-    """
-    Outputs a price DataFrame to a temp folder.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-    assets
-    name
-
-    Returns
-    -------
-
-    """
-    if isinstance(assets, TradingPair):
-        exchange_folder = assets.exchange
-        asset_folder = assets.symbol
-    else:
-        exchange_folder = ','.join([asset.exchange for asset in assets])
-        asset_folder = ','.join([asset.symbol for asset in assets])
-
-    folder = os.path.join(
-        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
-    )
-    ensure_directory(folder)
-
-    if name is None:
-        name = 'output'
-
-    path = os.path.join(folder, '{}.csv'.format(name))
-    df.to_csv(path)
-
-    return path
-
-
-class Validator(object):
-    def __init__(self, data_portal):
-        self.data_portal = data_portal
-
-    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     sample_minutes):
-        """
-        Creates DataFrames from the bundle and exchange for the specified
-        data set.
-
-        Parameters
-        ----------
-        exchange: Exchange
-        assets
-        end_dt
-        bar_count
-        sample_minutes
-
-        Returns
-        -------
-
-        """
-        freq = '{}T'.format(sample_minutes)
-
-        log.info('creating data sample from bundle')
-        df1 = self.data_portal.get_history_window(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=bar_count,
-            frequency=freq,
-            field='close',
-            data_frequency='minute'
-        )
-        path = output_df(df1, assets, '{}_resampled'.format(freq))
-        log.info('saved resampled bundle candles: {}\n{}'.format(
-            path, df1.tail(10))
-        )
-
-        log.info('creating data sample from exchange api')
-        candles = exchange.get_candles(
-            end_dt=end_dt,
-            freq='{}T'.format(sample_minutes),
-            assets=assets,
-            bar_count=bar_count
-        )
-
-        series = dict()
-        for asset in assets:
-            series[asset] = pd.Series(
-                data=[candle['close'] for candle in candles[asset]],
-                index=[candle['last_traded'] for candle in candles[asset]]
-            )
-
-        df2 = pd.DataFrame(series)
-        path = output_df(df2, assets, '{}_api'.format(freq))
-        log.info('saved exchange api candles: {}\n{}'.format(
-            path, df2.tail(10))
-        )
-
-        try:
-            assert_frame_equal(df1, df2)
-            return True
-        except:
-            log.warn('differences found in dataframes')
-            return False
-
-
-if __name__ == '__main__':
-    exchanges = get_exchanges(['poloniex'])
-    exchange = six.next(six.itervalues(exchanges))
-    assets = exchange.get_assets(symbols=['eth_btc'])
-
-    open_calendar = get_calendar('OPEN')
-    asset_finder = AssetFinderExchange()
-    data_portal = DataPortalExchangeBacktest(
-        exchanges=exchanges,
-        asset_finder=asset_finder,
-        trading_calendar=open_calendar,
-        first_trading_day=None  # will set dynamically based on assets
-    )
-    validator = Validator(data_portal=data_portal)
-
-    validator.compare_bundle_with_exchange(
-        exchange=exchange,
-        assets=assets,
-        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
-        bar_count=200,
-        sample_minutes=30
-    )

catalyst/finance/risk/cumulative.py

@@ -27,15 +27,15 @@ from .risk import (
     choose_treasury
 )
 
-from empyrical import (
+from catalyst.patches.stats import (
     alpha_beta_aligned,
     annual_volatility,
-    cum_returns,
     downside_risk,
     information_ratio,
     max_drawdown,
     sharpe_ratio,
     sortino_ratio,
+    cum_returns,
 )
 import warnings
 from catalyst.constants import LOG_LEVEL
@@ -161,9 +161,13 @@ class RiskMetricsCumulative(object):
             if len(self.algorithm_returns) == 1:
                 self.algorithm_returns = np.append(0.0, self.algorithm_returns)
 
-        self.algorithm_cumulative_returns[dt_loc] = cum_returns(
-            self.algorithm_returns
-        )[-1]
+        try:
+            self.algorithm_cumulative_returns[dt_loc] = cum_returns(
+                self.algorithm_returns
+            )[-1]
+        except Exception as e:
+            log.debug('unable to calculate cum returns: {}'.format(e))
+            self.algorithm_cumulative_returns[dt_loc] = np.nan
 
         algo_cumulative_returns_to_date = \
             self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -196,8 +200,11 @@ class RiskMetricsCumulative(object):
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
-        except Exception:
-            self.benchmark_cumulative_returns[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.benchmark_cumulative_returns[dt_loc] = np.nan
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -269,9 +276,16 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.sharpe[dt_loc] = sharpe_ratio(
             self.algorithm_returns,
         )
-        self.downside_risk[dt_loc] = downside_risk(
-            self.algorithm_returns
-        )
+
+        try:
+            self.downside_risk[dt_loc] = downside_risk(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate downside risk returns: {}'.format(e)
+            )
+            self.downside_risk[dt_loc] = np.nan
 
         try:
             risk = self.downside_risk[dt_loc]
@@ -279,17 +293,26 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
                 self.algorithm_returns,
                 _downside_risk=risk
             )
-        except Exception:
-            # TODO: what causes it to error out?
-            self.sortino[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.sortino[dt_loc] = np.nan
 
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
         )
-        self.max_drawdown = max_drawdown(
-            self.algorithm_returns
-        )
+        try:
+            self.max_drawdown = max_drawdown(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate max drawdown: {}'.format(e)
+            )
+            self.max_drawdown = np.nan
+
         self.max_drawdowns[dt_loc] = self.max_drawdown
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage

catalyst/pipeline/engine.py

@@ -7,6 +7,7 @@ from abc import (
 )
 from uuid import uuid4
 
+import six
 from six import (
     iteritems,
     with_metaclass,
@@ -33,7 +34,6 @@ from catalyst.utils.sharedoc import copydoc
 
 
 class PipelineEngine(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def run_pipeline(self, pipeline, start_date, end_date):
         """
@@ -118,6 +118,7 @@ class ExplodingPipelineEngine(PipelineEngine):
     """
     A PipelineEngine that doesn't do anything.
     """
+
     def run_pipeline(self, pipeline, start_date, end_date):
         raise NoEngineRegistered(
             "Attempted to run a pipeline but no pipeline "
@@ -484,8 +485,10 @@ class SimplePipelineEngine(PipelineEngine):
             )
 
             if isinstance(term, LoadableTerm):
+                term_key = loader_group_key(term)
+                # TODO: temp workaround
                 to_load = sorted(
-                    loader_groups[loader_group_key(term)],
+                    six.next(six.itervalues(loader_groups)),
                     key=lambda t: t.dataset
                 )
                 loader = get_loader(term)
@@ -565,9 +568,10 @@ class SimplePipelineEngine(PipelineEngine):
                 index=MultiIndex.from_arrays([empty_dates, empty_assets]),
             )
 
-        resolved_assets = array(self._finder.retrieve_all(assets))
+        # TODO: not sure what's wrong with the resolved_assets
+        # resolved_assets = array(self._finder.retrieve_all(assets))
         dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
-        assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
+        assets_kept = repeat_first_axis(assets, len(dates))[mask]
 
         final_columns = {}
         for name in data:

catalyst/support/issue_126.py

@@ -0,0 +1,52 @@
+from catalyst import run_algorithm
+from catalyst.api import order, record, symbol
+import pandas as pd
+
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+
+
+def initialize(context):
+    context.assets = [symbol('eth_btc'), symbol('eth_usdt')]
+
+
+def handle_data(context, data):
+    order(context.assets[0], 1)
+
+    prices = data.current(context.assets, 'price')
+    record(price=prices)
+    pass
+
+
+def analyze(context, perf):
+    stats = get_pretty_stats(perf)
+    print(stats)
+    pass
+
+
+if __name__ == '__main__':
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='btc',
+            analyze=analyze,
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='usd',
+            analyze=analyze,
+            start=pd.to_datetime('2017-01-01', utc=True),
+            end=pd.to_datetime('2017-12-25', utc=True),
+        )

catalyst/support/issue_30.py

@@ -0,0 +1,28 @@
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+
+def initialize(context):
+    context.asset = symbol('bcc_usdt')
+
+
+def handle_data(context, data):
+    data.history(context.asset, ['close'], bar_count=100, frequency='5T')
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=100,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bittrex',
+        algo_namespace="bittrex_is_broken",
+        base_currency='usdt',
+        data_frequency='minute',
+        simulate_orders=True,
+        live=True)

catalyst/utils/run_algo.py

@@ -12,7 +12,9 @@ from logbook import Logger
 
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
+    TradingPairPricing
+from catalyst.exchange.utils.factory import get_exchange
 
 try:
     from pygments import highlight
@@ -37,7 +39,7 @@ from catalyst.exchange.exchange_algorithm import (
 )
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, NotEnoughCapitalError)
@@ -68,35 +70,7 @@ class _RunAlgoError(click.ClickException, ValueError):
         return self.pyfunc_msg
 
 
-def _run(handle_data,
-         initialize,
-         before_trading_start,
-         analyze,
-         algofile,
-         algotext,
-         defines,
-         data_frequency,
-         capital_base,
-         data,
-         bundle,
-         bundle_timestamp,
-         start,
-         end,
-         output,
-         print_algo,
-         local_namespace,
-         environ,
-         live,
-         exchange,
-         algo_namespace,
-         base_currency,
-         live_graph,
-         simulate_orders,
-         stats_output):
-    """Run a backtest for the given algorithm.
-
-    This is shared between the cli and :func:`catalyst.run_algo`.
-    """
+def _build_namespace(algotext, local_namespace, defines):
     if algotext is not None:
         if local_namespace:
             ip = get_ipython()  # noqa
@@ -110,164 +84,197 @@ def _run(handle_data,
             except ValueError:
                 raise ValueError(
                     'invalid define %r, should be of the form name=value' %
-                    assign,
-                )
+                    assign)
             try:
                 # evaluate in the same namespace so names may refer to
                 # eachother
                 namespace[name] = eval(value, namespace)
             except Exception as e:
                 raise ValueError(
-                    'failed to execute definition for name %r: %s' % (name, e),
-                )
+                    'failed to execute definition for name %r: %s' % (name, e))
     elif defines:
         raise _RunAlgoError(
             'cannot pass define without `algotext`',
-            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
-        )
+            "cannot pass '-D' / '--define' without '-t' / '--algotext'")
     else:
         namespace = {}
-        if algofile is not None:
-            algotext = algofile.read()
 
-    if print_algo:
-        if PYGMENTS:
-            highlight(
-                algotext,
-                PythonLexer(),
-                TerminalFormatter(),
-                outfile=sys.stdout,
-            )
-        else:
-            click.echo(algotext)
+    return namespace
 
-    mode = 'paper-trading' if simulate_orders else 'live-trading' \
-        if live else 'backtest'
-    log.info('running algo in {mode} mode'.format(mode=mode))
 
+def _mode(simulate_orders, live):
+    if not live:
+        return 'backtest'
+    elif simulate_orders:
+        return 'paper-trading'
+    else:
+        return 'live-trading'
+
+
+def _build_exchanges_dict(exchange, live, simulate_orders, base_currency):
     exchange_name = exchange
     if exchange_name is None:
         raise ValueError('Please specify at least one exchange.')
 
     exchange_list = [x.strip().lower() for x in exchange.split(',')]
 
-    exchanges = dict()
-    for exchange_name in exchange_list:
-        exchanges[exchange_name] = get_exchange(
-            exchange_name=exchange_name,
+    exchanges = {exchange_name: get_exchange(
+                                    exchange_name=exchange_name,
+                                    base_currency=base_currency,
+                                    must_authenticate=(live and not simulate_orders))
+                 for exchange_name in exchange_list}
+
+    return exchanges
+
+
+def _pretty_print_code(algotext):
+    if PYGMENTS:
+        highlight(
+            algotext,
+            PythonLexer(),
+            TerminalFormatter(),
+            outfile=sys.stdout)
+    else:
+        click.echo(algotext)
+
+
+def _choose_loader(data_frequency, column):
+    bound_cols = TradingPairPricing.columns
+    if column in bound_cols:
+        return ExchangePricingLoader(data_frequency)
+    raise ValueError(
+        "No PipelineLoader registered for column %s." % column)
+
+
+def _get_live_time_range():
+    start = pd.Timestamp.utcnow()
+    # TODO: fix the end data.
+    end = start + timedelta(hours=8760)
+    return start, end
+
+
+def _data_for_live_trading(sim_params, exchanges, env, open_calendar):
+    data = DataPortalExchangeLive(
+        exchanges=exchanges,
+        asset_finder=env.asset_finder,
+        trading_calendar=open_calendar,
+        first_trading_day=pd.to_datetime('today', utc=True))
+
+    return data
+
+
+# TODO use proper retry here
+def _fetch_capital_base(base_currency, exchange_name, exchange,
+                        attempt_index=0):
+    """
+    Fetch the base currency amount required to bootstrap
+    the algorithm against the exchange.
+
+    The algorithm cannot continue without this value.
+
+    :param exchange: the targeted exchange
+    :param attempt_index:
+    :return capital_base: the amount of base currency available for
+    trading
+    """
+    try:
+        log.debug('retrieving capital base in {} to bootstrap '
+                  'exchange {}'.format(base_currency, exchange_name))
+        balances = exchange.get_balances()
+    except ExchangeRequestError as e:
+        if attempt_index < 20:
+            log.warn(
+                'could not retrieve balances on {}: {}'.format(
+                    exchange.name, e))
+            sleep(5)
+            return _fetch_capital_base(base_currency, exchange_name, exchange,
+                                       attempt_index + 1)
+
+        else:
+            raise ExchangeRequestErrorTooManyAttempts(
+                attempts=attempt_index,
+                error=e)
+
+    if base_currency in balances:
+        base_currency_available = balances[base_currency]['free']
+        log.info(
+            'base currency available in the account: {} {}'.format(
+                base_currency_available, base_currency))
+
+        return base_currency_available
+    else:
+        raise BaseCurrencyNotFoundError(
             base_currency=base_currency,
-            must_authenticate=(live and not simulate_orders),
-        )
+            exchange=exchange_name)
 
-    open_calendar = get_calendar('OPEN')
 
-    env = TradingEnvironment(
-        load=partial(
-            load_crypto_market_data,
-            environ=environ,
-            start_dt=start,
-            end_dt=end
-        ),
-        environ=environ,
-        exchange_tz='UTC',
-        asset_db_path=None  # We don't need an asset db, we have exchanges
-    )
-    env.asset_finder = AssetFinderExchange()
-    choose_loader = None  # TODO: use the DataPortal in the algo class for this
+def _algorithm_class_for_live(algo_namespace, live_graph, stats_output,
+                              analyze_live, base_currency, simulate_orders,
+                              exchanges, capital_base):
+    if not simulate_orders:
+        for exchange_name in exchanges:
+            exchange = exchanges[exchange_name]
+            balance = _fetch_capital_base(base_currency, exchange_name,
+                                          exchange)
 
-    if live:
-        start = pd.Timestamp.utcnow()
-
-        # TODO: fix the end data.
-        end = start + timedelta(hours=8760)
-
-        data = DataPortalExchangeLive(
-            exchanges=exchanges,
-            asset_finder=env.asset_finder,
-            trading_calendar=open_calendar,
-            first_trading_day=pd.to_datetime('today', utc=True)
-        )
-
-        def fetch_capital_base(exchange, attempt_index=0):
-            """
-            Fetch the base currency amount required to bootstrap
-            the algorithm against the exchange.
-
-            The algorithm cannot continue without this value.
-
-            :param exchange: the targeted exchange
-            :param attempt_index:
-            :return capital_base: the amount of base currency available for
-            trading
-            """
-            try:
-                log.debug('retrieving capital base in {} to bootstrap '
-                          'exchange {}'.format(base_currency, exchange_name))
-                balances = exchange.get_balances()
-            except ExchangeRequestError as e:
-                if attempt_index < 20:
-                    log.warn(
-                        'could not retrieve balances on {}: {}'.format(
-                            exchange.name, e
-                        )
-                    )
-                    sleep(5)
-                    return fetch_capital_base(exchange, attempt_index + 1)
-
-                else:
-                    raise ExchangeRequestErrorTooManyAttempts(
-                        attempts=attempt_index,
-                        error=e
-                    )
-
-            if base_currency in balances:
-                base_currency_available = balances[base_currency]['free']
-                log.info(
-                    'base currency available in the account: {} {}'.format(
-                        base_currency_available, base_currency
-                    )
-                )
-
-                return base_currency_available
-            else:
-                raise BaseCurrencyNotFoundError(
+            if balance < capital_base:
+                raise NotEnoughCapitalError(
+                    exchange=exchange_name,
                     base_currency=base_currency,
-                    exchange=exchange_name
-                )
+                    balance=balance,
+                    capital_base=capital_base)
 
-        if not simulate_orders:
-            for exchange_name in exchanges:
-                exchange = exchanges[exchange_name]
-                balance = fetch_capital_base(exchange)
+    algorithm_class = partial(
+        ExchangeTradingAlgorithmLive,
+        exchanges=exchanges,
+        algo_namespace=algo_namespace,
+        live_graph=live_graph,
+        simulate_orders=simulate_orders,
+        stats_output=stats_output,
+        analyze_live=analyze_live,)
 
-                if balance < capital_base:
-                    raise NotEnoughCapitalError(
-                        exchange=exchange_name,
-                        base_currency=base_currency,
-                        balance=balance,
-                        capital_base=capital_base,
-                    )
+    return algorithm_class
 
-        sim_params = create_simulation_parameters(
-            start=start,
-            end=end,
-            capital_base=capital_base,
-            emission_rate='minute',
-            data_frequency='minute'
-        )
 
-        # TODO: use the constructor instead
-        sim_params._arena = 'live'
+def _bundle_trading_environment(bundle_data, environ):
+    prefix, connstr = re.split(
+        r'sqlite:///',
+        str(bundle_data.asset_finder.engine.url),
+        maxsplit=1)
+    if prefix:
+        raise ValueError(
+            "invalid url %r, must begin with 'sqlite:///'" %
+            str(bundle_data.asset_finder.engine.url))
 
-        algorithm_class = partial(
-            ExchangeTradingAlgorithmLive,
-            exchanges=exchanges,
-            algo_namespace=algo_namespace,
-            live_graph=live_graph,
-            simulate_orders=simulate_orders,
-            stats_output=stats_output,
-        )
-    elif exchanges:
+    return TradingEnvironment(asset_db_path=connstr, environ=environ)
+
+
+def _build_live_algo_and_data(sim_params, exchanges, env, open_calendar,
+                              simulate_orders, algo_namespace, capital_base,
+                              live_graph, stats_output, analyze_live,
+                              base_currency, namespace, choose_loader,
+                              algorithm_class_kwargs):
+    sim_params._arena = 'live'  # TODO: use the constructor instead
+
+    data = _data_for_live_trading(sim_params, exchanges, env, open_calendar)
+
+    algorithm_class = _algorithm_class_for_live(
+        algo_namespace, live_graph, stats_output, analyze_live,
+        base_currency, simulate_orders, exchanges, capital_base)
+
+    return data, algorithm_class(
+        namespace=namespace,
+        env=env,
+        get_pipeline_loader=choose_loader,
+        sim_params=sim_params,
+        **algorithm_class_kwargs)
+
+
+def _build_backtest_algo_and_data(
+        exchanges, bundle, env, environ, bundle_timestamp, open_calendar,
+        start, end, namespace, choose_loader, sim_params,
+        algorithm_class_kwargs):
+    if exchanges:
         # Removed the existing Poloniex fork to keep things simple
         # We can add back the complexity if required.
 
@@ -281,41 +288,19 @@ def _run(handle_data,
             asset_finder=None,
             trading_calendar=open_calendar,
             first_trading_day=start,
-            last_available_session=end
-        )
-
-        sim_params = create_simulation_parameters(
-            start=start,
-            end=end,
-            capital_base=capital_base,
-            data_frequency=data_frequency,
-            emission_rate=data_frequency,
-        )
+            last_available_session=end)
 
         algorithm_class = partial(
             ExchangeTradingAlgorithmBacktest,
-            exchanges=exchanges
-        )
-
+            exchanges=exchanges)
     elif bundle is not None:
-        bundle_data = load(
-            bundle,
-            environ,
-            bundle_timestamp,
-        )
+        # TODO This branch should probably be removed or fixed: it doesn't even
+        # build `algorithm_class`, so it will break when trying to instantiate
+        # it.
+        bundle_data = load(bundle, environ, bundle_timestamp)
 
-        prefix, connstr = re.split(
-            r'sqlite:///',
-            str(bundle_data.asset_finder.engine.url),
-            maxsplit=1,
-        )
-        if prefix:
-            raise ValueError(
-                "invalid url %r, must begin with 'sqlite:///'" %
-                str(bundle_data.asset_finder.engine.url),
-            )
+        env = _bundle_trading_environment(bundle_data, environ)
 
-        env = TradingEnvironment(asset_db_path=connstr, environ=environ)
         first_trading_day = \
             bundle_data.equity_minute_bar_reader.first_trading_day
 
@@ -324,27 +309,103 @@ def _run(handle_data,
             first_trading_day=first_trading_day,
             equity_minute_reader=bundle_data.equity_minute_bar_reader,
             equity_daily_reader=bundle_data.equity_daily_bar_reader,
-            adjustment_reader=bundle_data.adjustment_reader,
-        )
+            adjustment_reader=bundle_data.adjustment_reader)
 
-    perf = algorithm_class(
+    return data, algorithm_class(
         namespace=namespace,
         env=env,
         get_pipeline_loader=choose_loader,
         sim_params=sim_params,
-        **{
-            'initialize': initialize,
-            'handle_data': handle_data,
-            'before_trading_start': before_trading_start,
-            'analyze': analyze,
-        } if algotext is None else {
-            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
-            'script': algotext,
-        }
-    ).run(
+        **algorithm_class_kwargs)
+
+
+def _build_algo_and_data(handle_data, initialize, before_trading_start,
+                         analyze, algofile, algotext, defines, data_frequency,
+                         capital_base, data, bundle, bundle_timestamp, start,
+                         end, output, print_algo, local_namespace, environ,
+                         live, exchange, algo_namespace, base_currency,
+                         live_graph, analyze_live, simulate_orders,
+                         stats_output):
+    namespace = _build_namespace(algotext, local_namespace, defines)
+    if algotext is not None:
+        algotext = algofile.read()
+
+    if print_algo:
+        _pretty_print_code(algotext)
+
+    mode = _mode(simulate_orders, live)
+    log.info('running algo in {mode} mode'.format(mode=mode))
+
+    exchanges = _build_exchanges_dict(exchange, live, simulate_orders,
+                                      base_currency)
+
+    open_calendar = get_calendar('OPEN')
+
+    env = TradingEnvironment(
+        load=partial(load_crypto_market_data, environ=environ, start_dt=start,
+                     end_dt=end),
+        environ=environ,
+        exchange_tz='UTC',
+        asset_db_path=None)  # We don't need an asset db, we have exchanges
+
+    env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
+
+    choose_loader = partial(_choose_loader, data_frequency)
+
+    if live:
+        start, end = _get_live_time_range()
+        data_frequency = 'minute'  # TODO double check if this is the desired behavior
+
+    sim_params = create_simulation_parameters(
+        start=start,
+        end=end,
+        capital_base=capital_base,
+        emission_rate=data_frequency,
+        data_frequency=data_frequency)
+
+    if algotext is None:
+        algorithm_class_kwargs = {'initialize': initialize,
+                                  'handle_data': handle_data,
+                                  'before_trading_start': before_trading_start,
+                                  'analyze': analyze}
+    else:
+        algorithm_class_kwargs = {'algo_filename': getattr(algofile, 'name',
+                                                           '<algorithm>'),
+                                  'script': algotext}
+
+    if live:
+        return _build_live_algo_and_data(
+            sim_params, exchanges, env, open_calendar, simulate_orders,
+            algo_namespace, capital_base, live_graph, stats_output,
+            analyze_live, base_currency, namespace, choose_loader,
+            algorithm_class_kwargs)
+    else:
+        return _build_backtest_algo_and_data(
+            exchanges, bundle, env, environ, bundle_timestamp, open_calendar,
+            start, end, namespace, choose_loader, sim_params,
+            algorithm_class_kwargs)
+
+
+def _run(handle_data, initialize, before_trading_start, analyze, algofile,
+         algotext, defines, data_frequency, capital_base, data, bundle,
+         bundle_timestamp, start, end, output, print_algo, local_namespace,
+         environ, live, exchange, algo_namespace, base_currency, live_graph,
+         analyze_live, simulate_orders, stats_output):
+    """Run an algorithm in backtest,
+    paper-trading or live-trading mode.
+
+    This is shared between the cli and :func:`catalyst.run_algo`.
+    """
+
+    data, algorithm = _build_algo_and_data(
+        handle_data, initialize, before_trading_start, analyze, algofile,
+        algotext, defines, data_frequency, capital_base, data, bundle,
+        bundle_timestamp, start, end, output, print_algo, local_namespace,
+        environ, live, exchange, algo_namespace, base_currency, live_graph,
+        analyze_live, simulate_orders, stats_output)
+    perf = algorithm.run(
         data,
-        overwrite_sim_params=False,
-    )
+        overwrite_sim_params=False)
 
     if output == '-':
         click.echo(str(perf))
@@ -401,8 +462,7 @@ def load_extensions(default, extensions, strict, environ, reload=False):
             # without `strict` we should just log the failure
             warnings.warn(
                 'Failed to load extension: %r\n%s' % (ext, e),
-                stacklevel=2
-            )
+                stacklevel=2)
         else:
             _loaded_extensions.add(ext)
 
@@ -427,6 +487,7 @@ def run_algorithm(initialize,
                   base_currency=None,
                   algo_namespace=None,
                   live_graph=False,
+                  analyze_live=None,
                   simulate_orders=True,
                   stats_output=None,
                   output=os.devnull):
@@ -500,8 +561,7 @@ def run_algorithm(initialize,
     catalyst.data.bundles.bundles : The available data bundles.
     """
     load_extensions(
-        default_extension, extensions, strict_extensions, environ
-    )
+        default_extension, extensions, strict_extensions, environ)
 
     if capital_base is None:
         raise ValueError(
@@ -509,8 +569,7 @@ def run_algorithm(initialize,
             'amount of base currency available for trading. For example, '
             'if the `capital_base` is 5ETH, the '
             '`order_target_percent(asset, 1)` command will order 5ETH worth '
-            'of the specified asset.'
-        )
+            'of the specified asset.')
     # I'm not sure that we need this since the modified DataPortal
     # does not require extensions to be explicitly loaded.
 
@@ -528,13 +587,11 @@ def run_algorithm(initialize,
         elif len(non_none_data) != 1:
             raise ValueError(
                 'must specify one of `data`, `data_portal`, or `bundle`,'
-                ' got: %r' % non_none_data,
-            )
+                ' got: %r' % non_none_data)
 
         elif 'bundle' not in non_none_data and bundle_timestamp is not None:
             raise ValueError(
-                'cannot specify `bundle_timestamp` without passing `bundle`',
-            )
+                'cannot specify `bundle_timestamp` without passing `bundle`')
     return _run(
         handle_data=handle_data,
         initialize=initialize,
@@ -559,6 +616,6 @@ def run_algorithm(initialize,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=analyze_live,
         simulate_orders=simulate_orders,
-        stats_output=stats_output
-    )
+        stats_output=stats_output)

docs/source/install.rst

@@ -15,6 +15,9 @@ as an alternative installation method for MacOS and Linux, you can install
 Catalyst directly with ``pip`` (we recommend in combination with a virtual 
 environemnt). See :ref:`Installing with pip <pip>`.
 
+Alternatively you can install Catalyst using ``pipenv`` which is a mix of pip
+and virtualenv. See :ref:`Installing with pipenv <pipenv>`.
+
 Regardless of the method, each operating system (OS), has its own 
 prerequisites, make sure to review the corresponding sections for your system:
 :ref:`Linux <linux>`, :ref:`MacOS <macos>` and :ref:`Windows <windows>`.
@@ -293,6 +296,39 @@ Troubleshooting ``pip`` Install
 
       sudo apt-get install python-dev
 
+.. _pipenv:
+
+Installing with ``pipenv``
+-------------------------
+
+Installing Catalyst via ``pipenv`` is perhaps easier that installing it via
+``pip`` itself but you need to install ``pipenv`` first via ``pip``.
+
+.. code-block:: bash
+
+   $ pip install pipenv
+
+Once ``pipenv`` is installed you can proceed by creating a project folder and
+installing Catalyst on that project automagically as follows:
+
+.. code-block:: bash
+
+   $ mkdir project
+   $ cd project
+   $ pipenv --two
+   $ pipenv install enigma-catalyst matplotlib
+
+Until now the workflow compared to ``pip`` is almost identical, the difference
+is that you don't need to load manually any virtualenv however you need to use
+the `pipenv run` prefix to run the `catalyst` command as follows:
+
+.. code-block:: bash
+
+   $ pipenv run catalyst --version
+
+If you want to know more about ``pipenv`` go to the `pipenv github repo`_
+
+.. _`pipenv github repo`: https://github.com/pypa/pipenv
 
 .. _linux:
 

docs/source/releases.rst

@@ -2,23 +2,31 @@
 Release Notes
 =============
 
-Version 0.3.10
+Version 0.4.1
+^^^^^^^^^^^^^
+**Release Date**: 2017-01-03
+
+Bug Fixes
+~~~~~~~~~
+- Fixed cash synchronization issue (:issue:`133`)
+- Fixed positions synchronization issue (:issue:`132`)
+- Patched empyrical to resolve a np.log1p issue (:issue:`126`)
+- Fixed a paper trading issue (:issue:`124`)
+- Fixed a commission issue (:issue:`104`)
+- Fixed a poloniex specific issue in live trading (:issue:`103`)
+
+Build
+~~~~~
+- Caching CCXT market info to limit round-trips (:issue:`99`)
+- Tentative support for Pipeline (:issue:`96`)
+
+Version 0.4.0
 ^^^^^^^^^^^^^
 **Release Date**: 2017-12-12
 
 Bug Fixes
 ~~~~~~~~~
 
-- Fixed issue with fetching assets with daily frequency
-
-Version 0.3.10
-^^^^^^^^^^^^^
-**Release Date**: 2017-11-28
-
-Bug Fixes
-~~~~~~~~~
-
-- Fixed issue with fetching assets with daily frequency
 - Changed Poloniex interface (should solve :issue:`95` and :issue:`94`)
 - Solved issue with overriding commission and slippage (:issue:`87`)
 - Fixed inefficiency with Bittrex current prices (:issue:`76`)
@@ -30,6 +38,15 @@ Build
 - More granular commissions (:issue:`82`)
 - Added market orders in live mode (:issue:`81`)
 
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with fetching assets with daily frequency
+
 Version 0.3.9
 ^^^^^^^^^^^^^
 **Release Date**: 2017-11-28

docs/source/unit-tests.rst

@@ -0,0 +1,88 @@
+==========
+Unit Tests
+==========
+
+Exchanges
+~~~~~~~~~
+
+Markets
+-------
+Sample:
+    All markets in 3 random exchanges
+Test:
+    Fetch all TradingPair instances
+Assert:
+    No error
+
+Current Ticker
+------------------
+Sample:
+    3 random markets in each of the 3 random exchanges
+Test:
+    Fetch current price and volume
+Assert:
+    Not null and no error
+
+Historical Price Data
+---------------------
+Sample:
+    - 3 random markets for each of the 3 random exchanges supporting historical data
+    - For each market, randomly select one supported frequency
+Test:
+    Fetch historical data for each market using the selected frequency
+Assert:
+    - No error and not blank
+    - Date of each candle is consistent with the Catalyst desired pattern,
+        - All candle start at fix intervals
+        - Last candle partial and forward looking from the end date
+
+Authentication and Orders
+-------------------------
+Sample:
+    1 random market for each of 3 random authenticated exchanges
+Test:
+    - Create one limit order randomly buying or selling at least 10% out from the current price
+    - Retrieve the open order from the exchange
+    - Cancel the open order
+Assert:
+    No error
+
+
+Bundles
+~~~~~~~
+
+Validate Bundle Data
+--------------------
+Sample:
+    - 3 random market in bundles for exchanges supporting historical data
+    - For each market, randomly selected data range available in the exchange historical data
+Test:
+    - Clean the target exchange bundle
+    - Ingest the selected market data for the selected data range
+    - Retrieve the bundle data into a dataframe
+    - Retrieve the equivalent OHLCV data from the exchange into a dataframe
+Assert:
+    Matching data for the bundle and exchange
+
+
+Algo Stats
+----------
+Sample:
+    - 2 sample algorithms with built-in stats calculator
+    - 2 KPIs both calculated by each algo and by Catalyst
+Test:
+    - Run each algorithm
+    - Compare the results of the two methods or calculating stats
+Assert:
+    - Matching stats
+
+CSV Ingestion
+-------------
+Sample:
+    3 random CSV files containing price data
+Test:
+    - Ingest each CSV files
+    - Validate with the exchange like in the 'Validate Bundle Data' test
+Assert:
+    Matching data between the bundle and the exchange
+

etc/python2.7-environment.yml

@@ -20,7 +20,7 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
-  - ccxt==1.10.319
+  - ccxt==1.10.283
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0
@@ -45,6 +45,7 @@ dependencies:
   - python-dateutil==2.6.1
   - python-editor==1.0.3
   - pytz==2017.2
+  - redo==1.6
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1

etc/requirements.txt

@@ -83,3 +83,4 @@ tables==3.3.0
 #Catalyst dependencies
 ccxt==1.10.283
 boto3==1.4.8
+redo==1.6

tests/exchange/test_bcolz.py

@@ -1,15 +1,14 @@
-import shutil
 import random
+import shutil
 import tempfile
-import pandas as pd
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
+import pandas as pd
+from nose.tools import assert_equals
+
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
     BcolzExchangeBarReader
-
-from catalyst.exchange.bundle_utils import get_df_from_arrays
-
-from nose.tools import assert_equals
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.utils.bundle_utils import get_df_from_arrays
 
 
 class TestBcolzWriter(object):

tests/exchange/test_bitfinex.py

@@ -1,78 +0,0 @@
-from logbook import Logger
-
-from base import BaseExchangeTestCase
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.finance.execution import (LimitOrder)
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('test_bitfinex')
-
-
-@deprecated
-class TestBitfinex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        log.info('creating bitfinex object')
-        auth = get_exchange_auth('bitfinex')
-        self.exchange = Bitfinex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency='usd'
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('eth_usd')
-        order_id = self.exchange.order(
-            asset=asset,
-            style=LimitOrder(limit_price=200),
-            limit_price=200,
-            amount=0.5,
-            stop_price=None
-        )
-        log.info('order created {}'.format(order_id))
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # orders = self.exchange.get_open_orders()
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        pass
-
-    def test_cancel_order(self):
-        log.info('cancel order')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        # ohlcv_neo = self.exchange.get_candles(
-        #    freq='1T',
-        #    assets=self.exchange.get_asset('neo_btc'))
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        # tickers = self.exchange.tickers([
-        #     self.exchange.get_asset('eth_btc'),
-        #     self.exchange.get_asset('etc_btc')
-        # ])
-        pass
-
-    def test_get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def test_get_balances(self):
-        log.info('testing exchange balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for bitfinex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_bittrex.py

@@ -1,95 +0,0 @@
-# import pandas as pd
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.finance.order import Order
-from base import BaseExchangeTestCase
-from logbook import Logger
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('test_bittrex')
-
-
-@deprecated
-class TestBittrex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        auth = get_exchange_auth('bittrex')
-        self.exchange = Bittrex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency=None,
-            portfolio=None
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('neo_btc')
-        order_id = self.exchange.order(
-            asset=asset,
-            limit_price=0.0005,
-            amount=1,
-        )
-        log.info('order created {}'.format(order_id))
-        assert order_id is not None
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # asset = self.exchange.get_asset('neo_btc')
-        # orders = self.exchange.get_open_orders(asset)
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        order = self.exchange.get_order(
-            u'2c584020-9caf-4af5-bde0-332c0bba17e2')
-        assert isinstance(order, Order)
-        pass
-
-    def test_cancel_order(self, ):
-        log.info('cancel order')
-        self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        # ohlcv_neo = self.exchange.get_candles(
-        #     freq='5T',
-        #     assets=self.exchange.get_asset('neo_btc'),
-        #     bar_count=20,
-        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
-        # )
-        # ohlcv_neo_ubq = self.exchange.get_candles(
-        #     freq='1D',
-        #     assets=[
-        #         self.exchange.get_asset('neo_btc'),
-        #         self.exchange.get_asset('ubq_btc')
-        #     ],
-        #     bar_count=14,
-        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
-        # )
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
-        assert len(tickers) == 2
-        pass
-
-    def test_get_balances(self):
-        log.info('testing wallet balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_get_account(self):
-        log.info('testing account data')
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for bittrex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_bundle.py

@@ -5,15 +5,15 @@ from logging import getLogger
 
 import pandas as pd
 
-from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
-    get_start_dt, get_df_from_arrays
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
-from catalyst.exchange.exchange_utils import get_exchange_folder
-from catalyst.exchange.factory import get_exchange
-from catalyst.exchange.stats_utils import df_to_string
+from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
+    get_start_dt, get_df_from_arrays
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.exchange.utils.stats_utils import df_to_string
 from catalyst.utils.paths import ensure_directory
 
 log = getLogger('test_exchange_bundle')

tests/exchange/test_ccxt.py

@@ -1,10 +1,11 @@
 import pandas as pd
 from logbook import Logger
-from base import BaseExchangeTestCase
 
+from base import BaseExchangeTestCase
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth
 from catalyst.finance.order import Order
-from catalyst.exchange.exchange_utils import get_exchange_auth
 
 log = Logger('test_ccxt')
 
@@ -12,22 +13,22 @@ log = Logger('test_ccxt')
 class TestCCXT(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        exchange_name = 'gdax'
+        exchange_name = 'binance'
         auth = get_exchange_auth(exchange_name)
         self.exchange = CCXT(
             exchange_name=exchange_name,
             key=auth['key'],
             secret=auth['secret'],
             base_currency='eth',
-            portfolio=None
         )
+        self.exchange.init()
 
     def test_order(self):
         log.info('creating order')
         asset = self.exchange.get_asset('neo_eth')
         order_id = self.exchange.order(
             asset=asset,
-            limit_price=0.07,
+            style=ExchangeLimitOrder(limit_price=0.7),
             amount=1,
         )
         log.info('order created {}'.format(order_id))
@@ -68,9 +69,10 @@ class TestCCXT(BaseExchangeTestCase):
 
     def test_tickers(self):
         log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-        ])
+        assets = [
+            self.exchange.get_asset('eng_eth'),
+        ]
+        tickers = self.exchange.tickers(assets)
         assert len(tickers) == 1
         pass
 

tests/exchange/test_data_portal.py

@@ -2,13 +2,13 @@ import pandas as pd
 from logbook import Logger
 
 from catalyst import get_calendar
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
 from catalyst.exchange.exchange_data_portal import (
     DataPortalExchangeBacktest,
     DataPortalExchangeLive
 )
-from catalyst.exchange.exchange_utils import get_common_assets
-from catalyst.exchange.factory import get_exchanges
+from catalyst.exchange.utils.exchange_utils import get_common_assets
+from catalyst.exchange.utils.factory import get_exchanges
 from test_utils import rnd_history_date_days, rnd_bar_count
 
 log = Logger('test_bitfinex')
@@ -20,7 +20,7 @@ class TestExchangeDataPortal:
         log.info('creating bitfinex exchange')
         exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
         open_calendar = get_calendar('OPEN')
-        asset_finder = AssetFinderExchange()
+        asset_finder = ExchangeAssetFinder()
 
         self.data_portal_live = DataPortalExchangeLive(
             exchanges=exchanges,

tests/exchange/test_poloniex.py

@@ -1,96 +0,0 @@
-from catalyst.exchange.poloniex.poloniex import Poloniex
-from catalyst.finance.order import Order
-from base import BaseExchangeTestCase
-from logbook import Logger
-from catalyst.exchange.exchange_utils import get_exchange_auth
-import pandas as pd
-
-from catalyst.utils.deprecate import deprecated
-from test_utils import output_df
-
-log = Logger('test_poloniex')
-
-
-@deprecated
-class TestPoloniex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        print ('creating poloniex object')
-        auth = get_exchange_auth('poloniex')
-        self.exchange = Poloniex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency='btc'
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('neos_btc')
-        order_id = self.exchange.order(
-            asset=asset,
-            limit_price=0.0005,
-            amount=1,
-        )
-        log.info('order created {}'.format(order_id))
-        assert order_id is not None
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # asset = self.exchange.get_asset('neos_btc')
-        # orders = self.exchange.get_open_orders(asset)
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        order = self.exchange.get_order(
-            u'2c584020-9caf-4af5-bde0-332c0bba17e2')
-        assert isinstance(order, Order)
-        pass
-
-    def test_cancel_order(self, ):
-        log.info('cancel order')
-        self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        assets = self.exchange.get_asset('eth_btc')
-        ohlcv = self.exchange.get_candles(
-            # end_dt=pd.to_datetime('2017-11-01', utc=True),
-            end_dt=None,
-            freq='5T',
-            assets=assets,
-            bar_count=200
-        )
-        df = pd.DataFrame(ohlcv)
-        df.set_index('last_traded', drop=True, inplace=True)
-        log.info(df.tail(25))
-
-        path = output_df(df, assets, '5min_candles')
-        log.info('saved candles: {}'.format(path))
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
-        assert len(tickers) == 2
-        pass
-
-    def test_get_balances(self):
-        log.info('testing wallet balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_get_account(self):
-        log.info('testing account data')
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for poloniex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_server_bundle.py

@@ -1,17 +1,18 @@
-import os
 import importlib
+import os
 
-import pandas as pd
 import matplotlib
 import matplotlib.pyplot as plt
-from matplotlib.finance import candlestick2_ohlc
 # from matplotlib.finance import volume_overlay
 import matplotlib.ticker as ticker
+import pandas as pd
+from matplotlib.finance import candlestick2_ohlc
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
-from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
-from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.utils.bundle_utils import get_df_from_arrays, \
+    get_bcolz_chunk
+from catalyst.exchange.utils.factory import get_exchange
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 exchanges = dict((e, getattr(importlib.import_module(

tests/exchange/test_suite_bundle.py

@@ -0,0 +1,145 @@
+import random
+
+import pandas as pd
+from logbook import Logger
+from pandas.util.testing import assert_frame_equal
+
+from catalyst import get_calendar
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
+from catalyst.exchange.utils.exchange_utils import get_candles_df
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.exchange.utils.test_utils import output_df, \
+    select_random_assets
+
+log = Logger('TestSuiteExchange')
+
+pd.set_option('display.expand_frame_repr', False)
+pd.set_option('precision', 8)
+pd.set_option('display.width', 1000)
+pd.set_option('display.max_colwidth', 1000)
+
+
+class TestSuiteBundle:
+    @staticmethod
+    def get_data_portal(exchange_names):
+        open_calendar = get_calendar('OPEN')
+        asset_finder = ExchangeAssetFinder()
+
+        data_portal = DataPortalExchangeBacktest(
+            exchange_names=exchange_names,
+            asset_finder=asset_finder,
+            trading_calendar=open_calendar,
+            first_trading_day=None  # will set dynamically based on assets
+        )
+        return data_portal
+
+    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
+                                     freq, data_frequency, data_portal):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        sample_minutes
+
+        Returns
+        -------
+
+        """
+        data = dict()
+
+        log.info('creating data sample from bundle')
+        data['bundle'] = data_portal.get_history_window(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            frequency=freq,
+            field='close',
+            data_frequency=data_frequency,
+        )
+        log.info('bundle data:\n{}'.format(
+            data['bundle'].tail(10))
+        )
+
+        log.info('creating data sample from exchange api')
+        candles = exchange.get_candles(
+            end_dt=end_dt,
+            freq=freq,
+            assets=assets,
+            bar_count=bar_count,
+        )
+        data['exchange'] = get_candles_df(
+            candles=candles,
+            field='close',
+            freq=freq,
+            bar_count=bar_count,
+            end_dt=end_dt,
+        )
+        log.info('exchange data:\n{}'.format(
+            data['exchange'].tail(10))
+        )
+        for source in data:
+            df = data[source]
+            path = output_df(df, assets, '{}_{}'.format(freq, source))
+            log.info('saved {}:\n{}'.format(source, path))
+
+        assert_frame_equal(
+            right=data['bundle'],
+            left=data['exchange'],
+            check_less_precise=True,
+        )
+
+    def test_validate_bundles(self):
+        # exchange_population = 3
+        asset_population = 3
+        data_frequency = random.choice(['minute', 'daily'])
+
+        # bundle = 'dailyBundle' if data_frequency
+        #  == 'daily' else 'minuteBundle'
+        # exchanges = select_random_exchanges(
+        #     population=exchange_population,
+        #     features=[bundle],
+        # )  # Type: list[Exchange]
+        exchanges = [get_exchange('bitfinex', skip_init=True)]
+
+        data_portal = TestSuiteBundle.get_data_portal(
+            [exchange.name for exchange in exchanges]
+        )
+        for exchange in exchanges:
+            exchange.init()
+
+            frequencies = exchange.get_candle_frequencies(data_frequency)
+            freq = random.sample(frequencies, 1)[0]
+
+            bar_count = random.randint(1, 10)
+
+            assets = select_random_assets(
+                exchange.assets, asset_population
+            )
+            end_dt = None
+            for asset in assets:
+                attribute = 'end_{}'.format(data_frequency)
+                asset_end_dt = getattr(asset, attribute)
+
+                if end_dt is None or asset_end_dt < end_dt:
+                    end_dt = asset_end_dt
+
+            dt_range = pd.date_range(
+                end=end_dt, periods=bar_count, freq=freq
+            )
+            self.compare_bundle_with_exchange(
+                exchange=exchange,
+                assets=assets,
+                end_dt=dt_range[-1],
+                bar_count=bar_count,
+                freq=freq,
+                data_frequency=data_frequency,
+                data_portal=data_portal,
+            )
+        pass

tests/exchange/test_suite_exchange.py

@@ -0,0 +1,189 @@
+import json
+import os
+import random
+from logging import Logger
+from time import sleep
+
+import pandas as pd
+
+from catalyst.exchange.exchange_errors import ExchangeRequestError
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.test_utils import select_random_exchanges, \
+    handle_exchange_error, select_random_assets
+
+log = Logger('TestSuiteExchange')
+
+
+class TestSuiteExchange:
+    def _test_markets_exchange(self, exchange, attempts=0):
+        assets = None
+        try:
+            exchange.init()
+
+            # Verify that the assets and markets are populated
+            if not exchange.markets:
+                raise ValueError(
+                    'no markets found'
+                )
+            if not exchange.assets:
+                raise ValueError(
+                    'no assets derived from markets'
+                )
+            assets = exchange.assets
+
+        except ExchangeRequestError as e:
+            sleep(5)
+
+            if attempts > 5:
+                handle_exchange_error(exchange, e)
+
+            else:
+                print(
+                    're-trying an exchange request {} {}'.format(
+                        exchange.name, attempts
+                    )
+                )
+                self._test_markets_exchange(exchange, attempts + 1)
+
+        except Exception as e:
+            handle_exchange_error(exchange, e)
+
+        return assets
+
+    def test_markets(self):
+        population = 3
+        results = dict()
+
+        exchanges = select_random_exchanges(population)  # Type: list[Exchange]
+        for exchange in exchanges:
+            assets = self._test_markets_exchange(exchange)
+
+            if assets is not None:
+                results[exchange.name] = len(assets)
+
+                folder = get_exchange_folder(exchange.name)
+                filename = os.path.join(folder, 'whitelist.json')
+
+                symbols = [asset.symbol for asset in assets]
+                with open(filename, 'wt') as handle:
+                    json.dump(symbols, handle, indent=4)
+
+        series = pd.Series(results)
+        print('the tested markets\n{}'.format(series))
+
+        if population is not None:
+            assert (len(results) == population)
+
+        pass
+
+    def test_tickers(self):
+        exchange_population = 3
+        asset_population = 3
+
+        exchanges = select_random_exchanges(
+            exchange_population,
+            features=['fetchTickers'],
+        )  # Type: list[Exchange]
+        for exchange in exchanges:
+            exchange.init()
+
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+                tickers = exchange.tickers(assets)
+
+                assert len(tickers) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
+        pass
+
+    def test_candles(self):
+        exchange_population = 3
+        asset_population = 3
+
+        exchanges = select_random_exchanges(
+            population=exchange_population,
+            features=['fetchOHLCV'],
+        )  # Type: list[Exchange]
+        for exchange in exchanges:
+            exchange.init()
+
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                frequencies = exchange.get_candle_frequencies()
+                freq = random.sample(frequencies, 1)[0]
+
+                bar_count = random.randint(1, 10)
+                end_dt = pd.Timestamp.utcnow().floor('1T')
+                dt_range = pd.date_range(
+                    end=end_dt, periods=bar_count, freq=freq
+                )
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+
+                candles = exchange.get_candles(
+                    freq=freq,
+                    assets=assets,
+                    bar_count=bar_count,
+                    start_dt=dt_range[0],
+                    end_dt=dt_range[-1],
+                )
+
+                assert len(candles) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
+        pass
+
+    def test_orders(self):
+        population = 3
+        quote_currency = 'eth'
+        order_amount = 0.1
+
+        exchanges = select_random_exchanges(
+            population=population,
+            features=['fetchOrder'],
+            is_authenticated=True,
+            base_currency=quote_currency,
+        )  # Type: list[Exchange]
+
+        for exchange in exchanges:
+            exchange.init()
+
+            assets = exchange.get_assets(quote_currency=quote_currency)
+            asset = select_random_assets(assets, 1)[0]
+            assert asset
+
+            tickers = exchange.tickers([asset])
+            price = tickers[asset]['last_price']
+
+            amount = order_amount / price
+
+            limit_price = price * 0.8
+            style = ExchangeLimitOrder(limit_price=limit_price)
+
+            order = exchange.order(
+                asset=asset,
+                amount=amount,
+                style=style,
+            )
+            sleep(1)
+
+            open_order, _ = exchange.get_order(order.id, asset)
+            assert open_order.status == 0
+
+            exchange.cancel_order(open_order, asset)
+            sleep(1)
+
+            canceled_order, _ = exchange.get_order(open_order.id, asset)
+            assert canceled_order.status == 2
+        pass