Revert "Allow jupyter to run as root inside Docker image"

Allow jupyter to run as root inside Docker image

.gitignore

@@ -40,6 +40,7 @@ develop-eggs
 coverage.xml
 htmlcov
 nosetests.xml
+.python-version
 
 # C Extensions
 *.o

Dockerfile

@@ -11,13 +11,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

Dockerfile-dev

@@ -5,7 +5,7 @@
 #
 # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile
+#    docker build -t quantopian/catalyst -f Dockerfile .
 #
 # To run the container:
 #
@@ -15,13 +15,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

catalyst/__main__.py

@@ -3,13 +3,14 @@ import os
 from functools import wraps
 
 import click
+import sys
 import logbook
 import pandas as pd
 from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_utils import delete_algo_folder
+from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -257,7 +258,7 @@ def run(ctx,
     if capital_base is None:
         ctx.fail("must specify a capital base with '--capital-base'")
 
-    click.echo('Running in backtesting mode.')
+    click.echo('Running in backtesting mode.', sys.stdout)
 
     perf = _run(
         initialize=None,
@@ -282,13 +283,15 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
+        analyze_live=None,
         live_graph=False,
         simulate_orders=True,
+        auth_aliases=None,
         stats_output=None,
     )
 
     if output == '-':
-        click.echo(str(perf))
+        click.echo(str(perf), sys.stdout)
     elif output != os.devnull:  # make the catalyst magic not write any data
         perf.to_pickle(output)
 
@@ -312,11 +315,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                # these options are set when running in line magic mode
-                # set a non None algo text to use the ipython user_ns
-                '--algotext', '',
-                '--local-namespace',
-            ] if cell is None else []) + line.split(),
+                     # these options are set when running in line magic mode
+                     # set a non None algo text to use the ipython user_ns
+                     '--algotext', '',
+                     '--local-namespace',
+                 ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,
@@ -393,6 +396,12 @@ def catalyst_magic(line, cell=None):
     help='The base currency used to calculate statistics '
          '(e.g. usd, btc, eth).',
 )
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
 @click.option(
     '--live-graph/--no-live-graph',
     is_flag=True,
@@ -406,6 +415,15 @@ def catalyst_magic(line, cell=None):
     help='Simulating orders enable the paper trading mode. No orders will be '
          'sent to the exchange unless set to false.',
 )
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
 @click.pass_context
 def live(ctx,
          algofile,
@@ -418,7 +436,9 @@ def live(ctx,
          exchange_name,
          algo_namespace,
          base_currency,
+         end,
          live_graph,
+         auth_aliases,
          simulate_orders):
     """Trade live with the given algorithm.
     """
@@ -441,10 +461,10 @@ def live(ctx,
         ctx.fail("must specify a capital base with '--capital-base'")
 
     if simulate_orders:
-        click.echo('Running in paper trading mode.')
+        click.echo('Running in paper trading mode.', sys.stdout)
 
     else:
-        click.echo('Running in live trading mode.')
+        click.echo('Running in live trading mode.', sys.stdout)
 
     perf = _run(
         initialize=None,
@@ -460,7 +480,7 @@ def live(ctx,
         bundle=None,
         bundle_timestamp=None,
         start=None,
-        end=None,
+        end=end,
         output=output,
         print_algo=print_algo,
         local_namespace=local_namespace,
@@ -470,12 +490,14 @@ def live(ctx,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=None,
         simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
         stats_output=None,
     )
 
     if output == '-':
-        click.echo(str(perf))
+        click.echo(str(perf), sys.stdout)
     elif output != os.devnull:  # make the catalyst magic not write any data
         perf.to_pickle(output)
 
@@ -557,7 +579,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,
@@ -580,10 +602,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 @click.pass_context
 def clean_algo(ctx, algo_namespace):
     click.echo(
-        'Cleaning algo state: {}'.format(algo_namespace)
+        'Cleaning algo state: {}'.format(algo_namespace),
+        sys.stdout
     )
     delete_algo_folder(algo_namespace)
-    click.echo('Done')
+    click.echo('Done', sys.stdout)
 
 
 @main.command(name='clean-exchange')
@@ -610,11 +633,11 @@ def clean_exchange(ctx, exchange_name, data_frequency):
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout)
     exchange_bundle.clean(
         data_frequency=data_frequency,
     )
-    click.echo('Done')
+    click.echo('Done', sys.stdout)
 
 
 @main.command()
@@ -735,7 +758,7 @@ def bundles():
         # because there were no entries, print a single message indicating that
         # no ingestions have yet been made.
         for timestamp in ingestions or ["<no ingestions>"]:
-            click.echo("%s %s" % (bundle, timestamp))
+            click.echo("%s %s" % (bundle, timestamp), sys.stdout)
 
 
 if __name__ == '__main__':

catalyst/assets/_assets.pyx

@@ -17,6 +17,7 @@
 """
 Cythonized Asset object.
 """
+
 import hashlib
 
 cimport cython
@@ -38,7 +39,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
-from catalyst.exchange.exchange_utils import get_sid
+from catalyst.exchange.utils.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 

catalyst/curate/poloniex.py

@@ -1,16 +1,16 @@
-import os
-import time
-import shutil
-import json
 import csv
+import json
+import os
+import shutil
+import time
 from datetime import datetime
 
+import logbook
 import pandas as pd
 import requests
-import logbook
-
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
+from catalyst.exchange.utils.exchange_utils import \
+    get_exchange_symbols_filename
 
 DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
 DT_END = pd.to_datetime('today').value // 10 ** 9
@@ -193,7 +193,8 @@ class PoloniexCurator(object):
                    for this currencyPair
         '''
         try:
-            if('end_file' in locals() and end_file + 3600 < end):
+            if(temp is not None
+                    or ('end_file' in locals() and end_file + 3600 < end)):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
@@ -261,7 +262,7 @@ class PoloniexCurator(object):
         vol = df['total'].to_frame('volume')           # set Vol aside
         df.drop('total', axis=1, inplace=True)         # Drop volume data
         ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
-        ohlc.cols = ohlc.cols.map(lambda t: t[1])      # Raname cols
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Rename cols
         closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
         ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
         vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin

catalyst/data/dispatch_bar_reader.py

@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         if self._last_available_dt is not None:
             return self._last_available_dt
         else:
-            return min(r.last_available_dt for r in self._readers.values())
+            return min(r.last_available_dt for r in list(self._readers.values()))
 
     @lazyval
     def first_trading_day(self):
-        return max(r.first_trading_day for r in self._readers.values())
+        return max(r.first_trading_day for r in list(self._readers.values()))
 
     def get_value(self, sid, dt, field):
         asset = self._asset_finder.retrieve_asset(sid)

catalyst/data/loader.py

@@ -22,6 +22,7 @@ from pandas_datareader.data import DataReader
 from six import iteritems
 from six.moves.urllib_error import HTTPError
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.utils.calendars import get_calendar
 from . import treasuries, treasuries_can
 from .benchmarks import get_benchmark_returns
@@ -31,8 +32,6 @@ from ..utils.paths import (
     data_root,
 )
 
-from catalyst.constants import LOG_LEVEL
-
 logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
@@ -102,7 +101,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         trading_day = get_calendar('OPEN').trading_day
 
     # TODO: consider making configurable
-    bm_symbol = 'btc_usdt'
+    bm_symbol = 'btc_usd'
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
@@ -143,10 +142,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.factory import get_exchange
+        from catalyst.exchange.utils.factory import get_exchange
         exchange = get_exchange(
-            exchange_name='poloniex', base_currency='usdt'
+            exchange_name='bitfinex', base_currency='usd'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 

catalyst/examples/arbitrage_with_interface.py

@@ -6,7 +6,7 @@ from catalyst.api import (
     symbol,
     get_open_orders
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'arbitrage_eth_btc'

catalyst/examples/buy_and_hodl.py

@@ -23,7 +23,7 @@ from catalyst.api import (order_target_value, symbol, record,
 
 
 def initialize(context):
-    context.ASSET_NAME = 'btc_usd'
+    context.ASSET_NAME = 'btc_usdt'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
@@ -140,9 +140,9 @@ if __name__ == '__main__':
         initialize=initialize,
         handle_data=handle_data,
         analyze=analyze,
-        exchange_name='bitfinex',
+        exchange_name='poloniex',
         algo_namespace='buy_and_hodl',
-        base_currency='usd',
+        base_currency='usdt',
         start=pd.to_datetime('2015-03-01', utc=True),
         end=pd.to_datetime('2017-10-31', utc=True),
     )

catalyst/examples/buy_btc_simple.py

@@ -27,7 +27,7 @@ import pandas as pd
 
 
 def initialize(context):
-    context.asset = symbol('btc_usd')
+    context.asset = symbol('btc_usdt')
 
 
 def handle_data(context, data):
@@ -41,9 +41,9 @@ if __name__ == '__main__':
         data_frequency='daily',
         initialize=initialize,
         handle_data=handle_data,
-        exchange_name='bitfinex',
+        exchange_name='poloniex',
         algo_namespace='buy_and_hodl',
-        base_currency='usd',
+        base_currency='usdt',
         start=pd.to_datetime('2015-03-01', utc=True),
         end=pd.to_datetime('2017-10-31', utc=True),
     )

catalyst/examples/buy_low_sell_high.py

@@ -1,19 +1,7 @@
-'''
-This algorithm requires an additional library (ta-lib) beyond those
-required by catalyst. Install it first by running:
-$ pip install TA-Lib
-
-If you get build errors like:
-    "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and it
-needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
-instructions on how to install the required dependencies.
-'''
-
 import talib
+import pandas as pd
 from logbook import Logger
 
-from catalyst import run_algorithm
 from catalyst.api import (
     order,
     order_target_percent,
@@ -21,59 +9,52 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
-import pandas as pd
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+from catalyst.utils.run_algo import run_algorithm
 
-algo_namespace = 'buy_low_sell_high_xrp'
-log = Logger(algo_namespace)
+algo_namespace = 'buy_the_dip_live'
+log = Logger('buy low sell high')
 
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USDT'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 5000
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.05
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.swallow_errors = True
+    context.SLIPPAGE_ALLOWED = 0.02
 
     context.errors = []
     pass
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1D'
     )
-
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
     elif rsi <= 70:
-        buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -141,11 +122,11 @@ def _handle_data(context, data):
 
 def handle_data(context, data):
     log.info('handling bar {}'.format(data.current_dt))
-    try:
-        _handle_data(context, data)
-    except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
+    # try:
+    _handle_data(context, data)
+    # except Exception as e:
+    #     log.warn('aborting the bar on error {}'.format(e))
+    #     context.errors.append(e)
 
     log.info('completed bar {}, total execution errors {}'.format(
         data.current_dt,
@@ -162,15 +143,29 @@ def analyze(context, stats):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        algo_namespace='buy_and_hodl',
-        base_currency='usd',
-        start=pd.to_datetime('2015-03-01', utc=True),
-        end=pd.to_datetime('2017-10-31', utc=True),
-    )
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.001,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='binance',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='buy_and_hodl',
+            base_currency='usdt',
+            start=pd.to_datetime('2015-03-01', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )

catalyst/examples/buy_low_sell_high_live.py

@@ -1,160 +0,0 @@
-import talib
-from logbook import Logger
-
-import pandas as pd
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_the_dip_live'
-log = Logger('buy low sell high')
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.ASSET_NAME = 'btc_usdt'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.TARGET_POSITIONS = 30
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='1D'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.2
-    else:
-        buy_increment = 0.1
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(
-        price=price,
-        rsi=rsi,
-    )
-
-    orders = get_open_orders(context.asset)
-    if orders:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif (position.amount > 0
-              and price > cost_basis * (1 + context.PROFIT_TARGET)):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            log.info('the rsi is too high to consider buying {}'.format(rsi))
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
-        )
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-    pass
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=0.001,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='binance',
-        live=True,
-        algo_namespace=algo_namespace,
-        base_currency='btc',
-        simulate_orders=True,
-    )

catalyst/examples/dual_moving_average.py

@@ -1,12 +1,12 @@
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from logbook import Logger
-import matplotlib.pyplot as plt
 
 from catalyst import run_algorithm
 from catalyst.api import (record, symbol, order_target_percent,
                           get_open_orders)
-from catalyst.exchange.stats_utils import extract_transactions
+from catalyst.exchange.utils.stats_utils import extract_transactions
 
 NAMESPACE = 'dual_moving_average'
 log = Logger(NAMESPACE)
@@ -84,7 +84,8 @@ def handle_data(context, data):
 def analyze(context, perf):
 
     # Get the base_currency that was passed as a parameter to the simulation
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    exchange = list(context.exchanges.values())[0]
+    base_currency = exchange.base_currency.upper()
 
     # First chart: Plot portfolio value using base_currency
     ax1 = plt.subplot(411)

catalyst/examples/mean_reversion_simple.py

@@ -12,8 +12,7 @@ from logbook import Logger
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.stats_utils import extract_transactions
-
+from catalyst.exchange.utils.stats_utils import extract_transactions
 # We give a name to the algorithm which Catalyst will use to persist its state.
 # In this example, Catalyst will create the `.catalyst/data/live_algos`
 # directory. If we stop and start the algorithm, Catalyst will resume its
@@ -34,18 +33,18 @@ def initialize(context):
     # parameters or values you're going to use.
 
     # In our example, we're looking at Neo in Ether.
-    context.market = symbol('neo_eth')
+    context.market = symbol('eth_btc')
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 30
-    context.RSI_OVERBOUGHT = 80
-    context.CANDLE_SIZE = '5T'
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '15T'
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.1, taker=0.2)
-    context.set_slippage(spread=0.0001)
+    context.set_commission(maker=0.001, taker=0.002)
+    context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -115,7 +114,7 @@ def handle_data(context, data):
     # TODO: retest with open orders
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.market)
+    orders = context.blotter.open_orders
     if len(orders) > 0:
         log.info('exiting because orders are open: {}'.format(orders))
         return
@@ -162,7 +161,7 @@ def analyze(context=None, perf=None):
 
     import matplotlib.pyplot as plt
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -245,9 +244,25 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    MODE = 'backtest'
+    live = True
 
-    if MODE == 'backtest':
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+            # auth_aliases=dict(poloniex='auth2')
+        )
+
+    else:
         folder = os.path.join(
             tempfile.gettempdir(), 'catalyst', NAMESPACE
         )
@@ -266,24 +281,9 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out
         )
         log.info('saved perf stats: {}'.format(out))
-
-    elif MODE == 'live':
-        run_algorithm(
-            capital_base=0.05,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='binance',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            live_graph=False,
-            simulate_orders=True,
-            stats_output=None
-        )

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/examples/rsi_profit_target.py

@@ -175,7 +175,7 @@ def handle_data(context, data):
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
 
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results.loc[:, 'portfolio_value'].plot(ax=ax1)

catalyst/examples/simple_loop.py

@@ -1,23 +1,26 @@
-import talib
 import pandas as pd
+import talib
+from logbook import Logger, INFO
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record
-from catalyst.exchange.stats_utils import get_pretty_stats, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
     extract_transactions
 
+log = Logger('simple_loop', level=INFO)
+
 
 def initialize(context):
-    print('initializing')
+    log.info('initializing')
     context.asset = symbol('eth_btc')
     context.base_price = None
 
 
 def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
+    log.info('handling bar: {}'.format(data.current_dt))
 
     price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
+    log.info('got price {price}'.format(price=price))
 
     prices = data.history(
         context.asset,
@@ -26,10 +29,10 @@ def handle_data(context, data):
         frequency='30T'
     )
     last_traded = prices.index[-1]
-    print('last candle date: {}'.format(last_traded))
+    log.info('last candle date: {}'.format(last_traded))
 
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    print('got rsi: {}'.format(rsi))
+    log.info('got rsi: {}'.format(rsi))
 
     # If base_price is not set, we use the current value. This is the
     # price at the first bar which we reference to calculate price_change.
@@ -51,10 +54,10 @@ def handle_data(context, data):
 
 def analyze(context, perf):
     import matplotlib.pyplot as plt
-    print('the stats: {}'.format(get_pretty_stats(perf)))
+    log.info('the stats: {}'.format(get_pretty_stats(perf)))
 
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -111,15 +114,31 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=1,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=None,
-        exchange_name='poloniex',
-        live=True,
-        algo_namespace='simple_loop',
-        base_currency='eth',
-        live_graph=False,
-        simulate_orders=True
-    )
+    mode = 'backtest'
+
+    if mode == 'backtest':
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='poloniex',
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            data_frequency='minute',
+            start=pd.to_datetime('2017-9-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='binance',
+            live=True,
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True
+        )

catalyst/examples/simple_universe.py

@@ -35,14 +35,14 @@ import numpy as np
 import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.api import (symbols, )
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
 
 
 def initialize(context):
     context.i = -1  # minute counter
-    context.exchange = context.exchanges.values()[0].name.lower()
-    context.base_currency = context.exchanges.values()[0].base_currency.lower()
+    context.exchange = list(context.exchanges.values())[0].name.lower()
+    context.base_currency = list(context.exchanges.values())[0].base_currency.lower()
 
 
 def handle_data(context, data):
@@ -65,7 +65,7 @@ def handle_data(context, data):
     minutes = 30
 
     # get lookback_days of history data: that is 'lookback' number of bins
-    lookback = one_day_in_minutes / minutes * lookback_days
+    lookback = int(one_day_in_minutes / minutes * lookback_days)
     if not context.i % minutes and context.universe:
         # we iterate for every pair in the current universe
         for coin in context.coins:

catalyst/examples/talib_simple.py

@@ -23,7 +23,7 @@ from catalyst.api import (
     order_target_percent,
     symbol,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 
 algo_namespace = 'talib_sample'
 log = Logger(algo_namespace)

catalyst/exchange/asset_finder_exchange.py

@@ -1,99 +0,0 @@
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-
-log = Logger('AssetFinderExchange', level=LOG_LEVEL)
-
-
-class AssetFinderExchange(object):
-    def __init__(self):
-        self._asset_cache = {}
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        return list()
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        # for sid in sids:
-        #     if sid in self._asset_cache:
-        #         log.debug('got asset from cache: {}'.format(sid))
-        #     else:
-        #         log.debug('fetching asset: {}'.format(sid))
-        return list()
-
-    def lookup_symbol(self, symbol, exchange, data_frequency=None,
-                      as_of_date=None, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
-
-        if data_frequency is not None:
-            key = ','.join([exchange.name, symbol, data_frequency])
-
-        else:
-            key = ','.join([exchange.name, symbol])
-
-        if key in self._asset_cache:
-            return self._asset_cache[key]
-        else:
-            asset = exchange.get_asset(symbol, data_frequency)
-            self._asset_cache[key] = asset
-            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,709 +0,0 @@
-import base64
-import datetime
-import hashlib
-import hmac
-import json
-import re
-import time
-
-import numpy as np
-import pandas as pd
-import pytz
-import requests
-import six
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
-
-# Trying to account for REST api instability
-# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-from catalyst.utils.deprecate import deprecated
-
-requests.adapters.DEFAULT_RETRIES = 20
-
-BITFINEX_URL = 'https://api.bitfinex.com'
-
-log = Logger('Bitfinex', level=LOG_LEVEL)
-warning_logger = Logger('AlgoWarning')
-
-
-@deprecated
-class Bitfinex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.url = BITFINEX_URL
-        self.key = key
-        self.secret = secret.encode('UTF-8')
-        self.name = 'bitfinex'
-        self.color = 'green'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-
-        # The candle limit for each request
-        self.num_candles_limit = 1000
-
-        # Max is 90 but playing it safe
-        # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 80
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def _request(self, operation, data, version='v1'):
-        payload_object = {
-            'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 1000000),
-            # convert to string
-            'options': {}
-        }
-
-        if data is None:
-            payload_dict = payload_object
-        else:
-            payload_dict = payload_object.copy()
-            payload_dict.update(data)
-
-        payload_json = json.dumps(payload_dict)
-        if six.PY3:
-            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
-        else:
-            payload = base64.b64encode(payload_json)
-
-        m = hmac.new(self.secret, payload, hashlib.sha384)
-        m = m.hexdigest()
-
-        # headers
-        headers = {
-            'X-BFX-APIKEY': self.key,
-            'X-BFX-PAYLOAD': payload,
-            'X-BFX-SIGNATURE': m
-        }
-
-        if data is None:
-            request = requests.get(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ), data={},
-                headers=headers)
-        else:
-            request = requests.post(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ),
-                headers=headers)
-
-        return request
-
-    def _get_v2_symbol(self, asset):
-        pair = asset.symbol.split('_')
-        symbol = 't' + pair[0].upper() + pair[1].upper()
-        return symbol
-
-    def _get_v2_symbols(self, assets):
-        """
-        Workaround to support Bitfinex v2
-        TODO: Might require a separate asset dictionary
-
-        :param assets:
-        :return:
-        """
-
-        v2_symbols = []
-        for asset in assets:
-            v2_symbols.append(self._get_v2_symbol(asset))
-
-        return v2_symbols
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from a Bitfinex order dictionary
-        :param order_status:
-        :return: Order
-        """
-        if order_status['is_cancelled']:
-            status = ORDER_STATUS.CANCELLED
-        elif not order_status['is_live']:
-            log.info('found executed order {}'.format(order_status))
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['original_amount'])
-        filled = float(order_status['executed_amount'])
-
-        if order_status['side'] == 'sell':
-            amount = -amount
-            filled = -filled
-
-        price = float(order_status['price'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        if order_type.endswith('limit'):
-            limit_price = price
-        elif order_type.endswith('stop'):
-            stop_price = price
-
-        executed_price = float(order_status['avg_execution_price'])
-
-        # TODO: bitfinex does not specify comission.
-        # I could calculate it but not sure if it's worth it.
-        commission = None
-
-        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        date = pytz.utc.localize(date)
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['id']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        log.debug('retrieving wallets balances')
-        try:
-            self.ask_request()
-            response = self._request('balances', None)
-            balances = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['message'])
-            )
-
-        std_balances = dict()
-        for balance in balances:
-            currency = balance['currency'].lower()
-            std_balances[currency] = float(balance['available'])
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Bitfinex
-
-        :param data_frequency:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
-         '1M'
-        """
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
-                               '360T', '720T', '1D', '7D', '14D', '30D']
-        if freq not in allowed_frequencies:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
-        if freq_match:
-            number = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-            if unit == 'T':
-                if number in [60, 180, 360, 720]:
-                    number = number / 60
-                    converted_unit = 'h'
-                else:
-                    converted_unit = 'm'
-            else:
-                converted_unit = unit
-
-            frequency = '{}{}'.format(number, converted_unit)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-        for asset in asset_list:
-            symbol = self._get_v2_symbol(asset)
-            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
-                url=self.url,
-                frequency=frequency,
-                symbol=symbol
-            )
-
-            if bar_count:
-                is_list = True
-                url += '/hist?limit={}'.format(int(bar_count))
-
-                def get_ms(date):
-                    epoch = datetime.datetime.utcfromtimestamp(0)
-                    epoch = epoch.replace(tzinfo=pytz.UTC)
-
-                    return (date - epoch).total_seconds() * 1000.0
-
-                if start_dt is not None:
-                    start_ms = get_ms(start_dt)
-                    url += '&start={0:f}'.format(start_ms)
-
-                if end_dt is not None:
-                    end_ms = get_ms(end_dt)
-                    url += '&end={0:f}'.format(end_ms)
-
-            else:
-                is_list = False
-                url += '/last'
-
-            try:
-                self.ask_request()
-                response = requests.get(url)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response.content:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            candles = response.json()
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(
-                    candle[0] / 1000.0)
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-                ohlc = dict(
-                    open=np.float64(candle[1]),
-                    high=np.float64(candle[3]),
-                    low=np.float64(candle[4]),
-                    close=np.float64(candle[2]),
-                    volume=np.float64(candle[5]),
-                    price=np.float64(candle[2]),
-                    last_traded=last_traded
-                )
-                return ohlc
-
-            if is_list:
-                ohlc_bars = []
-                # We can to list candles from old to new
-                for candle in reversed(candles):
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-            else:
-                ohlc = ohlc_from_candle(candles)
-                ohlc_map[asset] = ohlc
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-        if isinstance(style, ExchangeLimitOrder) \
-                or isinstance(style, ExchangeStopLimitOrder):
-            price = style.get_limit_price(is_buy)
-            order_type = 'limit'
-
-        elif isinstance(style, ExchangeStopOrder):
-            price = style.get_stop_price(is_buy)
-            order_type = 'stop'
-
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-        req = dict(
-            symbol=exchange_symbol,
-            amount=str(float(abs(amount))),
-            price="{:.20f}".format(float(price)),
-            side='buy' if is_buy else 'sell',
-            type='exchange ' + order_type,  # TODO: support margin trades
-            exchange=self.name,
-            is_hidden=False,
-            is_postonly=False,
-            use_all_available=0,
-            ocoorder=False,
-            buy_price_oco=0,
-            sell_price_oco=0
-        )
-
-        date = pd.Timestamp.utcnow()
-        try:
-            self.ask_request()
-            response = self._request('order/new', req)
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='unable to create Bitfinex order {}'.format(
-                    order_status['message'])
-            )
-
-        order_id = str(order_status['id'])
-        order = Order(
-            dt=date,
-            asset=asset,
-            amount=amount,
-            stop=style.get_stop_price(is_buy),
-            limit=style.get_limit_price(is_buy),
-            id=order_id
-        )
-
-        return order
-
-    def get_open_orders(self, asset=None):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        try:
-            self.ask_request()
-            response = self._request('orders', None)
-            order_statuses = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_statuses:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        orders = []
-        for order_status in order_statuses:
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-        try:
-            self.ask_request()
-            response = self._request(
-                'order/status', {'order_id': int(order_id)})
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='Unable to retrieve order status: {}'.format(
-                    order_status['message'])
-            )
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-
-        try:
-            self.ask_request()
-            response = self._request('order/cancel', {'order_id': order_id})
-            status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self._get_v2_symbols(assets)
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            self.ask_request()
-            response = requests.get(
-                '{url}/v2/tickers?symbols={symbols}'.format(
-                    url=self.url,
-                    symbols=','.join(symbols),
-                )
-            )
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response.content:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response.content)
-            )
-
-        try:
-            tickers = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        ticks = dict()
-        for index, ticker in enumerate(tickers):
-            if not len(ticker) == 11:
-                raise ExchangeRequestError(
-                    error='Invalid ticker in response: {}'.format(ticker)
-                )
-
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker[1],
-                ask=ticker[3],
-                last_price=ticker[7],
-                low=ticker[10],
-                high=ticker[9],
-                volume=ticker[8],
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self._request('symbols', None)
-
-        for symbol in response.json():
-            if (source_dates):
-                start_date = self.get_symbol_start_date(symbol)
-            else:
-                try:
-                    start_date = cached_symbols[symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[symbol] = dict(
-                symbol=symbol[:-3] + '_' + symbol[-3:],
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-
-        print(symbol)
-        symbol_v2 = 't' + symbol.upper()
-
-        """
-            For each symbol we retrieve candles with Monhtly resolution
-            We get the first month, and query again with daily resolution
-            around that date, and we get the first date
-        """
-        url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
-            url=self.url,
-            symbol=symbol_v2
-        )
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        """
-            If we don't get any data back for our monthly-resolution query
-            it means that symbol started trading less than a month ago, so
-            arbitrarily set the ref. date to 15 days ago to be safe with
-            +/- 31 days
-        """
-        if (len(response.json())):
-            startmonth = response.json()[-1][0]
-        else:
-            startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
-
-        """
-            Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than
-            now: time.time()
-        """
-        url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
-               '&end={end}').format(
-            url=self.url,
-            symbol=symbol_v2,
-            start=startmonth - 3600 * 24 * 31 * 1000,
-            end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000)))
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d',
-                             time.gmtime(int(response.json()[-1][0] / 1000)))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        exchange_symbol = asset.exchange_symbol
-        try:
-            self.ask_request()
-            # TODO: implement limit
-            response = self._request(
-                'book/{}'.format(exchange_symbol), None)
-            data = response.json()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        # TODO: filter by type
-        result = dict()
-        for order_type in data:
-            result[order_type] = []
-
-            for entry in data[order_type]:
-                result[order_type].append(dict(
-                    rate=float(entry['price']),
-                    quantity=float(entry['amount'])
-                ))
-
-        return result

catalyst/exchange/bitfinex/symbols.json

@@ -1,127 +0,0 @@
-{
-  "neobtc": {
-    "symbol": "neo_btc",
-    "start_date": "2017-09-07",
-    "precision": 5
-  },
-  "neousd": {
-    "symbol": "neo_usd",
-    "start_date": "2017-09-07"
-  },
-  "neoeth": {
-    "symbol": "neo_eth",
-    "start_date": "2017-09-07"
-  },
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bchusd": {
-    "symbol": "bch_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2017-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcbtc": {
-    "symbol": "etc_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcusd": {
-    "symbol": "etc_usd",
-    "start_date": "2017-01-01"
-  },
-  "rrtusd": {
-    "symbol": "rrt_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtbtc": {
-    "symbol": "rrt_btc",
-    "start_date": "2010-01-01"
-  },
-  "zecusd": {
-    "symbol": "zec_usd",
-    "start_date": "2010-01-01"
-  },
-  "zecbtc": {
-    "symbol": "zec_btc",
-    "start_date": "2010-01-01"
-  },
-  "xmrusd": {
-    "symbol": "xmr_usd",
-    "start_date": "2010-01-01"
-  },
-  "xmrbtc": {
-    "symbol": "xmr_btc",
-    "start_date": "2010-01-01"
-  },
-  "dshusd": {
-    "symbol": "dsh_usd",
-    "start_date": "2010-01-01"
-  },
-  "dshbtc": {
-    "symbol": "dsh_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccbtc": {
-    "symbol": "bcc_btc",
-    "start_date": "2010-01-01"
-  },
-  "bcubtc": {
-    "symbol": "bcu_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccusd": {
-    "symbol": "bcc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bcuusd": {
-    "symbol": "bcu_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpusd": {
-    "symbol": "xrp_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpbtc": {
-    "symbol": "xrp_btc",
-    "start_date": "2010-01-01"
-  },
-  "iotusd": {
-    "symbol": "iot_usd",
-    "start_date": "2010-01-01"
-  },
-  "iotbtc": {
-    "symbol": "iot_btc",
-    "start_date": "2010-01-01"
-  },
-  "ioteth": {
-    "symbol": "iot_eth",
-    "start_date": "2010-01-01"
-  },
-  "eosusd": {
-    "symbol": "eos_usd",
-    "start_date": "2010-01-01"
-  },
-  "eosbtc": {
-    "symbol": "eos_btc",
-    "start_date": "2010-01-01"
-  },
-  "eoseth": {
-    "symbol": "eos_eth",
-    "start_date": "2010-01-01"
-  }
-}

catalyst/exchange/bittrex/bittrex.py

@@ -1,417 +0,0 @@
-import json
-import time
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from six.moves import urllib
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-    ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
-    CreateOrderError
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-
-# TODO: consider using this: https://github.com/mondeja/bittrex_v2
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Bittrex', level=LOG_LEVEL)
-
-URL2 = 'https://bittrex.com/Api/v2.0'
-
-
-@deprecated
-class Bittrex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret)
-        self.name = 'bittrex'
-        self.color = 'blue'
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-
-        self.num_candles_limit = 2000
-
-        # Not sure what the rate limit is but trying to play it safe
-        # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.minute_writer = None
-        self.minute_reader = None
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.bundle = ExchangeBundle(self.name)
-
-    @property
-    def account(self):
-        pass
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def get_balances(self):
-        balances = self.api.getbalances()
-        try:
-            log.debug('retrieving wallet balances')
-            self.ask_request()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        std_balances = dict()
-        try:
-            for balance in balances:
-                currency = balance['Currency'].lower()
-                std_balances[currency] = balance['Available']
-
-        except TypeError:
-            raise ExchangeRequestError(error=balances)
-
-        return std_balances
-
-    def create_order(self, asset, amount, is_buy, style):
-        log.info('creating {} order'.format('buy' if is_buy else 'sell'))
-        exchange_symbol = self.get_symbol(asset)
-
-        if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
-            if isinstance(style, StopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-            try:
-                self.ask_request()
-                if is_buy:
-                    order_status = self.api.buylimit(exchange_symbol, amount,
-                                                     price)
-                else:
-                    order_status = self.api.selllimit(exchange_symbol,
-                                                      abs(amount), price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'uuid' in order_status:
-                order_id = order_status['uuid']
-                order = Order(
-                    dt=pd.Timestamp.utcnow(),
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                if order_status == 'INSUFFICIENT_FUNDS':
-                    log.warn('not enough funds to create order')
-                    return None
-                elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
-                    log.warn('Your order is too small, order at least 50K'
-                             ' Satoshi')
-                    return None
-                else:
-                    raise CreateOrderError(
-                        exchange=self.name,
-                        error=order_status
-                    )
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset):
-        symbol = self.get_symbol(asset)
-        try:
-            self.ask_request()
-            open_orders = self.api.getopenorders(symbol)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        orders = list()
-        for order_status in open_orders:
-            order = self._create_order(order_status)
-            orders.append(order)
-
-        return orders
-
-    def _create_order(self, order_status):
-        log.info(
-            'creating catalyst order from Bittrex {}'.format(order_status))
-        if order_status['CancelInitiated']:
-            status = ORDER_STATUS.CANCELLED
-        elif order_status['Closed'] is not None:
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        date = pd.to_datetime(order_status['Opened'], utc=True)
-        amount = order_status['Quantity']
-        filled = amount - order_status['QuantityRemaining']
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['Exchange']],
-            amount=amount,
-            stop=None,  # Not yet supported by Bittrex
-            limit=order_status['Limit'],
-            filled=filled,
-            id=order_status['OrderUuid'],
-            commission=order_status['CommissionPaid']
-        )
-        order.status = status
-
-        executed_price = order_status['PricePerUnit']
-
-        return order, executed_price
-
-    def get_order(self, order_id):
-        log.info('retrieving order {}'.format(order_id))
-        try:
-            self.ask_request()
-            order_status = self.api.getorder(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if order_status is None:
-            raise OrderNotFound(order_id=order_id, exchange=self.name)
-
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-        log.info('cancelling order {}'.format(order_id))
-
-        try:
-            self.ask_request()
-            status = self.api.cancel(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Supported Intervals
-        -------------------
-        day, oneMin, fiveMin, thirtyMin, hour
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :param start_dt
-        :param end_dt
-        :return:
-        """
-
-        # TODO: this has no effect at the moment
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T':
-            frequency = 'oneMin'
-        elif freq == '5T':
-            frequency = 'fiveMin'
-        elif freq == '30T':
-            frequency = 'thirtyMin'
-        elif freq == '60T':
-            frequency = 'hour'
-        elif freq == '1D':
-            frequency = 'day'
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        for asset in asset_list:
-            end = int(time.mktime(end_dt.timetuple()))
-            url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_={end}'.format(
-                    url=URL2,
-                    symbol=self.get_symbol(asset),
-                    frequency=frequency,
-                    end=end, )
-
-            try:
-                data = json.loads(urllib.request.urlopen(url).read().decode())
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if data['message']:
-                raise ExchangeRequestError(
-                    error='Unable to fetch candles {}'.format(data['message'])
-                )
-
-            candles = data['result']
-
-            def ohlc_from_candle(candle):
-                ohlc = dict(
-                    open=candle['O'],
-                    high=candle['H'],
-                    low=candle['L'],
-                    close=candle['C'],
-                    volume=candle['V'],
-                    price=candle['C'],
-                    last_traded=pd.to_datetime(candle['T'], utc=True)
-                )
-                return ohlc
-
-            ordered_candles = list(reversed(candles))
-            ohlc_map = dict()
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
-            else:
-                # TODO: optimize
-                ohlc_bars = []
-                for candle in ordered_candles[:bar_count]:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def tickers(self, assets):
-        """
-        As of v1.1, Bittrex only allows one ticker at the time.
-        So we have to make multiple calls to fetch multiple assets.
-
-        :param assets:
-        :return:
-        """
-        log.info('retrieving tickers')
-
-        ticks = dict()
-        for asset in assets:
-            symbol = self.get_symbol(asset)
-            try:
-                self.ask_request()
-                ticker = self.api.getticker(symbol)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            # TODO: catch invalid ticker
-            ticks[asset] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker['Bid'],
-                ask=ticker['Ask'],
-                last_price=ticker['Last']
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def generate_symbols_json(self, filename=None):
-        symbol_map = {}
-
-        fn, r = download_exchange_symbols(self.name)
-        with open(fn) as data_file:
-            cached_symbols = json.load(data_file)
-
-        markets = self.api.getmarkets()
-        for market in markets:
-            exchange_symbol = market['MarketName']
-            symbol = '{market}_{base}'.format(
-                market=self.sanitize_curency_symbol(market['MarketCurrency']),
-                base=self.sanitize_curency_symbol(market['BaseCurrency'])
-            )
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=pd.to_datetime(market['Created'],
-                                          utc=True).strftime("%Y-%m-%d"),
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        if order_type == 'all':
-            order_type = 'both'
-        elif order_type == 'bid':
-            order_type = 'buy'
-        elif order_type == 'ask':
-            order_type = 'sell'
-        else:
-            raise ValueError('invalid type')
-
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(
-            market=exchange_symbol,
-            type=order_type,
-            depth=100
-        )
-
-        result = dict()
-        for exchange_type in data:
-            if exchange_type == 'buy':
-                order_type = 'bids'
-            elif exchange_type == 'sell':
-                order_type = 'asks'
-
-            result[order_type] = []
-            for entry in data[exchange_type]:
-                result[order_type].append(dict(
-                    rate=entry['Rate'],
-                    quantity=entry['Quantity']
-                ))
-
-        return result

catalyst/exchange/bittrex/bittrex_api.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-from six.moves import urllib
-
-urlopen = urllib.request.urlopen
-
-
-class Bittrex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-        self.public = ['getmarkets', 'getcurrencies', 'getticker',
-                       'getmarketsummaries', 'getmarketsummary',
-                       'getorderbook', 'getmarkethistory']
-        self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
-                       'cancel', 'getopenorders']
-        self.account = ['getbalances', 'getbalance', 'getdepositaddress',
-                        'withdraw', 'getorder', 'getorderhistory',
-                        'getwithdrawalhistory', 'getdeposithistory']
-
-    def query(self, method, values={}):
-        if method in self.public:
-            url = 'https://bittrex.com/api/v1.1/public/'
-        elif method in self.market:
-            url = 'https://bittrex.com/api/v1.1/market/'
-        elif method in self.account:
-            url = 'https://bittrex.com/api/v1.1/account/'
-        else:
-            return 'Something went wrong, sorry.'
-
-        url += method + '?' + urllib.parse.urlencode(values)
-
-        if method not in self.public:
-            url += '&apikey=' + self.key
-            url += '&nonce=' + str(int(time.time()))
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 url.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'apisign': signature}
-        else:
-            headers = {}
-
-        req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(
-            req, context=ssl._create_unverified_context()).read())
-
-        if response["result"]:
-            return response["result"]
-        else:
-            return response["message"]
-
-    def getmarkets(self):
-        return self.query('getmarkets')
-
-    def getcurrencies(self):
-        return self.query('getcurrencies')
-
-    def getticker(self, market):
-        return self.query('getticker', {'market': market})
-
-    def getmarketsummaries(self):
-        return self.query('getmarketsummaries')
-
-    def getmarketsummary(self, market):
-        return self.query('getmarketsummary', {'market': market})
-
-    def getorderbook(self, market, type, depth=20):
-        return self.query('getorderbook',
-                          {'market': market, 'type': type, 'depth': depth})
-
-    def getmarkethistory(self, market, count=20):
-        return self.query('getmarkethistory',
-                          {'market': market, 'count': count})
-
-    def buylimit(self, market, quantity, rate):
-        return self.query('buylimit', {'market': market, 'quantity': quantity,
-                                       'rate': rate})
-
-    def buymarket(self, market, quantity):
-        return self.query('buymarket',
-                          {'market': market, 'quantity': quantity})
-
-    def selllimit(self, market, quantity, rate):
-        return self.query('selllimit', {'market': market, 'quantity': quantity,
-                                        'rate': rate})
-
-    def sellmarket(self, market, quantity):
-        return self.query('sellmarket',
-                          {'market': market, 'quantity': quantity})
-
-    def cancel(self, uuid):
-        return self.query('cancel', {'uuid': uuid})
-
-    def getopenorders(self, market):
-        return self.query('getopenorders', {'market': market})
-
-    def getbalances(self):
-        return self.query('getbalances')
-
-    def getbalance(self, currency):
-        return self.query('getbalance', {'currency': currency})
-
-    def getdepositaddress(self, currency):
-        return self.query('getdepositaddress', {'currency': currency})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'quantity': quantity,
-                           'address': address})
-
-    def getorder(self, uuid):
-        return self.query('getorder', {'uuid': uuid})
-
-    def getorderhistory(self, market, count):
-        return self.query('getorderhistory',
-                          {'market': market, 'count': count})
-
-    def getwithdrawalhistory(self, currency, count):
-        return self.query('getwithdrawalhistory',
-                          {'currency': currency, 'count': count})
-
-    def getdeposithistory(self, currency, count):
-        return self.query('getdeposithistory',
-                          {'currency': currency, 'count': count})

catalyst/exchange/bittrex/extensions-example.py

@@ -1,7 +0,0 @@
-from catalyst.data.bundles import register
-from catalyst.exchange.exchange_bundle import exchange_bundle
-
-symbols = (
-    'neo_btc',
-)
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -1,10 +1,13 @@
+import json
+import os
 import re
 from collections import defaultdict
 
 import ccxt
 import pandas as pd
 import six
-from ccxt import ExchangeNotAvailable, InvalidOrder
+from ccxt import InvalidOrder, NetworkError, \
+    ExchangeError
 from logbook import Logger
 from six import string_types
 
@@ -15,11 +18,17 @@ from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
-    ExchangeNotFoundError, CreateOrderError
+    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
+    UnsupportedHistoryFrequencyError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-from catalyst.exchange.exchange_utils import mixin_market_params, \
-    from_ms_timestamp, get_epoch
+from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
+    get_exchange_folder, get_catalyst_symbol, \
+    get_exchange_auth
+from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
+    get_epoch, \
+    get_periods_range
 from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.finance.transaction import Transaction
 
 log = Logger('CCXT', level=LOG_LEVEL)
 
@@ -52,32 +61,116 @@ class CCXT(Exchange):
                 'apiKey': key,
                 'secret': secret,
             })
+            self.api.enableRateLimit = True
 
         except Exception:
             raise ExchangeNotFoundError(exchange_name=exchange_name)
 
         self._symbol_maps = [None, None]
 
-        try:
-            markets_symbols = self.api.load_markets()
-            log.debug('the markets:\n{}'.format(markets_symbols))
-
-        except ExchangeNotAvailable as e:
-            raise ExchangeRequestError(error=e)
-
         self.name = exchange_name
 
-        self.markets = self.api.fetch_markets()
-        self.load_assets()
-
         self.base_currency = base_currency
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
         self.max_requests_per_minute = 60
+        self.low_balance_threshold = 0.1
         self.request_cpt = dict()
+        self._common_symbols = dict()
 
         self.bundle = ExchangeBundle(self.name)
+        self.markets = None
+        self._is_init = False
+
+    def init(self):
+        if self._is_init:
+            return
+
+        exchange_folder = get_exchange_folder(self.name)
+        filename = os.path.join(exchange_folder, 'cctx_markets.json')
+
+        if os.path.exists(filename):
+            timestamp = os.path.getmtime(filename)
+            dt = pd.to_datetime(timestamp, unit='s', utc=True)
+
+            if dt >= pd.Timestamp.utcnow().floor('1D'):
+                with open(filename) as f:
+                    self.markets = json.load(f)
+
+                log.debug('loaded markets for {}'.format(self.name))
+
+        if self.markets is None:
+            try:
+                markets_symbols = self.api.load_markets()
+                log.debug(
+                    'fetching {} markets:\n{}'.format(
+                        self.name, markets_symbols
+                    )
+                )
+
+                self.markets = self.api.fetch_markets()
+                with open(filename, 'w+') as f:
+                    json.dump(self.markets, f, indent=4)
+
+            except (ExchangeError, NetworkError) as e:
+                log.warn(
+                    'unable to fetch markets {}: {}'.format(
+                        self.name, e
+                    )
+                )
+                raise ExchangeRequestError(error=e)
+
+        self.load_assets()
+        self._is_init = True
+
+    @staticmethod
+    def find_exchanges(features=None, is_authenticated=False):
+        ccxt_features = []
+        if features is not None:
+            for feature in features:
+                if not feature.endswith('Bundle'):
+                    ccxt_features.append(feature)
+
+        exchange_names = []
+        for exchange_name in ccxt.exchanges:
+            if is_authenticated:
+                exchange_auth = get_exchange_auth(exchange_name)
+
+                has_auth = (exchange_auth['key'] != ''
+                            and exchange_auth['secret'] != '')
+
+                if not has_auth:
+                    continue
+
+            log.debug('loading exchange: {}'.format(exchange_name))
+            exchange = getattr(ccxt, exchange_name)()
+
+            if ccxt_features is None:
+                has_feature = True
+
+            else:
+                try:
+                    has_feature = all(
+                        [exchange.has[feature] for feature in ccxt_features]
+                    )
+
+                except Exception:
+                    has_feature = False
+
+            if has_feature:
+                try:
+                    log.info('initializing {}'.format(exchange_name))
+                    exchange_names.append(exchange_name)
+
+                except Exception as e:
+                    log.warn(
+                        'unable to initialize exchange {}: {}'.format(
+                            exchange_name, e
+                        )
+                    )
+
+        return exchange_names
 
     def account(self):
         return None
@@ -85,6 +178,30 @@ class CCXT(Exchange):
     def time_skew(self):
         return None
 
+    def get_candle_frequencies(self, data_frequency=None):
+        frequencies = []
+        try:
+            for timeframe in self.api.timeframes:
+                freq = CCXT.get_frequency(timeframe, raise_error=False)
+
+                # TODO: support all frequencies
+                if data_frequency == 'minute' and not freq.endswith('T'):
+                    continue
+
+                elif data_frequency == 'daily' and not freq.endswith('D'):
+                    continue
+
+                frequencies.append(freq)
+
+        except Exception as e:
+            log.warn(
+                'candle frequencies not available for exchange {}'.format(
+                    self.name
+                )
+            )
+
+        return frequencies
+
     def get_market(self, symbol):
         """
         The CCXT market.
@@ -106,48 +223,149 @@ class CCXT(Exchange):
         )
         return market
 
-    def get_symbol(self, asset_or_symbol):
+    def substitute_currency_code(self, currency, source='catalyst'):
+        if source == 'catalyst':
+            currency = currency.upper()
+
+            key = self.api.common_currency_code(currency)
+            self._common_symbols[key] = currency.lower()
+            return key
+
+        else:
+            if currency in self._common_symbols:
+                return self._common_symbols[currency]
+
+            else:
+                return currency.lower()
+
+    def get_symbol(self, asset_or_symbol, source='catalyst'):
         """
         The CCXT symbol.
 
         Parameters
         ----------
         asset_or_symbol
+        source
 
         Returns
         -------
 
         """
-        symbol = asset_or_symbol if isinstance(
-            asset_or_symbol, string_types
-        ) else asset_or_symbol.symbol
 
-        parts = symbol.split('_')
-        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+        if source == 'ccxt':
+            if isinstance(asset_or_symbol, string_types):
+                parts = asset_or_symbol.split('/')
+                base_currency = self.substitute_currency_code(
+                    parts[0], source
+                )
+                quote_currency = self.substitute_currency_code(
+                    parts[1], source
+                )
+                return '{}_{}'.format(base_currency, quote_currency)
 
-    def get_catalyst_symbol(self, market_or_symbol):
+            else:
+                return asset_or_symbol.symbol
+
+        else:
+            symbol = asset_or_symbol if isinstance(
+                asset_or_symbol, string_types
+            ) else asset_or_symbol.symbol
+
+            parts = symbol.split('_')
+            base_currency = self.substitute_currency_code(
+                parts[0], source
+            )
+            quote_currency = self.substitute_currency_code(
+                parts[1], source
+            )
+            return '{}/{}'.format(base_currency, quote_currency)
+
+    @staticmethod
+    def map_frequency(value, source='ccxt', raise_error=True):
         """
-        The Catalyst symbol.
+        Map a frequency value between CCXT and Catalyst
 
         Parameters
         ----------
-        market_or_symbol
+        value: str
+        source: str
+        raise_error: bool
 
         Returns
         -------
 
+        Notes
+        -----
+        The Pandas offset aliases supported by Catalyst:
+        Alias	Description
+        W	weekly frequency
+        M	month end frequency
+        D	calendar day frequency
+        H	hourly frequency
+        T, min	minutely frequency
+
+        The CCXT timeframes:
+        '1m': '1minute',
+        '1h': '1hour',
+        '1d': '1day',
+        '1w': '1week',
+        '1M': '1month',
+        '1y': '1year',
         """
-        if isinstance(market_or_symbol, string_types):
-            parts = market_or_symbol.split('/')
-            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+        match = re.match(
+            r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value, re.M | re.I
+        )
+        if match:
+            candle_size = int(match.group(1)) \
+                if match.group(1) else 1
+
+            unit = match.group(2)
 
         else:
-            return '{}_{}'.format(
-                market_or_symbol['base'].lower(),
-                market_or_symbol['quote'].lower(),
-            )
+            raise ValueError('Unable to parse frequency or timeframe')
 
-    def get_timeframe(self, freq):
+        if source == 'ccxt':
+            if unit == 'd':
+                result = '{}D'.format(candle_size)
+
+            elif unit == 'm':
+                result = '{}T'.format(candle_size)
+
+            elif unit == 'h':
+                result = '{}H'.format(candle_size)
+
+            elif unit == 'w':
+                result = '{}W'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryTimeframeError(timeframe=value)
+
+        else:
+            if unit == 'D':
+                result = '{}d'.format(candle_size)
+
+            elif unit == 'min' or unit == 'T':
+                result = '{}m'.format(candle_size)
+
+            elif unit == 'H':
+                result = '{}h'.format(candle_size)
+
+            elif unit == 'W':
+                result = '{}w'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryFrequencyError(frequency=value)
+
+        return result
+
+    @staticmethod
+    def get_timeframe(freq, raise_error=True):
         """
         The CCXT timeframe from the Catalyst frequency.
 
@@ -161,67 +379,98 @@ class CCXT(Exchange):
         str
 
         """
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) \
-                if freq_match.group(1) else 1
+        return CCXT.map_frequency(
+            freq, source='catalyst', raise_error=raise_error
+        )
 
-            unit = freq_match.group(2)
+    @staticmethod
+    def get_frequency(timeframe, raise_error=True):
+        """
+        Test Catalyst frequency from the CCXT timeframe
 
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
+        Catalyst uses the Pandas offset alias convention:
+        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
 
-        if unit.lower() == 'd':
-            timeframe = '{}d'.format(candle_size)
+        Parameters
+        ----------
+        timeframe
 
-        elif unit.lower() == 'm' or unit == 'T':
-            timeframe = '{}m'.format(candle_size)
+        Returns
+        -------
 
-        elif unit.lower() == 'h' or unit == 'T':
-            timeframe = '{}h'.format(candle_size)
+        """
+        return CCXT.map_frequency(
+            timeframe, source='ccxt', raise_error=raise_error
+        )
 
-        return timeframe
-
-    def get_candles(self, freq, assets, bar_count=None, start_dt=None,
+    def get_candles(self, freq, assets, bar_count=1, start_dt=None,
                     end_dt=None):
         is_single = (isinstance(assets, TradingPair))
         if is_single:
             assets = [assets]
 
         symbols = self.get_symbols(assets)
-        timeframe = self.get_timeframe(freq)
+        timeframe = CCXT.get_timeframe(freq)
 
-        ms = None
+        if timeframe not in self.api.timeframes:
+            freqs = [CCXT.get_frequency(t) for t in self.api.timeframes]
+            raise UnsupportedHistoryFrequencyError(
+                exchange=self.name,
+                freq=freq,
+                freqs=freqs,
+            )
+
+        if start_dt is not None and end_dt is not None:
+            raise ValueError(
+                'Please provide either start_dt or end_dt, not both.'
+            )
+
+        elif end_dt is not None:
+            # Make sure that end_dt really wants data in the past
+            # if it's close to now, we skip the 'since' parameters to
+            # lower the probability of error
+            bars_to_now = pd.date_range(
+                end_dt, pd.Timestamp.utcnow(), freq=freq
+            )
+            # See: https://github.com/ccxt/ccxt/issues/1360
+            if len(bars_to_now) > 1 or self.name in ['poloniex']:
+                dt_range = get_periods_range(
+                    end_dt=end_dt,
+                    periods=bar_count,
+                    freq=freq,
+                )
+                start_dt = dt_range[0]
+
+        since = None
         if start_dt is not None:
             delta = start_dt - get_epoch()
-            ms = int(delta.total_seconds()) * 1000
+            since = int(delta.total_seconds()) * 1000
 
         candles = dict()
-        for asset in assets:
-            try:
-                ohlcvs = self.api.fetch_ohlcv(
-                    symbol=symbols[0],
-                    timeframe=timeframe,
-                    since=ms,
-                    limit=bar_count,
-                    params={}
-                )
+        for index, asset in enumerate(assets):
+            ohlcvs = self.api.fetch_ohlcv(
+                symbol=symbols[index],
+                timeframe=timeframe,
+                since=since,
+                limit=bar_count,
+                params={}
+            )
 
-                candles[asset] = []
-                for ohlcv in ohlcvs:
-                    candles[asset].append(dict(
-                        last_traded=pd.to_datetime(
-                            ohlcv[0], unit='ms', utc=True
-                        ),
-                        open=ohlcv[1],
-                        high=ohlcv[2],
-                        low=ohlcv[3],
-                        close=ohlcv[4],
-                        volume=ohlcv[5]
-                    ))
-
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
+            candles[asset] = []
+            for ohlcv in ohlcvs:
+                candles[asset].append(dict(
+                    last_traded=pd.to_datetime(
+                        ohlcv[0], unit='ms', utc=True
+                    ),
+                    open=ohlcv[1],
+                    high=ohlcv[2],
+                    low=ohlcv[3],
+                    close=ohlcv[4],
+                    volume=ohlcv[5]
+                ))
+            candles[asset] = sorted(
+                candles[asset], key=lambda c: c['last_traded']
+            )
 
         if is_single:
             return six.next(six.itervalues(candles))
@@ -232,6 +481,7 @@ class CCXT(Exchange):
     def _fetch_symbol_map(self, is_local):
         try:
             return self.fetch_symbol_map(is_local)
+
         except ExchangeSymbolsNotFound:
             return None
 
@@ -339,16 +589,21 @@ class CCXT(Exchange):
                    and asset_def['end_minute'] != 'N/A' else None
 
         else:
-            params['symbol'] = self.get_catalyst_symbol(market)
+            params['symbol'] = get_catalyst_symbol(market)
             # TODO: add as an optional column
             params['leverage'] = 1.0
 
         return TradingPair(**params)
 
     def load_assets(self):
+        log.debug('loading assets for {}'.format(self.name))
         self.assets = []
 
         for market in self.markets:
+            if 'id' not in market:
+                log.warn('invalid market: {}'.format(market))
+                continue
+
             asset_defs = self.get_asset_defs(market)
 
             asset = None
@@ -378,8 +633,12 @@ class CCXT(Exchange):
             for key in balances:
                 balances_lower[key.lower()] = balances[key]
 
-        except Exception as e:
-            log.debug('error retrieving balances: {}', e)
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch balance {}: {}'.format(
+                    self.name, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return balances_lower
@@ -399,21 +658,61 @@ class CCXT(Exchange):
             The Catalyst order object
 
         """
-        if order_status['status'] == 'canceled':
+        order_id = order_status['id']
+        symbol = self.get_symbol(order_status['symbol'], source='ccxt')
+        asset = self.get_asset(symbol)
+
+        s = order_status['status']
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if s == 'canceled' or (s == 'closed' and filled == 0):
             status = ORDER_STATUS.CANCELLED
 
-        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
-            log.debug('found executed order {}'.format(order_status))
+        elif s == 'closed' and filled > 0:
+            if filled < amount:
+                log.warn(
+                    'order {id} is executed but only partially filled:'
+                    ' {filled} {symbol} out of {amount}'.format(
+                        id=order_status['status'],
+                        filled=order_status['filled'],
+                        symbol=asset.symbol,
+                        amount=order_status['amount'],
+                    )
+                )
+            else:
+                log.info(
+                    'order {id} executed in full: {filled} {symbol}'.format(
+                        id=order_id,
+                        filled=filled,
+                        symbol=asset.symbol,
+                    )
+                )
+
             status = ORDER_STATUS.FILLED
 
-        elif order_status['status'] == 'open':
+        elif s == 'open':
+            status = ORDER_STATUS.OPEN
+
+        elif filled > 0:
+            log.info(
+                'order {id} partially filled: {filled} {symbol} out of '
+                '{amount}, waiting for complete execution'.format(
+                    id=order_id,
+                    filled=filled,
+                    symbol=asset.symbol,
+                    amount=amount,
+                )
+            )
             status = ORDER_STATUS.OPEN
 
         else:
-            raise ValueError('invalid state for order')
-
-        amount = order_status['amount']
-        filled = order_status['filled']
+            log.warn(
+                'invalid state {} for order {}'.format(
+                    s, order_id
+                )
+            )
+            status = ORDER_STATUS.OPEN
 
         if order_status['side'] == 'sell':
             amount = -amount
@@ -423,25 +722,16 @@ class CCXT(Exchange):
         order_type = order_status['type']
 
         limit_price = price if order_type == 'limit' else None
-        stop_price = None  # TODO: add support
 
         executed_price = order_status['cost'] / order_status['amount']
         commission = order_status['fee']
         date = from_ms_timestamp(order_status['timestamp'])
 
-        # order_id = str(order_status['info']['clientOrderId'])
-        order_id = order_status['id']
-
-        # TODO: this won't work, redo the packages with a different key.
-        symbol = order_status['info']['symbol'] \
-            if 'symbol' in order_status['info'] \
-            else order_status['info']['Exchange']
-
         order = Order(
             dt=date,
-            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            asset=asset,
             amount=amount,
-            stop=stop_price,
+            stop=None,
             limit=limit_price,
             filled=filled,
             id=order_id,
@@ -469,7 +759,6 @@ class CCXT(Exchange):
             )
 
         side = 'buy' if amount > 0 else 'sell'
-
         if hasattr(self.api, 'amount_to_lots'):
             adj_amount = self.api.amount_to_lots(
                 symbol=symbol,
@@ -484,6 +773,12 @@ class CCXT(Exchange):
         else:
             adj_amount = abs(amount)
 
+        if adj_amount == 0:
+            raise CreateOrderError(
+                exchange=self.name,
+                e='order amount lower than the smallest lot: {}'.format(amount)
+            )
+
         try:
             result = self.api.create_order(
                 symbol=symbol,
@@ -492,14 +787,18 @@ class CCXT(Exchange):
                 amount=adj_amount,
                 price=price
             )
-        except ExchangeNotAvailable as e:
-            log.debug('unable to create order: {}'.format(e))
-            raise ExchangeRequestError(error=e)
-
         except InvalidOrder as e:
             log.warn('the exchange rejected the order: {}'.format(e))
             raise CreateOrderError(exchange=self.name, error=e)
 
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to create order {} / {}: {}'.format(
+                    self.name, symbol, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
         if 'info' not in result:
             raise ValueError('cannot use order without info attribute')
 
@@ -524,18 +823,121 @@ class CCXT(Exchange):
                 limit=None,
                 params=dict()
             )
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch open orders {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         orders = []
         for order_status in result:
-            order, executed_price = self._create_order(order_status)
+            order, _ = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 
         return orders
 
-    def get_order(self, order_id, asset_or_symbol=None):
+    def _process_order_fallback(self, order):
+        """
+        Fallback method for exchanges which do not play nice with
+        fetch-my-trades. Apparently, about 60% of exchanges will return
+        the correct executed values with this method. Others will support
+        fetch-my-trades.
+
+        Parameters
+        ----------
+        order: Order
+
+        Returns
+        -------
+        float
+
+        """
+        exc_order, price = self.get_order(
+            order.id, order.asset, return_price=True
+        )
+        order.status = exc_order.status
+
+        order.commission = exc_order.commission
+        if order.amount != exc_order.amount:
+            log.warn(
+                'executed order amount {} differs '
+                'from original'.format(
+                    exc_order.amount, order.amount
+                )
+            )
+            order.amount = exc_order.amount
+
+        if order.status == ORDER_STATUS.FILLED:
+            transaction = Transaction(
+                asset=order.asset,
+                amount=order.amount,
+                dt=pd.Timestamp.utcnow(),
+                price=price,
+                order_id=order.id,
+                commission=order.commission
+            )
+        return [transaction]
+
+    def process_order(self, order):
+        # TODO: move to parent class after tracking features in the parent
+        if not self.api.hasFetchMyTrades:
+            return self._process_order_fallback(order)
+
+        try:
+            all_trades = self.get_trades(order.asset)
+        except ExchangeRequestError as e:
+            log.warn(
+                'unable to fetch account trades, trying an alternate '
+                'method to find executed order {} / {}: {}'.format(
+                    order.id, order.asset.symbol, e
+                )
+            )
+            return self._process_order_fallback(order)
+
+        transactions = []
+        trades = [t for t in all_trades if t['order'] == order.id]
+        if not trades:
+            log.debug(
+                'order {} / {} not found in trades'.format(
+                    order.id, order.asset.symbol
+                )
+            )
+            return transactions
+
+        trades.sort(key=lambda t: t['timestamp'], reverse=False)
+        order.filled = 0
+        order.commission = 0
+        for trade in trades:
+            # status property will update automatically
+            filled = trade['amount'] * order.direction
+            order.filled += filled
+
+            commission = 0
+            if 'fee' in trade and 'cost' in trade['fee']:
+                commission = trade['fee']['cost']
+                order.commission += commission
+
+            order.check_triggers(
+                price=trade['price'],
+                dt=pd.to_datetime(trade['timestamp'], unit='ms', utc=True),
+            )
+            transaction = Transaction(
+                asset=order.asset,
+                amount=filled,
+                dt=pd.Timestamp.utcnow(),
+                price=trade['price'],
+                order_id=order.id,
+                commission=commission
+            )
+            transactions.append(transaction)
+
+        order.broker_order_id = ', '.join([t['id'] for t in trades])
+        return transactions
+
+    def get_order(self, order_id, asset_or_symbol=None, return_price=False):
         if asset_or_symbol is None:
             log.debug(
                 'order not found in memory, the request might fail '
@@ -547,10 +949,19 @@ class CCXT(Exchange):
             order_status = self.api.fetch_order(id=order_id, symbol=symbol)
             order, executed_price = self._create_order(order_status)
 
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
+            if return_price:
+                return order, executed_price
 
-        return order, executed_price
+            else:
+                return order
+
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
 
     def cancel_order(self, order_param, asset_or_symbol=None):
         order_id = order_param.id \
@@ -566,7 +977,12 @@ class CCXT(Exchange):
                 if asset_or_symbol is not None else None
             self.api.cancel_order(id=order_id, symbol=symbol)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to cancel order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
     def tickers(self, assets):
@@ -582,31 +998,46 @@ class CCXT(Exchange):
         list[dict[str, float]
 
         """
-        tickers = dict()
+        tickers = {}
         for asset in assets:
+            symbol = self.get_symbol(asset)
+
+            # Test the CCXT throttling further to see if we need this
+            self.ask_request()
+
+            # TODO: use fetch_tickers() for efficiency
+            # I tried using fetch_tickers() but noticed some
+            # inconsistencies, see issue:
+            # https://github.com/ccxt/ccxt/issues/870
             try:
-                ccxt_symbol = self.get_symbol(asset)
-                ticker = self.api.fetch_ticker(ccxt_symbol)
-
-                ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
-
-                if 'last_price' not in ticker:
-                    # TODO: any more exceptions?
-                    ticker['last_price'] = ticker['last']
-
-                # Using the volume represented in the base currency
-                ticker['volume'] = ticker['baseVolume'] \
-                    if 'baseVolume' in ticker else 0
-
-                tickers[asset] = ticker
-
-            except ExchangeNotAvailable as e:
+                ticker = self.api.fetch_ticker(symbol=symbol)
+            except (ExchangeError, NetworkError) as e:
                 log.warn(
-                    'unable to fetch ticker: {} {}'.format(
-                        self.name, asset.symbol
+                    'unable to fetch ticker {} / {}: {}'.format(
+                        self.name, asset.symbol, e
                     )
                 )
-                raise ExchangeRequestError(error=e)
+                continue
+
+            ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
+
+            if 'last_price' not in ticker:
+                # TODO: any more exceptions?
+                ticker['last_price'] = ticker['last']
+
+            if 'baseVolume' in ticker and ticker['baseVolume'] is not None:
+                # Using the volume represented in the base currency
+                ticker['volume'] = ticker['baseVolume']
+
+            elif 'info' in ticker and 'bidQty' in ticker['info'] \
+                    and 'askQty' in ticker['info']:
+                ticker['volume'] = float(ticker['info']['bidQty']) + \
+                                   float(ticker['info']['askQty'])
+
+            else:
+                ticker['volume'] = 0
+
+            tickers[asset] = ticker
 
         return tickers
 
@@ -636,3 +1067,27 @@ class CCXT(Exchange):
                 ))
 
         return result
+
+    def get_trades(self, asset, my_trades=True, start_dt=None, limit=None):
+        if not my_trades:
+            raise NotImplemented(
+                'get_trades only supports "my trades"'
+            )
+
+        # TODO: is it possible to sort this? Limit is useless otherwise.
+        ccxt_symbol = self.get_symbol(asset)
+        try:
+            trades = self.api.fetch_my_trades(
+                symbol=ccxt_symbol,
+                since=start_dt,
+                limit=limit,
+            )
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch trades {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
+        return trades

catalyst/exchange/exchange.py

@@ -5,19 +5,20 @@ from time import sleep
 
 import numpy as np
 import pandas as pd
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
-from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError
-from catalyst.exchange.exchange_utils import get_exchange_symbols, \
-    get_frequency, resample_history_df
+    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
+    TickerNotFoundError, NotEnoughCashError
+from catalyst.exchange.utils.datetime_utils import get_delta, \
+    get_periods_range, \
+    get_periods, get_start_dt, get_frequency
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
+    resample_history_df, has_bundle
+from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -38,6 +39,8 @@ class Exchange:
         self.request_cpt = None
         self.bundle = ExchangeBundle(self.name)
 
+        self.low_balance_threshold = None
+
     @abstractproperty
     def account(self):
         pass
@@ -46,6 +49,9 @@ class Exchange:
     def time_skew(self):
         pass
 
+    def has_bundle(self, data_frequency):
+        return has_bundle(self.name, data_frequency)
+
     def is_open(self, dt):
         """
         Is the exchange open
@@ -148,7 +154,7 @@ class Exchange:
 
     def get_assets(self, symbols=None, data_frequency=None,
                    is_exchange_symbol=False,
-                   is_local=None):
+                   is_local=None, quote_currency=None):
         """
         The list of markets for the specified symbols.
 
@@ -172,6 +178,14 @@ class Exchange:
         if symbols is None:
             # Make a distinct list of all symbols
             symbols = list(set([asset.symbol for asset in self.assets]))
+
+            if quote_currency is not None:
+                for symbol in symbols[:]:
+                    suffix = '_{}'.format(quote_currency.lower())
+
+                    if not symbol.endswith(suffix):
+                        symbols.remove(symbol)
+
             is_exchange_symbol = False
 
         assets = []
@@ -220,11 +234,15 @@ class Exchange:
         """
         asset = None
 
+        # TODO: temp mapping, fix to use a single symbol convention
+        og_symbol = symbol
+        symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol
         log.debug(
             'searching assets for: {} {}'.format(
                 self.name, symbol
             )
         )
+        # TODO: simplify and loose the loop
         for a in self.assets:
             if asset is not None:
                 break
@@ -235,10 +253,10 @@ class Exchange:
 
             elif data_frequency is not None:
                 applies = (
-                        (
-                                data_frequency == 'minute' and a.end_minute is not None)
-                        or (
-                                data_frequency == 'daily' and a.end_daily is not None)
+                    (
+                        data_frequency == 'minute' and a.end_minute is not None)
+                    or (
+                        data_frequency == 'daily' and a.end_daily is not None)
                 )
 
             else:
@@ -246,15 +264,24 @@ class Exchange:
 
             # The symbol provided may use the Catalyst or the exchange
             # convention
-            key = a.exchange_symbol if is_exchange_symbol else a.symbol
-            if not asset and key.lower() == symbol.lower() and applies:
-                asset = a
+            key = a.exchange_symbol if \
+                is_exchange_symbol else self.get_symbol(a)
+            if not asset and key.lower() == symbol.lower():
+                if applies:
+                    asset = a
+
+                else:
+                    raise NoDataAvailableOnExchange(
+                        symbol=key,
+                        exchange=self.name,
+                        data_frequency=data_frequency,
+                    )
 
         if asset is None:
             supported_symbols = sorted([a.symbol for a in self.assets])
 
             raise SymbolNotFoundOnExchange(
-                symbol=symbol,
+                symbol=og_symbol,
                 exchange=self.name.title(),
                 supported_symbols=supported_symbols
             )
@@ -272,6 +299,16 @@ class Exchange:
             self._symbol_maps[index] = symbol_map
             return symbol_map
 
+    @abstractmethod
+    def init(self):
+        """
+        Load the asset list from the network.
+
+        Returns
+        -------
+
+        """
+
     @abstractmethod
     def load_assets(self, is_local=False):
         """
@@ -377,6 +414,7 @@ class Exchange:
 
         return value
 
+    # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 data_frequency, field, previous_value=None):
         """
@@ -401,7 +439,7 @@ class Exchange:
         series = pd.Series(values, index=dates)
 
         periods = get_periods_range(
-            start_dt, end_dt, data_frequency
+            start_dt=start_dt, end_dt=end_dt, freq=data_frequency
         )
         # TODO: ensure that this working as expected, if not use fillna
         series = series.reindex(
@@ -465,39 +503,37 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
-        adj_bar_count = candle_size * bar_count
-
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
             bar_count=bar_count,
-            start_dt=start_dt if not is_current else None,
             end_dt=end_dt if not is_current else None,
         )
 
         series = dict()
         for asset in candles:
+            first_candle = candles[asset][0]
             asset_series = self.get_series_from_candles(
                 candles=candles[asset],
-                start_dt=start_dt,
+                start_dt=first_candle['last_traded'],
                 end_dt=end_dt,
                 data_frequency=frequency,
                 field=field,
             )
-            if end_dt is not None:
-                delta = get_delta(candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_traded = asset_series.index[-1]
 
-                if last_traded < adj_end_dt:
-                    raise LastCandleTooEarlyError(
-                        last_traded=last_traded,
-                        end_dt=adj_end_dt,
-                        exchange=self.name,
-                    )
+            # Checking to make sure that the dates match
+            delta = get_delta(candle_size, data_frequency)
+            adj_end_dt = end_dt - delta
+            last_traded = asset_series.index[-1]
+
+            if last_traded < adj_end_dt:
+                raise LastCandleTooEarlyError(
+                    last_traded=last_traded,
+                    end_dt=adj_end_dt,
+                    exchange=self.name,
+                )
+
             series[asset] = asset_series
 
         df = pd.DataFrame(series)
@@ -551,11 +587,11 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
+        # TODO: this function needs some work, we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
-
         try:
             series = self.bundle.get_history_window_series_and_load(
                 assets=assets,
@@ -582,15 +618,14 @@ class Exchange:
                 # The get_history method supports multiple asset
                 # Use the original frequency to let each api optimize
                 # the size of result sets
-                trailing_bar_count = get_periods(
+                trailing_bars = get_periods(
                     trailing_dt, end_dt, freq
                 )
                 candles = self.get_candles(
                     freq=freq,
                     assets=asset,
-                    bar_count=trailing_bar_count,
-                    start_dt=start_dt,
-                    end_dt=end_dt
+                    end_dt=end_dt,
+                    bar_count=trailing_bars if trailing_bars < 500 else 500,
                 )
 
                 last_value = series[asset].iloc(0) if asset in series \
@@ -619,46 +654,98 @@ class Exchange:
 
         return df
 
-    def calculate_totals(self, check_cash=False, positions=None):
+    def _check_low_balance(self, currency, balances, amount):
+        free = balances[currency]['free'] if currency in balances else 0.0
+
+        if free < amount:
+            return free, True
+
+        else:
+            return free, False
+
+    def sync_positions(self, positions, cash=None, check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
+        Parameters
+        ----------
+        positions:
+            The positions to synchronize.
+
+        check_balances:
+            Check balances amounts against the exchange.
+
         """
-        log.debug('synchronizing portfolio with exchange {}'.format(self.name))
-
-        cash = None
-        if check_cash:
+        free_cash = 0.0
+        if check_balances:
+            log.debug('fetching {} balances'.format(self.name))
             balances = self.get_balances()
-
-            cash = balances[self.base_currency]['free'] \
-                if self.base_currency in balances else None
-
-            if cash is None:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=self.base_currency,
-                    exchange=self.name
+            log.debug(
+                'got free balances for {} currencies'.format(
+                    len(balances)
                 )
-            log.debug('found base currency balance: {}'.format(cash))
+            )
+            if cash is not None:
+                free_cash, is_lower = self._check_low_balance(
+                    currency=self.base_currency,
+                    balances=balances,
+                    amount=cash,
+                )
+                if is_lower:
+                    raise NotEnoughCashError(
+                        currency=self.base_currency,
+                        exchange=self.name,
+                        free=free_cash,
+                        cash=cash,
+                    )
 
         positions_value = 0.0
-        if positions:
+        if positions is not None:
             assets = set([position.asset for position in positions])
             tickers = self.tickers(assets)
-            log.debug('got tickers for positions: {}'.format(tickers))
 
-            for asset in tickers:
+            for position in positions:
+                asset = position.asset
+                if asset not in tickers:
+                    raise TickerNotFoundError(
+                        symbol=asset.symbol,
+                        exchange=self.name,
+                    )
+
                 ticker = tickers[asset]
-                positions = [p for p in positions if p.asset == asset]
+                log.debug(
+                    'updating {symbol} position, last traded on {dt} for '
+                    '{price}{currency}'.format(
+                        symbol=asset.symbol,
+                        dt=ticker['last_traded'],
+                        price=ticker['last_price'],
+                        currency=asset.quote_currency,
+                    )
+                )
+                position.last_sale_price = ticker['last_price']
+                position.last_sale_date = ticker['last_traded']
 
-                for position in positions:
-                    position.last_sale_price = ticker['last_price']
-                    position.last_sale_date = ticker['last_traded']
+                positions_value += \
+                    position.amount * position.last_sale_price
 
-                    positions_value += \
-                        position.amount * position.last_sale_price
+                if check_balances:
+                    free, is_lower = self._check_low_balance(
+                        currency=asset.base_currency,
+                        balances=balances,
+                        amount=position.amount,
+                    )
 
-        return cash, positions_value
+                    if is_lower:
+                        log.warn(
+                            'detected lower balance for {} on {}: {} < {}, '
+                            'updating position amount'.format(
+                                asset.symbol, self.name, free, position.amount
+                            )
+                        )
+                        position.amount = free
+
+        return free_cash, positions_value
 
     def order(self, asset, amount, style):
         """Place an order.
@@ -815,6 +902,22 @@ class Exchange:
         """
         pass
 
+    @abstractmethod
+    def process_order(self, order):
+        """
+        Similar to get_order but looks only for executed orders.
+
+        Parameters
+        ----------
+        order: Order
+
+        Returns
+        -------
+        float
+            Avg execution price
+
+        """
+
     @abstractmethod
     def cancel_order(self, order_param, symbol_or_asset=None):
         """Cancel an open order.
@@ -829,8 +932,7 @@ class Exchange:
         pass
 
     @abstractmethod
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
+    def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None):
         """
         Retrieve OHLCV candles for the given assets
 
@@ -895,7 +997,7 @@ class Exchange:
     @abc.abstractmethod
     def get_orderbook(self, asset, order_type, limit):
         """
-        Retrieve the the orderbook for the given trading pair.
+        Retrieve the orderbook for the given trading pair.
 
         Parameters
         ----------
@@ -909,3 +1011,20 @@ class Exchange:
         list[dict[str, float]
         """
         pass
+
+    @abc.abstractmethod
+    def get_trades(self, asset, my_trades, start_dt, limit):
+        """
+        Retrieve a list of trades.
+
+        Parameters
+        ----------
+        my_trades: bool
+            List only my trades.
+        start_dt
+        limit
+
+        Returns
+        -------
+
+        """

catalyst/exchange/exchange_algorithm.py

@@ -10,44 +10,44 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import pickle
 import signal
 import sys
 from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
-from time import sleep
-
-import logbook
-import pandas as pd
 
 import catalyst.protocol as zp
+import logbook
+import pandas as pd
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangePortfolioDataError,
-    OrderTypeNotSupported, )
+    OrderTypeNotSupported)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-from catalyst.exchange.exchange_utils import (
+from catalyst.exchange.live_graph_clock import LiveGraphClock
+from catalyst.exchange.simple_clock import SimpleClock
+from catalyst.exchange.utils.exchange_utils import (
     save_algo_object,
     get_algo_object,
     get_algo_folder,
     get_algo_df,
     save_algo_df,
     group_assets_by_exchange, )
-from catalyst.exchange.live_graph_clock import LiveGraphClock
-from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
     stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
+from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
 from catalyst.utils.api_support import api_method
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
+from redo import retry
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
@@ -70,12 +70,26 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                 and self.sim_params.arena == 'backtest':
             self.simulate_orders = True
 
+        # Operations with retry features
+        self.attempts = dict(
+            get_transactions_attempts=5,
+            order_attempts=5,
+            synchronize_portfolio_attempts=5,
+            get_order_attempts=5,
+            get_open_orders_attempts=5,
+            cancel_order_attempts=5,
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
+
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
             cancel_policy=self.cancel_policy,
             simulate_orders=self.simulate_orders,
-            exchanges=self.exchanges
+            exchanges=self.exchanges,
+            attempts=self.attempts,
         )
 
     @staticmethod
@@ -115,7 +129,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_commission(self, maker=None, taker=None):
-        key = self.blotter.commission_models.keys()[0]
+        key = list(self.blotter.commission_models.keys())[0]
         if maker is not None:
             self.blotter.commission_models[key].maker = maker
 
@@ -124,7 +138,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_slippage(self, spread=None):
-        key = self.blotter.slippage_models.keys()[0]
+        key = list(self.blotter.slippage_models.keys())[0]
         if spread is not None:
             self.blotter.slippage_models[key].spread = spread
 
@@ -218,28 +232,28 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         """
         tracker = self.perf_tracker
-        period = tracker.todays_performance
+        cum = tracker.cumulative_performance
 
-        pos_stats = period.position_tracker.stats()
-        period_stats = calc_period_stats(pos_stats, period.ending_cash)
+        pos_stats = cum.position_tracker.stats()
+        period_stats = calc_period_stats(pos_stats, cum.ending_cash)
 
         stats = dict(
             period_start=tracker.period_start,
             period_end=tracker.period_end,
             capital_base=tracker.capital_base,
             progress=tracker.progress,
-            ending_value=period.ending_value,
-            ending_exposure=period.ending_exposure,
-            capital_used=period.cash_flow,
-            starting_value=period.starting_value,
-            starting_exposure=period.starting_exposure,
-            starting_cash=period.starting_cash,
-            ending_cash=period.ending_cash,
-            portfolio_value=period.ending_cash + period.ending_value,
-            pnl=period.pnl,
-            returns=period.returns,
-            period_open=period.period_open,
-            period_close=period.period_close,
+            ending_value=cum.ending_value,
+            ending_exposure=cum.ending_exposure,
+            capital_used=cum.cash_flow,
+            starting_value=cum.starting_value,
+            starting_exposure=cum.starting_exposure,
+            starting_cash=cum.starting_cash,
+            ending_cash=cum.ending_cash,
+            portfolio_value=cum.ending_cash + cum.ending_value,
+            pnl=cum.pnl,
+            returns=cum.returns,
+            period_open=start_dt,
+            period_close=end_dt,
             gross_leverage=period_stats.gross_leverage,
             net_leverage=period_stats.net_leverage,
             short_exposure=pos_stats.short_exposure,
@@ -256,6 +270,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
+        period = tracker.todays_performance
         stats['positions'] = period.position_tracker.get_positions_list()
 
         # we want the key to be absent, not just empty
@@ -276,12 +291,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         return stats
 
+    def run(self, data=None, overwrite_sim_params=True):
+        data.attempts = self.attempts
+        return super(ExchangeTradingAlgorithmBase, self).run(
+            data, overwrite_sim_params
+        )
+
 
 class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
         self.frame_stats = list()
+        self.state = {}
         log.info('initialized trading algorithm in backtest mode')
 
     def is_last_frame_of_day(self, data):
@@ -328,6 +350,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
         self.stats_output = kwargs.pop('stats_output', None)
+        self._analyze_live = kwargs.pop('analyze_live', None)
+        self.end = kwargs.pop('end', None)
 
         self._clock = None
         self.frame_stats = list()
@@ -342,20 +366,51 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         self.is_running = True
 
-        self.retry_check_open_orders = 5
-        self.retry_synchronize_portfolio = 5
-        self.retry_get_open_orders = 5
-        self.retry_order = 2
-        self.retry_delay = 5
+        self.stats_minutes = 1
 
-        self.stats_minutes = 10
+        self._last_orders = []
+        self.trading_client = None
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
-        signal.signal(signal.SIGINT, self.signal_handler)
+        try:
+            signal.signal(signal.SIGINT, self.signal_handler)
+        except ValueError:
+            log.warn("Can't initialize signal handler inside another thread."
+                     "Exit should be handled by the user.")
 
         log.info('initialized trading algorithm in live mode')
 
+    def interrupt_algorithm(self):
+        self.is_running = False
+
+        if self._analyze is None:
+            log.info('Exiting the algorithm.')
+
+        else:
+            log.info('Exiting the algorithm. Calling `analyze()` '
+                     'before exiting the algorithm.')
+
+            algo_folder = get_algo_folder(self.algo_namespace)
+            folder = join(algo_folder, 'daily_performance')
+            files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+            daily_perf_list = []
+            for item in files:
+                filename = join(folder, item)
+
+                with open(filename, 'rb') as handle:
+                    perf_period = pickle.load(handle)
+                    perf_period_dict = perf_period.to_dict()
+                    daily_perf_list.append(perf_period_dict)
+
+            stats = pd.DataFrame(daily_perf_list)
+            stats.set_index('period_close', drop=False, inplace=True)
+
+            self.analyze(stats)
+
+        sys.exit(0)
+
     def signal_handler(self, signal, frame):
         """
         Handles the keyboard interruption signal.
@@ -369,31 +424,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         -------
 
         """
-        self.is_running = False
-
-        if self._analyze is None:
-            log.info('Interruption signal detected {}, exiting the '
-                     'algorithm'.format(signal))
-
-        else:
-            log.info('Interruption signal detected {}, calling `analyze()` '
-                     'before exiting the algorithm'.format(signal))
-
-            algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
-
-            daily_perf_list = []
-            for item in files:
-                filename = join(folder, item)
-                with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
-
-            stats = pd.DataFrame(daily_perf_list)
-
-            self.analyze(stats)
-
-        sys.exit(0)
+        log.info('Interruption signal detected {}, exiting the '
+                 'algorithm'.format(signal))
+        self.interrupt_algorithm()
 
     @property
     def clock(self):
@@ -419,10 +452,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
-        if self.live_graph:
+        if self.live_graph or self._analyze_live is not None:
             self._clock = LiveGraphClock(
                 self.sim_params.sessions,
-                context=self
+                context=self,
+                callback=self._analyze_live,
             )
         else:
             self._clock = SimpleClock(
@@ -431,25 +465,82 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def _create_generator(self, sim_params):
-        if self.perf_tracker is None:
-            self.perf_tracker = get_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker'
-            )
+    def _init_trading_client(self):
+        """
+        This replaces Ziplines `_create_generator` method. The main difference
+        is that we are restoring performance tracker objects if available.
+        This allows us to stop/start algos without loosing their state.
 
-        # Call the simulation trading algorithm for side-effects:
-        # it creates the perf tracker
-        TradingAlgorithm._create_generator(self, sim_params)
-        self.trading_client = ExchangeAlgorithmExecutor(
-            self,
-            sim_params,
-            self.data_portal,
-            self.clock,
-            self._create_benchmark_source(),
-            self.restrictions,
-            universe_func=self._calculate_universe
+        """
+        self.state = get_algo_object(
+            algo_name=self.algo_namespace,
+            key='context.state',
         )
+        if self.state is None:
+            self.state = {}
+
+        if self.perf_tracker is None:
+            # Note from the Zipline dev:
+            # HACK: When running with the `run` method, we set perf_tracker to
+            # None so that it will be overwritten here.
+            tracker = self.perf_tracker = PerformanceTracker(
+                sim_params=self.sim_params,
+                trading_calendar=self.trading_calendar,
+                env=self.trading_environment,
+            )
+            # Set the dt initially to the period start by forcing it to change.
+            self.on_dt_changed(self.sim_params.start_session)
+
+            new_position_tracker = tracker.position_tracker
+            tracker.position_tracker = None
+
+            # Unpacking the perf_tracker and positions if available
+            cum_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key='cumulative_performance',
+            )
+            if cum_perf is not None:
+                tracker.cumulative_performance = cum_perf
+                # Ensure single common position tracker
+                tracker.position_tracker = cum_perf.position_tracker
+
+            today = pd.Timestamp.utcnow().floor('1D')
+            todays_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                rel_path='daily_performance',
+            )
+            if todays_perf is not None:
+                # Ensure single common position tracker
+                if tracker.position_tracker is not None:
+                    todays_perf.position_tracker = tracker.position_tracker
+                else:
+                    tracker.position_tracker = todays_perf.position_tracker
+
+                tracker.todays_performance = todays_perf
+
+            if tracker.position_tracker is None:
+                # Use a new position_tracker if not is found in the state
+                tracker.position_tracker = new_position_tracker
+
+        if not self.initialized:
+            # Calls the initialize function of the algorithm
+            self.initialize(*self.initialize_args, **self.initialize_kwargs)
+            self.initialized = True
+
+        self.trading_client = ExchangeAlgorithmExecutor(
+            algo=self,
+            sim_params=self.sim_params,
+            data_portal=self.data_portal,
+            clock=self.clock,
+            benchmark_source=self._create_benchmark_source(),
+            restrictions=self.restrictions,
+            universe_func=self._calculate_universe,
+        )
+
+    def get_generator(self):
+        if self.trading_client is None:
+            self._init_trading_client()
 
         return self.trading_client.transform()
 
@@ -459,7 +550,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self):
         """
         Synchronizes the portfolio tracked by the algorithm to refresh
         its current value.
@@ -468,10 +559,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         positions, returning the available cash, and raising error
         if the data goes out of sync.
 
-        Parameters
-        ----------
-        attempt_index: int
-
         Returns
         -------
         float
@@ -481,63 +568,60 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The total value of all tracked positions.
 
         """
+        check_balances = (not self.simulate_orders)
+        base_currency = None
         tracker = self.perf_tracker.position_tracker
         total_cash = 0.0
         total_positions_value = 0.0
 
-        try:
-            # Position keys correspond to assets
-            positions = self.portfolio.positions
-            assets = list(positions)
-            exchange_assets = group_assets_by_exchange(assets)
-            for exchange_name in self.exchanges:
-                assets = exchange_assets[exchange_name] \
-                    if exchange_name in exchange_assets else []
+        # Position keys correspond to assets
+        positions = self.portfolio.positions
+        assets = list(positions)
+        exchange_assets = group_assets_by_exchange(assets)
+        for exchange_name in self.exchanges:
+            assets = exchange_assets[exchange_name] \
+                if exchange_name in exchange_assets else []
 
-                exchange_positions = \
-                    [positions[asset] for asset in assets]
-
-                check_cash = (not self.simulate_orders)
-
-                exchange = self.exchanges[exchange_name]  # Type: Exchange
-                cash, positions_value = exchange.calculate_totals(
-                    positions=exchange_positions,
-                    check_cash=check_cash,
-                )
-                total_positions_value += positions_value
-
-                if cash is not None:
-                    total_cash += cash
-
-                for position in exchange_positions:
-                    tracker.update_position(
-                        asset=position.asset,
-                        last_sale_date=position.last_sale_date,
-                        last_sale_price=position.last_sale_price
-                    )
-
-            if cash is None:
-                total_cash = self.portfolio.cash
-
-            elif total_cash < self.portfolio.cash:
-                raise ValueError('Cash on exchanges is lower than the algo.')
-
-            return total_cash, total_positions_value
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'update portfolio attempt {}: {}'.format(attempt_index, e)
+            exchange_positions = copy.deepcopy(
+                [positions[asset] for asset in assets if asset in positions]
             )
-            if attempt_index < self.retry_synchronize_portfolio:
-                sleep(self.retry_delay)
-                return self.synchronize_portfolio(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='update-portfolio',
-                    attempts=attempt_index,
-                    error=e
+
+            exchange = self.exchanges[exchange_name]  # Type: Exchange
+
+            if base_currency is None:
+                base_currency = exchange.base_currency
+
+            # Don't check the cash if there are open orders. This could
+            # results in false positives.
+            orders = []
+            for asset in self.blotter.open_orders:
+                asset_orders = self.blotter.open_orders[asset]
+                if asset_orders:
+                    orders += asset_orders
+
+            required_cash = self.portfolio.cash if not orders else None
+            cash, positions_value = exchange.sync_positions(
+                positions=exchange_positions,
+                check_balances=check_balances,
+                cash=required_cash,
+            )
+            total_cash += cash
+            total_positions_value += positions_value
+
+            # Applying modifications to the original positions
+            for position in exchange_positions:
+                tracker.update_position(
+                    asset=position.asset,
+                    amount=position.amount,
+                    last_sale_date=position.last_sale_date,
+                    last_sale_price=position.last_sale_price,
                 )
 
+        if not check_balances:
+            total_cash = self.portfolio.cash
+
+        return total_cash, total_positions_value
+
     def add_pnl_stats(self, period_stats):
         """
         Save p&l stats.
@@ -627,20 +711,39 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if not self.is_running:
             return
 
+        if self.end is not None and self.end < data.current_dt:
+            log.info('Algorithm has reached specified end time. Finishing...')
+            self.interrupt_algorithm()
+
         # Resetting the frame stats every day to minimize memory footprint
         today = data.current_dt.floor('1D')
         if self.current_day is not None and today > self.current_day:
             self.frame_stats = list()
 
-        new_transactions, new_commissions, closed_orders = \
-            self.blotter.get_transactions(data)
+        self.performance_needs_update = False
+        orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
+        if orders != self._last_orders:
+            self.performance_needs_update = True
 
-        if len(new_transactions) > 0:
+        # Saving current orders to detect changes in the next frame
+        self._last_orders = copy.deepcopy(orders)
+
+        if self.performance_needs_update:
             self.perf_tracker.update_performance()
+            self.performance_needs_update = False
+
+        if self.portfolio_needs_update:
+            cash, positions_value = retry(
+                action=self.synchronize_portfolio,
+                attempts=self.attempts['synchronize_portfolio_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.')
+            )
+            self.portfolio_needs_update = False
 
-        cash, positions_value = self.synchronize_portfolio()
         log.info(
-            'got totals from exchanges, cash: {} positions: {}'.format(
+            'portfolio balances, cash: {}, positions: {}'.format(
                 cash, positions_value
             )
         )
@@ -652,22 +755,34 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        self._save_algo_state(data)
+        self.current_day = data.current_dt.floor('1D')
+
+    def _save_algo_state(self, data):
+        today = data.current_dt.floor('1D')
         try:
             self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
-        # TODO: pickle does not seem to work in python 3
-        try:
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker',
-                obj=self.perf_tracker
-            )
-        except Exception as e:
-            log.warn('unable to save minute perfs to disk: {}'.format(e))
-
-        self.current_day = data.current_dt.floor('1D')
+        log.debug('saving cumulative performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='cumulative_performance',
+            obj=self.perf_tracker.cumulative_performance,
+        )
+        log.debug('saving todays performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=self.perf_tracker.todays_performance,
+            rel_path='daily_performance'
+        )
+        log.debug('saving context.state object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='context.state',
+            obj=self.state)
 
     def _process_stats(self, data):
         today = data.current_dt.floor('1D')
@@ -677,7 +792,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.perf_tracker.update_performance()
 
         frame_stats = self.prepare_period_stats(
-            data.current_dt, data.current_dt + timedelta(minutes=1))
+            data.current_dt, data.current_dt + timedelta(minutes=1)
+        )
 
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
@@ -699,7 +815,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 stats=get_pretty_stats(
                     stats=self.frame_stats,
                     recorded_cols=recorded_cols,
-                    num_rows=self.stats_minutes
+                    num_rows=self.stats_minutes,
                 )
             ))
 
@@ -709,12 +825,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             start_dt=today,
             end_dt=data.current_dt
         )
-        save_algo_object(
-            algo_name=self.algo_namespace,
-            key=today.strftime('%Y-%m-%d'),
-            obj=daily_stats,
-            rel_path='daily_perf'
-        )
 
         return recorded_cols
 
@@ -751,33 +861,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def batch_market_order(self, share_counts):
         raise NotImplementedError()
 
-    def _get_open_orders(self, asset=None, attempt_index=0):
-        try:
-            if asset:
-                exchange = self.exchanges[asset.exchange]
-                return exchange.get_open_orders(asset)
+    def _get_open_orders(self, asset=None):
+        if asset:
+            exchange = self.exchanges[asset.exchange]
+            return exchange.get_open_orders(asset)
 
-            else:
-                open_orders = []
-                for exchange_name in self.exchanges:
-                    exchange = self.exchanges[exchange_name]
-                    exchange_orders = exchange.get_open_orders()
-                    open_orders.append(exchange_orders)
+        else:
+            open_orders = []
+            for exchange_name in self.exchanges:
+                exchange = self.exchanges[exchange_name]
+                exchange_orders = exchange.get_open_orders()
+                open_orders.append(exchange_orders)
 
-                return open_orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_open_orders:
-                sleep(self.retry_delay)
-                return self._get_open_orders(asset, attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='open-orders',
-                    attempts=attempt_index,
-                    error=e
-                )
+            return open_orders
 
     @error_keywords(sid='Keyword argument `sid` is no longer supported for '
                         'get_open_orders. Use `asset` instead.')
@@ -799,7 +895,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             If an asset is passed then this will return a list of the open
             orders for this asset.
         """
-        return self._get_open_orders(asset)
+        return retry(
+            action=self._get_open_orders,
+            attempts=self.attempts['get_open_orders_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching open orders again.'),
+            args=(asset,))
 
     @api_method
     def get_order(self, order_id, exchange_name):
@@ -819,7 +921,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The execution price per share of the order
         """
         exchange = self.exchanges[exchange_name]
-        return exchange.get_order(order_id)
+        return retry(
+            action=exchange.get_order,
+            attempts=self.attempts['get_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching orders again.'),
+            args=(order_id,))
 
     @api_method
     def cancel_order(self, order_param, exchange_name):
@@ -836,4 +944,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if isinstance(order_param, zp.Order):
             order_id = order_param.id
 
-        exchange.cancel_order(order_id)
+        retry(
+            action=exchange.cancel_order,
+            attempts=self.attempts['cancel_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('cancelling order again.'),
+            args=(order_id,))

catalyst/exchange/exchange_asset_finder.py

@@ -0,0 +1,179 @@
+import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.factory import find_exchanges
+from logbook import Logger
+
+log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
+
+
+class ExchangeAssetFinder(object):
+    def __init__(self, exchanges):
+        self.exchanges = exchanges
+
+    @property
+    def sids(self):
+        """
+        This seems to be used to pre-fetch assets.
+        I don't think that we need this for live-trading.
+        Leaving the list empty.
+        """
+        all_sids = []
+        for exchange_name in self.exchanges:
+            # This is what initializes each exchanges at the beginning
+            # of an algo
+            exchange = self.exchanges[exchange_name]
+            exchange.init()
+
+            all_sids += [asset.sid for asset in exchange.assets]
+
+        sids = list(set(all_sids))
+        return sids
+
+    def retrieve_asset(self, sid, default_none=False):
+        """
+        Retrieve the first Asset found for a given sid.
+        """
+        asset = None
+        for exchange_name in self.exchanges:
+            if asset is not None:
+                break
+
+            exchange = self.exchanges[exchange_name]
+            assets = [asset for asset in exchange.assets if asset.sid == sid]
+            if assets:
+                asset = assets[0]
+
+        return asset
+
+    def retrieve_all(self, sids, default_none=False):
+        """
+        Retrieve all assets in `sids`.
+
+        Parameters
+        ----------
+        sids : iterable of int
+            Assets to retrieve.
+        default_none : bool
+            If True, return None for failed lookups.
+            If False, raise `SidsNotFound`.
+
+        Returns
+        -------
+        assets : list[Asset or None]
+            A list of the same length as `sids` containing Assets (or Nones)
+            corresponding to the requested sids.
+
+        Raises
+        ------
+        SidsNotFound
+            When a requested sid is not found and default_none=False.
+        """
+        assets = []
+        for exchange_name in self.exchanges:
+            exchange = self.exchanges[exchange_name]
+            xas = [asset for asset in exchange.assets if asset.sid in sids]
+            assets += xas
+
+        return assets
+
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
+        """Lookup an asset by symbol.
+
+        Parameters
+        ----------
+        symbol : str
+            The ticker symbol to resolve.
+        as_of_date : datetime or None
+            Look up the last owner of this symbol as of this datetime.
+            If ``as_of_date`` is None, then this can only resolve the equity
+            if exactly one equity has ever owned the ticker.
+        fuzzy : bool, optional
+            Should fuzzy symbol matching be used? Fuzzy symbol matching
+            attempts to resolve differences in representations for
+            shareclasses. For example, some people may represent the ``A``
+            shareclass of ``BRK`` as ``BRK.A``, where others could write
+            ``BRK_A``.
+
+        Returns
+        -------
+        equity : Asset
+            The equity that held ``symbol`` on the given ``as_of_date``, or the
+            only equity to hold ``symbol`` if ``as_of_date`` is None.
+
+        Raises
+        ------
+        SymbolNotFound
+            Raised when no equity has ever held the given symbol.
+        MultipleSymbolsFound
+            Raised when no ``as_of_date`` is given and more than one equity
+            has held ``symbol``. This is also raised when ``fuzzy=True`` and
+            there are multiple candidates for the given ``symbol`` on the
+            ``as_of_date``.
+        """
+        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
+
+        return exchange.get_asset(symbol, data_frequency)
+
+    def lifetimes(self, dates, include_start_date):
+        """
+        Compute a DataFrame representing asset lifetimes for the specified date
+        range.
+
+        Parameters
+        ----------
+        dates : pd.DatetimeIndex
+            The dates for which to compute lifetimes.
+        include_start_date : bool
+            Whether or not to count the asset as alive on its start_date.
+
+            This is useful in a backtesting context where `lifetimes` is being
+            used to signify "do I have data for this asset as of the morning of
+            this date?"  For many financial metrics, (e.g. daily close), data
+            isn't available for an asset until the end of the asset's first
+            day.
+
+        Returns
+        -------
+        lifetimes : pd.DataFrame
+            A frame of dtype bool with `dates` as index and an Int64Index of
+            assets as columns.  The value at `lifetimes.loc[date, asset]` will
+            be True iff `asset` existed on `date`.  If `include_start_date` is
+            False, then lifetimes.loc[date, asset] will be false when date ==
+            asset.start_date.
+
+        See Also
+        --------
+        numpy.putmask
+        catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
+        """
+        exchanges = find_exchanges(features=['minuteBundle'])
+        if not exchanges:
+            raise ValueError('exchange with minute bundles not found')
+
+        # TODO: find a way to support multiple exchanges
+        exchange = exchanges[0]
+        # Using a single exchange for now because are not unique for the
+        # same asset in different exchanges. I'd like to avoid binding
+        # pipeline to a single exchange.
+        exchange.init()
+
+        data = []
+        for dt in dates:
+            exists = []
+
+            for asset in exchange.assets:
+                if include_start_date:
+                    condition = (asset.start_date <= dt < asset.end_minute)
+
+                else:
+                    condition = (asset.start_date < dt < asset.end_minute)
+
+                exists.append(condition)
+
+            data.append(exists)
+
+        sids = [asset.sid for asset in exchange.assets]
+        df = pd.DataFrame(data, index=dates, columns=exchange.assets)
+
+        return df

catalyst/exchange/exchange_blotter.py

@@ -1,15 +1,14 @@
-from time import sleep
-
+import numpy as np
 import pandas as pd
-from catalyst.assets._assets import TradingPair
 from logbook import Logger
+from redo import retry
 
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import ExchangeRequestError, \
-    ExchangePortfolioDataError, ExchangeTransactionError
+from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
-from catalyst.finance.order import ORDER_STATUS, Order
+from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
@@ -44,6 +43,11 @@ class TradingPairFeeSchedule(CommissionModel):
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
@@ -57,15 +61,15 @@ class TradingPairFeeSchedule(CommissionModel):
         cost = abs(transaction.amount) * transaction.price
 
         asset = order.asset
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
+        maker, taker = self.get_maker_taker(asset)
 
-        multiplier = maker \
-            if ((order.amount > 0 and order.limit < transaction.price)
-                or (order.amount < 0 and order.limit > transaction.price)) \
-               and order.limit_reached else taker
+        multiplier = taker
+        if order.limit is not None:
+            multiplier = maker \
+                if ((order.amount > 0 and order.limit < transaction.price)
+                    or (order.amount < 0 and order.limit > transaction.price)) \
+                   and order.limit_reached else taker
 
-        # Assuming just the taker fee for now
         fee = cost * multiplier
         return fee
 
@@ -91,7 +95,6 @@ class TradingPairFixedSlippage(SlippageModel):
 
     def simulate(self, data, asset, orders_for_asset):
         self._volume_for_bar = 0
-
         price = data.current(asset, 'close')
 
         dt = data.current_dt
@@ -101,18 +104,20 @@ class TradingPairFixedSlippage(SlippageModel):
 
             order.check_triggers(price, dt)
             if not order.triggered:
-                log.debug('order has not reached the trigger at current '
-                          'price {}'.format(price))
+                log.info(
+                    'order has not reached the trigger at current '
+                    'price {}'.format(price)
+                )
                 continue
 
             execution_price, execution_volume = self.process_order(data, order)
+            if execution_price is not None:
+                transaction = create_transaction(
+                    order, dt, execution_price, execution_volume
+                )
 
-            transaction = create_transaction(
-                order, dt, execution_price, execution_volume
-            )
-
-            self._volume_for_bar += abs(transaction.amount)
-            yield order, transaction
+                self._volume_for_bar += abs(transaction.amount)
+                yield order, transaction
 
     def process_order(self, data, order):
         price = data.current(order.asset, 'close')
@@ -132,6 +137,7 @@ class TradingPairFixedSlippage(SlippageModel):
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
         self.simulate_orders = kwargs.pop('simulate_orders', False)
+        self.attempts = kwargs.pop('attempts', False)
 
         self.exchanges = kwargs.pop('exchanges', None)
         if not self.exchanges:
@@ -151,31 +157,11 @@ class ExchangeBlotter(Blotter):
             TradingPair: TradingPairFeeSchedule()
         }
 
-        self.retry_delay = 5
-        self.retry_check_open_orders = 5
-
-    def exchange_order(self, asset, amount, style=None, attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(
-                asset, amount, style
-            )
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-
-                return self.exchange_order(
-                    asset, amount, style, attempt_index + 1
-                )
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
+    def exchange_order(self, asset, amount, style=None):
+        exchange = self.exchanges[asset.exchange]
+        return exchange.order(
+            asset, amount, style
+        )
 
     @expect_types(asset=TradingPair)
     def order(self, asset, amount, style, order_id=None):
@@ -190,8 +176,13 @@ class ExchangeBlotter(Blotter):
             )
 
         else:
-            order = self.exchange_order(
-                asset, amount, style
+            order = retry(
+                action=self.exchange_order,
+                attempts=self.attempts['order_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.'),
+                args=(asset, amount, style),
             )
 
             self.open_orders[order.asset].append(order)
@@ -217,34 +208,29 @@ class ExchangeBlotter(Blotter):
             for order in self.open_orders[asset]:
                 log.debug('found open order: {}'.format(order.id))
 
-                new_order, executed_price = exchange.get_order(order.id, asset)
-                log.debug(
-                    'got updated order {} {}'.format(
-                        new_order, executed_price
+                transactions = exchange.process_order(order)
+                # This is a temporary measure, we should really update all
+                # trades, not just when the order gets filled. I just think
+                # that this is safer until we have a robust way to track
+                # the trades already processed by the algo. We can't loose
+                # them if the algo shuts down.
+                if transactions and order.open_amount == 0:
+                    avg_price = np.average(
+                        a=[t.price for t in transactions],
+                        weights=[t.amount for t in transactions],
                     )
-                )
-                order.status = new_order.status
-
-                if order.status == ORDER_STATUS.FILLED:
-                    order.commission = new_order.commission
-                    if order.amount != new_order.amount:
-                        log.warn(
-                            'executed order amount {} differs '
-                            'from original'.format(
-                                new_order.amount, order.amount
-                            )
+                    ostatus = 'filled' if order.open_amount == 0 else 'partial'
+                    log.info(
+                        '{} order {} / {}: {}, avg price: {}'.format(
+                            ostatus,
+                            order.id,
+                            asset.symbol,
+                            order.filled,
+                            avg_price,
                         )
-                        order.amount = new_order.amount
-
-                    transaction = Transaction(
-                        asset=order.asset,
-                        amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
-                        price=executed_price,
-                        order_id=order.id,
-                        commission=order.commission
                     )
-                    yield order, transaction
+                    for transaction in transactions:
+                        yield order, transaction
 
                 elif order.status == ORDER_STATUS.CANCELLED:
                     yield order, None
@@ -258,40 +244,31 @@ class ExchangeBlotter(Blotter):
                         )
                     )
 
-    def get_exchange_transactions(self, attempt_index=0):
+    def get_exchange_transactions(self):
         closed_orders = []
         transactions = []
         commissions = []
 
-        try:
-            for order, txn in self.check_open_orders():
-                order.dt = txn.dt
+        for order, txn in self.check_open_orders():
+            order.dt = txn.dt
+            transactions.append(txn)
 
-                transactions.append(txn)
+            if not order.open:
+                closed_orders.append(order)
 
-                if not order.open:
-                    closed_orders.append(order)
-
-            return transactions, commissions, closed_orders
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self.get_exchange_transactions(attempt_index + 1)
-
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
+        return transactions, commissions, closed_orders
 
     def get_transactions(self, bar_data):
         if self.simulate_orders:
             return super(ExchangeBlotter, self).get_transactions(bar_data)
 
         else:
-            return self.get_exchange_transactions()
+            return retry(
+                action=self.get_exchange_transactions,
+                attempts=self.attempts['get_transactions_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn(
+                    'Fetching exchange transactions again.'
+                )
+            )

catalyst/exchange/exchange_bundle.py

@@ -8,30 +8,29 @@ from operator import is_not
 import numpy as np
 import pandas as pd
 import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from pytz import UTC
-from six import itervalues
-
 from catalyst import get_calendar
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
-    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
-from catalyst.exchange.exchange_utils import get_exchange_folder, \
-    save_exchange_symbols, mixin_market_params
+from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
+    get_bcolz_chunk, get_df_from_arrays, get_assets
+from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
+    get_period_label, get_month_start_end, get_year_start_end
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
+    save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
+from logbook import Logger
+from pytz import UTC
+from six import itervalues
 
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 
@@ -233,12 +232,12 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                            name=asset.symbol,
-                            start_dt=asset.start_date.strftime(
-                                DATE_TIME_FORMAT),
-                            end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                            dates=[date.strftime(
-                                DATE_TIME_FORMAT) for date in dates])
+                    name=asset.symbol,
+                    start_dt=asset.start_date.strftime(
+                        DATE_TIME_FORMAT),
+                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                    dates=[date.strftime(
+                        DATE_TIME_FORMAT) for date in dates])
 
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
@@ -287,12 +286,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                        name=asset.symbol,
-                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                        threshold=threshold,
-                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                               for date in dates])
+                name=asset.symbol,
+                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                threshold=threshold,
+                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                       for date in dates])
 
             problems.append(problem)
 
@@ -462,7 +461,7 @@ class ExchangeBundle:
                 (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
-        if end is None or (last_entry is not None and end > last_entry):
+        if last_entry is not None and (end is None or end > last_entry):
             end = last_entry.replace(minute=59, hour=23) \
                 if data_frequency == 'minute' else last_entry
 
@@ -630,8 +629,8 @@ class ExchangeBundle:
                     show_progress,
                     label='Ingesting {frequency} price data on '
                           '{exchange}'.format(
-                            exchange=self.exchange_name,
-                            frequency=data_frequency,
+                        exchange=self.exchange_name,
+                        frequency=data_frequency,
                     )) as it:
                 for chunk in it:
                     problems += self.ingest_ctable(
@@ -668,7 +667,7 @@ class ExchangeBundle:
 
         if self.exchange is None:
             # Avoid circular dependencies
-            from catalyst.exchange.factory import get_exchange
+            from catalyst.exchange.utils.factory import get_exchange
             self.exchange = get_exchange(self.exchange_name)
 
         problems = []
@@ -681,6 +680,7 @@ class ExchangeBundle:
                 last_traded=np.object_,
                 open=np.float64,
                 high=np.float64,
+                low=np.float64,
                 close=np.float64,
                 volume=np.float64
             ),
@@ -730,7 +730,7 @@ class ExchangeBundle:
                     if data_frequency == 'minute' else asset_def['end_minute']
 
             else:
-                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+                params['symbol'] = get_catalyst_symbol(market)
 
                 params['end_daily'] = end_dt \
                     if data_frequency == 'daily' else 'N/A'
@@ -755,9 +755,10 @@ class ExchangeBundle:
         )
 
         for symbol in assets:
+            # here the symbol is the market['id']
             asset = assets[symbol]
             ohlcv_df = df.loc[
-                (df.index.get_level_values(0) == symbol)
+                (df.index.get_level_values(0) == asset.symbol)
             ]  # type: pd.DataFrame
             ohlcv_df.index = ohlcv_df.index.droplevel(0)
 
@@ -805,7 +806,7 @@ class ExchangeBundle:
         else:
             if self.exchange is None:
                 # Avoid circular dependencies
-                from catalyst.exchange.factory import get_exchange
+                from catalyst.exchange.utils.factory import get_exchange
                 self.exchange = get_exchange(self.exchange_name)
 
             assets = get_assets(
@@ -963,15 +964,15 @@ class ExchangeBundle:
                                   data_frequency,
                                   trailing_bar_count=None,
                                   reset_reader=False):
+        if trailing_bar_count:
+            delta = get_delta(trailing_bar_count, data_frequency)
+            end_dt += delta
+
         start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
         start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
 
-        if trailing_bar_count:
-            delta = get_delta(trailing_bar_count, data_frequency)
-            end_dt += delta
-
         # This is an attempt to resolve some caching with the reader
         # when auto-ingesting data.
         # TODO: needs more work

catalyst/exchange/exchange_data_portal.py

@@ -1,31 +1,29 @@
 import abc
-from time import sleep
 
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangeBarDataError,
     PricingDataNotLoadedError)
-from catalyst.exchange.exchange_utils import get_frequency, \
-    resample_history_df, group_assets_by_exchange
+from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
+from catalyst.exchange.utils.datetime_utils import get_frequency
+from logbook import Logger
+from redo import retry
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
-
-        # TODO: put somewhere accessible by each algo
-        self.retry_get_history_window = 5
-        self.retry_get_spot_value = 5
-        self.retry_delay = 5
+        self.attempts = dict(
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
 
         super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
 
@@ -36,33 +34,14 @@ class DataPortalExchangeBase(DataPortal):
                             frequency,
                             field,
                             data_frequency,
-                            ffill=True,
-                            attempt_index=0):
-        try:
-            exchange_assets = group_assets_by_exchange(assets)
-            if len(exchange_assets) > 1:
-                df_list = []
-                for exchange_name in exchange_assets:
-                    assets = exchange_assets[exchange_name]
+                            ffill=True):
+        exchange_assets = group_assets_by_exchange(assets)
+        if len(exchange_assets) > 1:
+            df_list = []
+            for exchange_name in exchange_assets:
+                assets = exchange_assets[exchange_name]
 
-                    df_exchange = self.get_exchange_history_window(
-                        exchange_name,
-                        assets,
-                        end_dt,
-                        bar_count,
-                        frequency,
-                        field,
-                        data_frequency,
-                        ffill)
-
-                    df_list.append(df_exchange)
-
-                # Merging the values values of each exchange
-                return pd.concat(df_list)
-
-            else:
-                exchange_name = list(exchange_assets.keys())[0]
-                return self.get_exchange_history_window(
+                df_exchange = self.get_exchange_history_window(
                     exchange_name,
                     assets,
                     end_dt,
@@ -72,26 +51,22 @@ class DataPortalExchangeBase(DataPortal):
                     data_frequency,
                     ffill)
 
-        except ExchangeRequestError as e:
-            log.warn(
-                'get history attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_history_window:
-                sleep(self.retry_delay)
-                return self._get_history_window(assets,
-                                                end_dt,
-                                                bar_count,
-                                                frequency,
-                                                field,
-                                                data_frequency,
-                                                ffill,
-                                                attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='history',
-                    attempts=attempt_index,
-                    error=e
-                )
+                df_list.append(df_exchange)
+
+            # Merging the values values of each exchange
+            return pd.concat(df_list)
+
+        else:
+            exchange_name = list(exchange_assets.keys())[0]
+            return self.get_exchange_history_window(
+                exchange_name,
+                assets,
+                end_dt,
+                bar_count,
+                frequency,
+                field,
+                data_frequency,
+                ffill)
 
     def get_history_window(self,
                            assets,
@@ -105,13 +80,19 @@ class DataPortalExchangeBase(DataPortal):
         if field == 'price':
             field = 'close'
 
-        return self._get_history_window(assets,
-                                        end_dt,
-                                        bar_count,
-                                        frequency,
-                                        field,
-                                        data_frequency,
-                                        ffill)
+        return retry(
+            action=self._get_history_window,
+            attempts=self.attempts['get_history_window_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching history again.'),
+            args=(assets,
+                  end_dt,
+                  bar_count,
+                  frequency,
+                  field,
+                  data_frequency,
+                  ffill))
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
@@ -125,69 +106,58 @@ class DataPortalExchangeBase(DataPortal):
                                     ffill=True):
         pass
 
-    def _get_spot_value(self, assets, field, dt, data_frequency,
-                        attempt_index=0):
-        try:
-            if isinstance(assets, TradingPair):
-                spot_values = self.get_exchange_spot_value(
-                    assets.exchange, [assets], field, dt, data_frequency)
+    def _get_spot_value(self, assets, field, dt, data_frequency):
+        if isinstance(assets, TradingPair):
+            spot_values = self.get_exchange_spot_value(
+                assets.exchange, [assets], field, dt, data_frequency)
 
-                if not spot_values:
-                    return np.nan
+            if not spot_values:
+                return np.nan
 
-                return spot_values[0]
+            return spot_values[0]
+
+        else:
+            exchange_assets = dict()
+            for asset in assets:
+                if asset.exchange not in exchange_assets:
+                    exchange_assets[asset.exchange] = list()
+
+                exchange_assets[asset.exchange].append(asset)
+
+            if len(list(exchange_assets.keys())) == 1:
+                exchange_name = list(exchange_assets.keys())[0]
+                return self.get_exchange_spot_value(
+                    exchange_name, assets, field, dt, data_frequency)
 
             else:
-                exchange_assets = dict()
-                for asset in assets:
-                    if asset.exchange not in exchange_assets:
-                        exchange_assets[asset.exchange] = list()
+                spot_values = []
+                for exchange_name in exchange_assets:
+                    assets = exchange_assets[exchange_name]
+                    exchange_spot_values = self.get_exchange_spot_value(
+                        exchange_name,
+                        assets,
+                        field,
+                        dt,
+                        data_frequency
+                    )
+                    if len(assets) == 1:
+                        spot_values.append(exchange_spot_values)
+                    else:
+                        spot_values += exchange_spot_values
 
-                    exchange_assets[asset.exchange].append(asset)
-
-                if len(list(exchange_assets.keys())) == 1:
-                    exchange_name = list(exchange_assets.keys())[0]
-                    return self.get_exchange_spot_value(
-                        exchange_name, assets, field, dt, data_frequency)
-
-                else:
-                    spot_values = []
-                    for exchange_name in exchange_assets:
-                        assets = exchange_assets[exchange_name]
-                        exchange_spot_values = self.get_exchange_spot_value(
-                            exchange_name,
-                            assets,
-                            field,
-                            dt,
-                            data_frequency
-                        )
-                        if len(assets) == 1:
-                            spot_values.append(exchange_spot_values)
-                        else:
-                            spot_values += exchange_spot_values
-
-                    return spot_values
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'get spot value attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_spot_value:
-                sleep(self.retry_delay)
-                return self._get_spot_value(assets, field, dt, data_frequency,
-                                            attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='spot',
-                    attempts=attempt_index,
-                    error=e
-                )
+                return spot_values
 
     def get_spot_value(self, assets, field, dt, data_frequency):
         if field == 'price':
             field = 'close'
 
-        return self._get_spot_value(assets, field, dt, data_frequency)
+        return retry(
+            action=self._get_spot_value,
+            attempts=self.attempts['get_spot_value_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching spot value again.'),
+            args=(assets, field, dt, data_frequency))
 
     @abc.abstractmethod
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -321,6 +291,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         DataFrame
 
         """
+        # TODO: verify that the exchange supports the timeframe
         bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
@@ -339,7 +310,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
-            trailing_bar_count=trailing_bar_count
+            trailing_bar_count=trailing_bar_count,
         )
 
         df = resample_history_df(pd.DataFrame(series), freq, field)

catalyst/exchange/exchange_errors.py

@@ -100,6 +100,19 @@ class InvalidHistoryFrequencyError(ZiplineError):
     ).strip()
 
 
+class UnsupportedHistoryFrequencyError(ZiplineError):
+    msg = (
+        '{exchange} does not support candle frequency {freq}, please choose '
+        'from: {freqs}.'
+    ).strip()
+
+
+class InvalidHistoryTimeframeError(ZiplineError):
+    msg = (
+        'CCXT timeframe {timeframe} not supported by the exchange.'
+    ).strip()
+
+
 class MismatchingFrequencyError(ZiplineError):
     msg = (
         'Bar aggregate frequency {frequency} not compatible with '
@@ -162,8 +175,8 @@ class SidHashError(ZiplineError):
 
 class BaseCurrencyNotFoundError(ZiplineError):
     msg = (
-        'Algorithm base currency {base_currency} not found in exchange '
-        '{exchange}.'
+        'Algorithm base currency {base_currency} not found in account '
+        'balances on {exchange}: {balances}'
     ).strip()
 
 
@@ -226,16 +239,20 @@ class PricingDataValueError(ZiplineError):
 
 
 class DataCorruptionError(ZiplineError):
-    msg = ('Unable to validate data for {exchange} {symbols} in date range '
-           '[{start_dt} - {end_dt}]. The data is either corrupted or '
-           'unavailable. Please try deleting this bundle:'
-           '\n`catalyst clean-exchange -x {exchange}\n'
-           'Then, ingest the data again. Please contact the Catalyst team if '
-           'the issue persists.').strip()
+    msg = (
+        'Unable to validate data for {exchange} {symbols} in date range '
+        '[{start_dt} - {end_dt}]. The data is either corrupted or '
+        'unavailable. Please try deleting this bundle:'
+        '\n`catalyst clean-exchange -x {exchange}\n'
+        'Then, ingest the data again. Please contact the Catalyst team if '
+        'the issue persists.'
+    ).strip()
 
 
 class ApiCandlesError(ZiplineError):
-    msg = ('Unable to fetch candles from the remote API: {error}.').strip()
+    msg = (
+        'Unable to fetch candles from the remote API: {error}.'
+    ).strip()
 
 
 class NoDataAvailableOnExchange(ZiplineError):
@@ -248,13 +265,16 @@ class NoDataAvailableOnExchange(ZiplineError):
 
 
 class NoValueForField(ZiplineError):
-    msg = ('Value not found for field: {field}.').strip()
+    msg = (
+        'Value not found for field: {field}.'
+    ).strip()
 
 
 class OrderTypeNotSupported(ZiplineError):
     msg = (
-        'Order type `{order_type}` not currencly supported by Catalyst. '
-        'Please use `limit` or `market` orders only.').strip()
+        'Order type `{order_type}` not currency supported by Catalyst. '
+        'Please use `limit` or `market` orders only.'
+    ).strip()
 
 
 class NotEnoughCapitalError(ZiplineError):
@@ -262,10 +282,43 @@ class NotEnoughCapitalError(ZiplineError):
         'Not enough capital on exchange {exchange} for trading. Each '
         'exchange should contain at least as much {base_currency} '
         'as the specified `capital_base`. The current balance {balance} is '
-        'lower than the `capital_base`: {capital_base}').strip()
+        'lower than the `capital_base`: {capital_base}'
+    ).strip()
+
+
+class NotEnoughCashError(ZiplineError):
+    msg = (
+        'Total {currency} amount on {exchange} is lower than the cash '
+        'reserved for this algo: {free} < {cash}. While trades can be made on '
+        'the exchange accounts outside of the algo, exchange must have enough '
+        'free {currency} to cover the algo cash.'
+    ).strip()
+
 
 class LastCandleTooEarlyError(ZiplineError):
     msg = (
         'The trade date of the last candle {last_traded} is before the '
         'specified end date minus one candle {end_dt}. Please verify how '
-        '{exchange} calculates the start date of OHLCV candles.').strip()
+        '{exchange} calculates the start date of OHLCV candles.'
+    ).strip()
+
+
+class TickerNotFoundError(ZiplineError):
+    msg = (
+        'Unable to fetch ticker for {symbol} on {exchange}.'
+    ).strip()
+
+
+class BalanceNotFoundError(ZiplineError):
+    msg = (
+        '{currency} not found in account balance on {exchange}: {balances}.'
+    ).strip()
+
+
+class BalanceTooLowError(ZiplineError):
+    msg = (
+        'Balance for {currency} on {exchange} too low: {free} < {amount}. '
+        'Positions have likely been sold outside of this algorithm. Please '
+        'add positions to hold a free amount greater than {amount}, or clean '
+        'the state of this algo and restart.'
+    ).strip()

catalyst/exchange/exchange_portfolio.py

@@ -1,8 +1,7 @@
 import numpy as np
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from logbook import Logger
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 

catalyst/exchange/exchange_pricing_loader.py

@@ -0,0 +1,177 @@
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from catalyst.constants import LOG_LEVEL
+from catalyst.data.us_equity_pricing import BcolzDailyBarReader
+from catalyst.errors import NoFurtherDataError
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.lib.adjusted_array import AdjustedArray
+from catalyst.pipeline.data import DataSet, Column
+from catalyst.pipeline.loaders.base import PipelineLoader
+from catalyst.utils.calendars import get_calendar
+from catalyst.utils.numpy_utils import float64_dtype
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
+
+UINT32_MAX = iinfo(uint32).max
+
+log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
+
+
+class TradingPairPricing(DataSet):
+    """
+    Dataset representing daily trading prices and volumes.
+    """
+    open = Column(float64_dtype)
+    high = Column(float64_dtype)
+    low = Column(float64_dtype)
+    close = Column(float64_dtype)
+    volume = Column(float64_dtype)
+
+
+class ExchangePricingLoader(PipelineLoader):
+    """
+    PipelineLoader for Crypto Pricing data
+
+    Delegates loading of baselines and adjustments.
+    """
+
+    def __init__(self, data_frequency):
+
+        cal = get_calendar('OPEN')
+
+        if data_frequency == 'daily':
+            reader = None
+            all_sessions = cal.all_sessions
+
+        elif data_frequency == 'minute':
+            reader = None
+            all_sessions = cal.all_minutes
+
+        else:
+            raise ValueError(
+                'Invalid data frequency: {}'.format(data_frequency)
+            )
+
+        self.data_frequency = data_frequency
+        self.raw_price_loader = reader
+        self._columns = TradingPairPricing.columns
+        self._all_sessions = all_sessions
+
+    @classmethod
+    def from_files(cls, pricing_path):
+        """
+        Create a loader from a bcolz equity pricing dir and a SQLite
+        adjustments path.
+
+        Parameters
+        ----------
+        pricing_path : str
+            Path to a bcolz directory written by a BcolzDailyBarWriter.
+        """
+        return cls(
+            BcolzDailyBarReader(pricing_path),
+        )
+
+    def load_adjusted_array(self, columns, dates, assets, mask):
+        # load_adjusted_array is called with dates on which the user's algo
+        # will be shown data, which means we need to return the data that would
+        # be known at the start of each date.  We assume that the latest data
+        # known on day N is the data from day (N - 1), so we shift all query
+        # dates back by a day.
+        start_date, end_date = _shift_dates(
+            self._all_sessions, dates[0], dates[-1], shift=1,
+        )
+        colnames = [c.name for c in columns]
+
+        if len(assets) == 0:
+            raise ValueError(
+                'Pipeline cannot load data with eligible assets.'
+            )
+
+        exchange_names = []
+        for asset in assets:
+            if asset.exchange not in exchange_names:
+                exchange_names.append(asset.exchange)
+
+        exchange = get_exchange(exchange_names[0])
+        reader = exchange.bundle.get_reader(self.data_frequency)
+
+        raw_arrays = reader.load_raw_arrays(
+            colnames,
+            start_date,
+            end_date,
+            assets,
+        )
+
+        out = {}
+        for c, c_raw in zip(columns, raw_arrays):
+            out[c] = AdjustedArray(
+                c_raw.astype(c.dtype),
+                mask,
+                {},
+                c.missing_value,
+            )
+        return out
+
+    @property
+    def columns(self):
+        return self._columns
+
+
+def _shift_dates(dates, start_date, end_date, shift):
+    try:
+        start = dates.get_loc(start_date)
+    except KeyError:
+        if start_date < dates[0]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requested data starting on {query_start}, "
+                    "but first known date is {calendar_start}"
+                ).format(
+                    query_start=str(start_date),
+                    calendar_start=str(dates[0]),
+                )
+            )
+        else:
+            raise ValueError("Query start %s not in calendar" % start_date)
+
+    # Make sure that shifting doesn't push us out of the calendar.
+    if start < shift:
+        raise NoFurtherDataError(
+            msg=(
+                "Pipeline Query requested data from {shift}"
+                " days before {query_start}, but first known date is only "
+                "{start} days earlier."
+            ).format(shift=shift, query_start=start_date, start=start),
+        )
+
+    try:
+        end = dates.get_loc(end_date)
+    except KeyError:
+        if end_date > dates[-1]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requesting data up to {query_end}, "
+                    "but last known date is {calendar_end}"
+                ).format(
+                    query_end=end_date,
+                    calendar_end=dates[-1],
+                )
+            )
+        else:
+            raise ValueError("Query end %s not in calendar" % end_date)
+    return dates[start - shift], dates[end - shift]

catalyst/exchange/factory.py

@@ -1,34 +0,0 @@
-import os
-
-from catalyst.exchange.ccxt.ccxt_exchange import CCXT
-from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_exchange_folder
-
-
-def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
-    exchange_auth = get_exchange_auth(exchange_name)
-
-    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
-    if must_authenticate and not has_auth:
-        raise ExchangeAuthEmpty(
-            exchange=exchange_name.title(),
-            filename=os.path.join(
-                get_exchange_folder(exchange_name), 'auth.json'
-            )
-        )
-
-    return CCXT(
-        exchange_name=exchange_name,
-        key=exchange_auth['key'],
-        secret=exchange_auth['secret'],
-        base_currency=base_currency,
-    )
-
-
-def get_exchanges(exchange_names):
-    exchanges = dict()
-    for exchange_name in exchange_names:
-        exchanges[exchange_name] = get_exchange(exchange_name)
-
-    return exchanges

catalyst/exchange/live_graph_clock.py

@@ -1,14 +1,12 @@
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.stats_utils import prepare_stats
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
@@ -38,177 +36,23 @@ class LiveGraphClock(object):
     the exchange and the live trading machine's clock. It's not used currently.
     """
 
-    def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
-
-        global mdates, plt  # TODO: Could be cleaner
-        import matplotlib.dates as mdates
-        from matplotlib import pyplot as plt
-        from matplotlib import style
+    def __init__(self, sessions, context, callback=None,
+                 time_skew=pd.Timedelta('0s')):
 
         self.sessions = sessions
         self.time_skew = time_skew
         self._last_emit = None
         self._before_trading_start_bar_yielded = True
         self.context = context
-        self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
-
-        style.use('dark_background')
-
-        fig = plt.figure()
-        fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
-            self.context.algo_namespace))
-
-        self.ax_pnl = fig.add_subplot(311)
-
-        self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
-
-        self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
-
-        if len(context.minute_stats) > 0:
-            self.draw_pnl()
-            self.draw_custom_signals()
-            self.draw_exposure()
-
-        # rotates and right aligns the x labels, and moves the bottom of the
-        # axes up to make room for them
-        fig.autofmt_xdate()
-        fig.subplots_adjust(hspace=0.5)
-
-        plt.tight_layout()
-        plt.ion()
-        plt.show()
-
-    def format_ax(self, ax):
-        """
-        Trying to assign reasonable parameters to the time axis.
-
-        Parameters
-        ----------
-        ax:
-
-        """
-        # TODO: room for improvement
-        ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-        ax.xaxis.set_major_formatter(self.fmt)
-
-        locator = mdates.HourLocator(interval=4)
-        locator.MAXTICKS = 5000
-        ax.xaxis.set_minor_locator(locator)
-
-        datemin = pd.Timestamp.utcnow()
-        ax.set_xlim(datemin)
-
-        ax.grid(True)
-
-    def set_legend(self, ax):
-        """
-        Set legend on the chart.
-
-        Parameters
-        ----------
-        ax
-
-        """
-        ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-    def draw_pnl(self):
-        """
-        Draw p&l line on the chart.
-
-        """
-        ax = self.ax_pnl
-        df = self.context.pnl_stats
-
-        ax.clear()
-        ax.set_title('Performance')
-        ax.plot(df.index, df['performance'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Performance'
-                )
-
-        def perc(val):
-            return '{:2f}'.format(val)
-
-        ax.format_ydata = perc
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_custom_signals(self):
-        """
-        Draw custom signals on the chart.
-
-        """
-        ax = self.ax_custom_signals
-        df = self.context.custom_signals_stats
-
-        colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-        ax.clear()
-        ax.set_title('Custom Signals')
-        for index, column in enumerate(df.columns.values.tolist()):
-            ax.plot(df.index, df[column], '-',
-                    color=colors[index],
-                    linewidth=1.0,
-                    label=column
-                    )
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_exposure(self):
-        """
-        Draw exposure line on the chart.
-
-        """
-        ax = self.ax_exposure
-        context = self.context
-        df = context.exposure_stats
-
-        # TODO: list exchanges in graph
-        base_currency = None
-        positions = []
-        for exchange_name in context.exchanges:
-            exchange = context.exchanges[exchange_name]
-
-            if not base_currency:
-                base_currency = exchange.base_currency
-            elif base_currency != exchange.base_currency:
-                raise MismatchingBaseCurrenciesExchanges(
-                    base_currency=base_currency,
-                    exchange_name=exchange.name,
-                    exchange_currency=exchange.base_currency
-                )
-
-            positions += exchange.portfolio.positions
-
-        ax.clear()
-        ax.set_title('Exposure')
-        ax.plot(df.index, df['base_currency'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Base Currency: {}'.format(base_currency.upper())
-                )
-
-        symbols = []
-        for position in positions:
-            symbols.append(position.symbol)
-
-        ax.plot(df.index, df['long_exposure'], '-',
-                color='blue',
-                linewidth=1.0,
-                label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-        self.set_legend(ax)
-        self.format_ax(ax)
+        self.callback = callback
 
     def __iter__(self):
+        from matplotlib import pyplot as plt
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:
             current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
+            current_minute = current_time.floor('1T')
 
             if self._last_emit is None or current_minute > self._last_emit:
                 log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -216,14 +60,11 @@ class LiveGraphClock(object):
                 self._last_emit = current_minute
                 yield current_minute, BAR
 
-                try:
-                    self.draw_pnl()
-                    self.draw_custom_signals()
-                    self.draw_exposure()
-
-                    plt.draw()
-                except Exception as e:
-                    log.warn('Unable to update the graph: {}'.format(e))
+                recorded_cols = list(self.context.recorded_vars.keys())
+                df, _ = prepare_stats(
+                    self.context.frame_stats, recorded_cols=recorded_cols
+                )
+                self.callback(self.context, df)
 
             else:
                 # I can't use the "animate" reactive approach here because

catalyst/exchange/poloniex/poloniex.py

@@ -1,661 +0,0 @@
-import json
-import time
-from collections import defaultdict
-
-import numpy as np
-import pandas as pd
-import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-# import six
-from six import iteritems
-
-from catalyst.constants import LOG_LEVEL
-# from websocket import create_connection
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle,
-    OrphanOrderError,
-    OrphanOrderReverseError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.protocol import Account
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Poloniex', level=LOG_LEVEL)
-
-
-@deprecated
-class Poloniex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret)
-        self.name = 'poloniex'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-        self.transactions = defaultdict(list)
-
-        self.num_candles_limit = 2000
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from the Exchange order dictionary
-        :param order_status:
-        :return: Order
-        """
-        # if order_status['is_cancelled']:
-        #    status = ORDER_STATUS.CANCELLED
-        # elif not order_status['is_live']:
-        #    log.info('found executed order {}'.format(order_status))
-        #    status = ORDER_STATUS.FILLED
-        # else:
-        status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['amount'])
-        # filled = float(order_status['executed_amount'])
-        filled = None
-
-        if order_status['type'] == 'sell':
-            amount = -amount
-            # filled = -filled
-
-        price = float(order_status['rate'])
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        # if order_type.endswith('limit'):
-        #    limit_price = price
-        # elif order_type.endswith('stop'):
-        #    stop_price = price
-
-        # executed_price = float(order_status['avg_execution_price'])
-        executed_price = price
-
-        # TODO: Set Poloniex comission
-        commission = None
-
-        # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date=pytz.utc.localize(date)
-        date = None
-
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            # No such field in Poloniex
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['orderNumber']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        balances = self.api.returnbalances()
-        try:
-            log.debug('retrieving wallets balances')
-        except Exception as e:
-            log.debug(e)
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['error'])
-            )
-
-        std_balances = dict()
-        for (key, value) in iteritems(balances):
-            currency = key.lower()
-            std_balances[currency] = float(value)
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Poloniex
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '5m', '15m', '30m', '2h', '4h', '1D'
-        """
-
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T' and (bar_count == 1 or bar_count is None):
-            # TODO: use the order book instead
-            # We use the 5m to fetch the last bar
-            frequency = 300
-        elif freq == '5T':
-            frequency = 300
-        elif freq == '15T':
-            frequency = 900
-        elif freq == '30T':
-            frequency = 1800
-        elif freq == '120T':
-            frequency = 7200
-        elif freq == '240T':
-            frequency = 14400
-        elif freq == '1D':
-            frequency = 86400
-        else:
-            # Poloniex does not offer 1m data candles
-            # It is likely to error out there frequently
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-
-        for asset in asset_list:
-            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
-            end = int(delta.total_seconds())
-
-            if bar_count is None:
-                start = end - 2 * frequency
-            else:
-                start = end - bar_count * frequency
-
-            try:
-                response = self.api.returnchartdata(
-                    self.get_symbol(asset), frequency, start, end
-                )
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-                ohlc = dict(
-                    open=np.float64(candle['open']),
-                    high=np.float64(candle['high']),
-                    low=np.float64(candle['low']),
-                    close=np.float64(candle['close']),
-                    volume=np.float64(candle['volume']),
-                    price=np.float64(candle['close']),
-                    last_traded=last_traded
-                )
-
-                return ohlc
-
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(response[0])
-            else:
-                ohlc_bars = []
-                for candle in response:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-
-        if (isinstance(style, ExchangeLimitOrder)
-                or isinstance(style, ExchangeStopLimitOrder)):
-            if isinstance(style, ExchangeStopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-
-            try:
-                if (is_buy):
-                    response = self.api.buy(exchange_symbol, amount, price)
-                else:
-                    response = self.api.sell(exchange_symbol, -amount, price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            date = pd.Timestamp.utcnow()
-
-            if ('orderNumber' in response):
-                order_id = str(response['orderNumber'])
-                order = Order(
-                    dt=date,
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                log.warn(
-                    '{} order failed: {}'.format('buy' if is_buy else 'sell',
-                                                 response['error']))
-                return None
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset='all'):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not 'all', return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If 'all' is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-
-        return self.portfolio.open_orders
-
-        """
-        TODO: Why going to the exchange if we already have this info locally?
-              And why creating all these Orders if we later discard them?
-        """
-
-        try:
-            if (asset == 'all'):
-                response = self.api.returnopenorders('all')
-            else:
-                response = self.api.returnopenorders(self.get_symbol(asset))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    response['message'])
-            )
-
-        print(self.portfolio.open_orders)
-
-        # TODO: Need to handle openOrders for 'all'
-        orders = list()
-        for order_status in response:
-            # will Throw error b/c Polo doesn't track order['symbol']
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-
-        try:
-            order = self._portfolio.open_orders[order_id]
-        except Exception as e:
-            raise OrphanOrderError(order_id=order_id, exchange=self.name)
-
-        return order
-
-        # TODO: Need to decide whether we fetch orders locally or from exchnage
-        # The code below is ignored
-
-        try:
-            response = self.api.returnopenorders(self.get_symbol(order.sid))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        for o in response:
-            if (int(o['orderNumber']) == int(order_id)):
-                return order
-
-        return None
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-
-        if (isinstance(order_param, Order)):
-            order = order_param
-        else:
-            order = self._portfolio.open_orders[order_param]
-
-        try:
-            response = self.api.cancelorder(order.id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} '
-                '{error}.'.format(
-                    order_id=order.id,
-                    exchange=self.name,
-                    error=response['error']
-                ))
-
-            # raise OrderCancelError(
-            #    order_id=order.id,
-            #    exchange=self.name,
-            #    error=response['error']
-            # )
-
-        self.portfolio.remove_order(order)
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self.get_symbols(assets)
-
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            response = self.api.returnticker()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response['error'])
-            )
-
-        ticks = dict()
-
-        for index, symbol in enumerate(symbols):
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=float(response[symbol]['highestBid']),
-                ask=float(response[symbol]['lowestAsk']),
-                last_price=float(response[symbol]['last']),
-                low=float(response[symbol]['lowestAsk']),
-                # TODO: Polo does not provide low
-                high=float(response[symbol]['highestBid']),
-                # TODO: Polo does not provide high
-                volume=float(response[symbol]['baseVolume']),
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self.api.returnticker()
-
-        for exchange_symbol in response:
-            base, market = self.sanitize_curency_symbol(exchange_symbol).split(
-                '_')
-            symbol = '{market}_{base}'.format(market=market, base=base)
-
-            if (source_dates):
-                start_date = self.get_symbol_start_date(exchange_symbol)
-            else:
-                try:
-                    start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-        try:
-            r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
-                '2010-1-1').value // 10 ** 9)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
-
-    def check_open_orders(self):
-        """
-        Need to override this function for Poloniex:
-
-        Loop through the list of open orders in the Portfolio object.
-        Check if any transactions have been executed:
-            If so, create a transaction and apply to the Portfolio.
-        Check if the order is still open:
-            If not, remove it from open orders
-
-        :return:
-        transactions: Transaction[]
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-
-                order = self._portfolio.open_orders[order_id]
-                log.debug('found open order: {}'.format(order_id))
-
-                try:
-                    order_open = self.get_order(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if (order_open):
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta)
-                    )
-
-                try:
-                    response = self.api.returnordertrades(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if ('error' in response):
-                    if (not order_open):
-                        raise OrphanOrderReverseError(order_id=order_id,
-                                                      exchange=self.name)
-                else:
-                    for tx in response:
-                        """
-                        We maintain a list of dictionaries of transactions that
-                        correspond to partially filled orders, indexed by
-                        order_id. Every time we query executed transactions
-                        from the exchange, we check if we had that transaction
-                        for that order already. If not, we process it.
-
-                        When an order if fully filled, we flush the dict of
-                        transactions associated with that order.
-                        """
-                        if (not filter(
-                                lambda item: item['order_id'] == tx['tradeID'],
-                                self.transactions[order_id])):
-                            log.debug(
-                                'Got new transaction for order {}: amount {}, '
-                                'price {}'.format(
-                                    order_id, tx['amount'], tx['rate']))
-                            tx['amount'] = float(tx['amount'])
-                            if (tx['type'] == 'sell'):
-                                tx['amount'] = -tx['amount']
-                            transaction = Transaction(
-                                asset=order.asset,
-                                amount=tx['amount'],
-                                dt=pd.to_datetime(tx['date'], utc=True),
-                                price=float(tx['rate']),
-                                order_id=tx['tradeID'],
-                                # it's a misnomer, but keep for compatibility
-                                commission=float(tx['fee'])
-                            )
-                            self.transactions[order_id].append(transaction)
-                            self.portfolio.execute_transaction(transaction)
-                            transactions.append(transaction)
-
-                    if (not order_open):
-                        """
-                            Since transactions have been executed individually
-                            the only thing left to do is remove them from list
-                            of open_orders
-                        """
-                        del self.portfolio.open_orders[order_id]
-                        del self.transactions[order_id]
-
-        return transactions
-
-    def get_orderbook(self, asset, order_type='all'):
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.returnOrderBook(market=exchange_symbol)
-
-        result = dict()
-        for order_type in data:
-            # TODO: filter by type
-            if order_type != 'asks' and order_type != 'bids':
-                continue
-
-            result[order_type] = []
-            for entry in data[order_type]:
-                if len(entry) == 2:
-                    result[order_type].append(
-                        dict(
-                            rate=float(entry[0]),
-                            quantity=float(entry[1])
-                        )
-                    )
-        return result

catalyst/exchange/poloniex/poloniex_api.py

@@ -1,212 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-from six.moves import urllib
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-urlopen = urllib.request.urlopen
-
-
-class Poloniex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-
-        self.max_requests_per_second = 6
-        self.request_cpt = dict()
-
-        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                       'returnTradeHistory', 'returnChartData',
-                       'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances', 'returnCompleteBalances',
-                        'returnDepositAddresses',
-                        'generateNewAddress', 'returnDepositsWithdrawals',
-                        'returnOpenOrders',
-                        'returnTradeHistory', 'returnOrderTrades',
-                        'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo',
-                        'returnAvailableAccountBalances',
-                        'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary', 'marginBuy',
-                        'marginSell',
-                        'getMarginPosition', 'closeMarginPosition',
-                        'createLoanOffer',
-                        'cancelLoanOffer', 'returnOpenLoanOffers',
-                        'returnActiveLoans',
-                        'returnLendingHistory', 'toggleAutoRenew']
-
-    def ask_request(self):
-        """
-        Asks permission to issue a request to the exchange.
-        The primary purpose is to avoid hitting rate limits.
-
-        The application will pause if the maximum requests per minute
-        permitted by the exchange is exceeded.
-
-        :return boolean:
-
-        """
-        now = time.time()
-        if not self.request_cpt:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        cpt_date = list(self.request_cpt.keys())[0]
-        cpt = self.request_cpt[cpt_date]
-
-        if now > cpt_date + 1:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        if cpt >= self.max_requests_per_second:
-
-            time.sleep(1)
-
-            now = time.time()
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-        else:
-            self.request_cpt[cpt_date] += 1
-
-    def query(self, method, req={}):
-
-        if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + \
-                  urllib.parse.urlencode(req)
-            headers = {}
-            post_data = None
-        elif method in self.trading:
-            url = 'https://poloniex.com/tradingApi'
-            req['command'] = method
-            req['nonce'] = int(time.time() * 1000)
-            post_data = urllib.parse.urlencode(req)
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 post_data.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'Sign': signature, 'Key': self.key}
-
-            post_data = post_data.encode('utf-8')
-        else:
-            raise ValueError(
-                'Method "' + method + '" not found in neither the Public API '
-                                      'or Trading API endpoints'
-            )
-
-        self.ask_request()
-        req = urllib.request.Request(
-            url,
-            data=post_data,
-            headers=headers,
-        )
-        resource = urlopen(req, context=ssl._create_unverified_context())
-        content = resource.read().decode('utf-8')
-        return json.loads(content)
-
-    def returnticker(self):
-        return self.query('returnTicker', {})
-
-    def return24volume(self):
-        return self.query('return24Volume', {})
-
-    def returnOrderBook(self, market='all'):
-        return self.query('returnOrderBook', {'currencyPair': market})
-
-    def returntradehistory(self, market, start=None, end=None):
-        if (start is not None and end is not None):
-            return self.query('returntradehistory',
-                              {'currencyPair': market, 'start': start,
-                               'end': end})
-        else:
-            return self.query('returntradehistory', {'currencyPair': market})
-
-    def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData',
-                          {'currencyPair': market, 'period': period,
-                           'start': start, 'end': end})
-
-    def returncurrencies(self):
-        return self.query('returnCurrencies', {})
-
-    def returnloadorders(self, market):
-        return self.query('returnLoanOrders', {'currency': market})
-
-    def returnbalances(self):
-        return self.query('returnBalances')
-
-    def returncompletebalances(self, account):
-        if (account):
-            return self.query('returnCompleteBalances', {'account': account})
-        else:
-            return self.query('returnCompleteBalances')
-
-    def returndepositaddresses(self):
-        return self.query('returnDepositAddresses')
-
-    def generatenewaddress(self, currency):
-        return self.query('generateNewAddress', {'currency': currency})
-
-    def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals',
-                          {'start': start, 'end': end})
-
-    def returnopenorders(self, market):
-        return self.query('returnOpenOrders', {'currencyPair': market})
-
-    def returnordertrades(self, ordernumber):
-        return self.query('returnOrderTrades', {'orderNumber': ordernumber})
-
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-            postonly=0):
-        if (fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'postOnly': postonly, })
-        else:
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount, })
-
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-             postonly=0):
-        if (fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'postOnly': postonly, })
-        else:
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount, })
-
-    def cancelorder(self, ordernumber):
-        return self.query('cancelOrder', {'orderNumber': ordernumber})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'amount': quantity,
-                           'address': address})
-
-    def returnfeeinfo(self):
-        return self.query('returnFeeInfo')

catalyst/exchange/simple_clock.py

@@ -14,14 +14,13 @@
 from time import sleep
 
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 

catalyst/exchange/utils/bundle_utils.py

@@ -0,0 +1,158 @@
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+
+EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
+API_URL = 'http://data.enigma.co/api/v1'
+
+
+def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
+    """
+    Download and extract a bcolz bundle.
+
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
+
+    Returns
+    -------
+    str
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+
+    """
+    root = get_exchange_bundles_folder(exchange_name)
+    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
+        exchange=exchange_name,
+        frequency=data_frequency,
+        symbol=symbol,
+        period=period
+    )
+    path = os.path.join(root, name)
+
+    if not os.path.isdir(path):
+        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
+              'exchange-{exchange}/{name}.tar.gz'.format(
+            exchange=exchange_name,
+            name=name)
+
+        bytes = download_without_progress(url)
+        with tarfile.open('r', fileobj=bytes) as tar:
+            tar.extractall(path)
+
+    return path
+
+
+def get_df_from_arrays(arrays, periods):
+    """
+    A DataFrame from the specified OHCLV arrays.
+
+    Parameters
+    ----------
+    arrays: Object
+    periods: DateTimeIndex
+
+    Returns
+    -------
+    DataFrame
+
+    """
+    ohlcv = dict()
+    for index, field in enumerate(
+            ['open', 'high', 'low', 'close', 'volume']):
+        ohlcv[field] = arrays[index].flatten()
+
+    df = pd.DataFrame(
+        data=ohlcv,
+        index=periods
+    )
+    return df
+
+
+def range_in_bundle(asset, start_dt, end_dt, reader):
+    """
+    Evaluate whether price data of an asset is included has been ingested in
+    the exchange bundle for the given date range.
+
+    Parameters
+    ----------
+    asset: TradingPair
+    start_dt: datetime
+    end_dt: datetime
+    reader: BcolzBarMinuteReader
+
+    Returns
+    -------
+    bool
+
+    """
+    has_data = True
+    dates = [start_dt, end_dt]
+
+    while dates and has_data:
+        try:
+            dt = dates.pop(0)
+            close = reader.get_value(asset.sid, dt, 'close')
+
+            if np.isnan(close):
+                has_data = False
+
+        except Exception:
+            has_data = False
+
+    return has_data
+
+
+def get_assets(exchange, include_symbols, exclude_symbols):
+    """
+    Get assets from an exchange, including or excluding the specified
+    symbols.
+
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
+
+        return exchange.get_assets(include_symbols_list)
+
+    else:
+        all_assets = exchange.get_assets()
+
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/utils/datetime_utils.py

@@ -1,17 +1,12 @@
 import calendar
-import os
-import tarfile
-from datetime import timedelta, datetime, date
+import re
+from datetime import datetime, timedelta, date
 
-import numpy as np
 import pandas as pd
 import pytz
 
-from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
-
-EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
-API_URL = 'http://data.enigma.co/api/v1'
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
+    InvalidHistoryFrequencyAlias
 
 
 def get_date_from_ms(ms):
@@ -49,45 +44,6 @@ def get_seconds_from_date(date):
     return int((date - epoch).total_seconds())
 
 
-def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
-    """
-    Download and extract a bcolz bundle.
-
-    Parameters
-    ----------
-    exchange_name: str
-    symbol: str
-    data_frequency: str
-    period: str
-
-    Returns
-    -------
-    str
-        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-
-    """
-    root = get_exchange_bundles_folder(exchange_name)
-    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
-        exchange=exchange_name,
-        frequency=data_frequency,
-        symbol=symbol,
-        period=period
-    )
-    path = os.path.join(root, name)
-
-    if not os.path.isdir(path):
-        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
-              'exchange-{exchange}/{name}.tar.gz'.format(
-                exchange=exchange_name,
-                name=name)
-
-        bytes = download_without_progress(url)
-        with tarfile.open('r', fileobj=bytes) as tar:
-            tar.extractall(path)
-
-    return path
-
-
 def get_delta(periods, data_frequency):
     """
     Get a time delta based on the specified data frequency.
@@ -106,7 +62,7 @@ def get_delta(periods, data_frequency):
         if data_frequency == 'minute' else timedelta(days=periods)
 
 
-def get_periods_range(start_dt, end_dt, freq):
+def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
     """
     Get a date range for the specified parameters.
 
@@ -127,7 +83,38 @@ def get_periods_range(start_dt, end_dt, freq):
     elif freq == 'daily':
         freq = 'D'
 
-    return pd.date_range(start_dt, end_dt, freq=freq)
+    if start_dt is not None and end_dt is not None and periods is None:
+
+        return pd.date_range(start_dt, end_dt, freq=freq)
+
+    elif periods is not None and (start_dt is not None or end_dt is not None):
+        _, unit_periods, unit, _ = get_frequency(freq)
+        adj_periods = periods * unit_periods
+
+        # TODO: standardize time aliases to avoid any mapping
+        unit = 'd' if unit == 'D' else 'm'
+        delta = pd.Timedelta(adj_periods, unit)
+
+        if start_dt is not None:
+            return pd.date_range(
+                start=start_dt,
+                end=start_dt + delta,
+                freq=freq,
+                closed='left',
+            )
+
+        else:
+            return pd.date_range(
+                start=end_dt - delta,
+                end=end_dt,
+                freq=freq,
+            )
+
+    else:
+        raise ValueError(
+            'Choose only two parameters between start_dt, end_dt '
+            'and periods.'
+        )
 
 
 def get_periods(start_dt, end_dt, freq):
@@ -145,7 +132,7 @@ def get_periods(start_dt, end_dt, freq):
     int
 
     """
-    return len(get_periods_range(start_dt, end_dt, freq))
+    return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
 
 
 def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
@@ -157,6 +144,7 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     end_dt: datetime
     bar_count: int
     data_frequency: str
+    include_first
 
     Returns
     -------
@@ -260,99 +248,80 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_df_from_arrays(arrays, periods):
+def get_frequency(freq, data_frequency=None):
     """
-    A DataFrame from the specified OHCLV arrays.
+    Get the frequency parameters.
+
+    Notes
+    -----
+    We're trying to use Pandas convention for frequency aliases.
 
     Parameters
     ----------
-    arrays: Object
-    periods: DateTimeIndex
+    freq: str
+    data_frequency: str
 
     Returns
     -------
-    DataFrame
+    str, int, str, str
 
     """
-    ohlcv = dict()
-    for index, field in enumerate(
-            ['open', 'high', 'low', 'close', 'volume']):
-        ohlcv[field] = arrays[index].flatten()
+    if data_frequency is None:
+        data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
 
-    df = pd.DataFrame(
-        data=ohlcv,
-        index=periods
-    )
-    return df
+    if freq == 'minute':
+        unit = 'T'
+        candle_size = 1
 
-
-def range_in_bundle(asset, start_dt, end_dt, reader):
-    """
-    Evaluate whether price data of an asset is included has been ingested in
-    the exchange bundle for the given date range.
-
-    Parameters
-    ----------
-    asset: TradingPair
-    start_dt: datetime
-    end_dt: datetime
-    reader: BcolzBarMinuteReader
-
-    Returns
-    -------
-    bool
-
-    """
-    has_data = True
-    dates = [start_dt, end_dt]
-
-    while dates and has_data:
-        try:
-            dt = dates.pop(0)
-            close = reader.get_value(asset.sid, dt, 'close')
-
-            if np.isnan(close):
-                has_data = False
-
-        except Exception:
-            has_data = False
-
-    return has_data
-
-
-def get_assets(exchange, include_symbols, exclude_symbols):
-    """
-    Get assets from an exchange, including or excluding the specified
-    symbols.
-
-    Parameters
-    ----------
-    exchange: Exchange
-    include_symbols: str
-    exclude_symbols: str
-
-    Returns
-    -------
-    list[TradingPair]
-
-    """
-    if include_symbols is not None:
-        include_symbols_list = include_symbols.split(',')
-
-        return exchange.get_assets(include_symbols_list)
+    elif freq == 'daily':
+        unit = 'D'
+        candle_size = 1
 
     else:
-        all_assets = exchange.get_assets()
-
-        if exclude_symbols is not None:
-            exclude_symbols_list = exclude_symbols.split(',')
-
-            assets = []
-            for asset in all_assets:
-                if asset.symbol not in exclude_symbols_list:
-                    assets.append(asset)
-
-            return assets
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
+                else 1
+            unit = freq_match.group(2)
 
         else:
-            return all_assets
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+    # TODO: some exchanges support H and W frequencies but not bundles
+    # Find a way to pass-through these parameters to exchanges
+    # but resample from minute or daily in backtest mode
+    # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
+    # Pandas offet aliases (used by Catalyst) and the CCXT timeframes
+    if unit.lower() == 'd':
+        unit = 'D'
+        alias = '{}D'.format(candle_size)
+
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm' or unit == 'T':
+        unit = 'T'
+        alias = '{}T'.format(candle_size)
+
+        if data_frequency == 'daily':
+            data_frequency = 'minute'
+
+    # elif unit.lower() == 'h':
+    #     candle_size = candle_size * 60
+    #
+    #     alias = '{}T'.format(candle_size)
+    #     if data_frequency == 'daily':
+    #         data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyAlias(freq=freq)
+
+    return alias, candle_size, unit, data_frequency
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
+
+
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)

catalyst/exchange/utils/exchange_utils.py

@@ -2,7 +2,6 @@ import hashlib
 import json
 import os
 import pickle
-import re
 import shutil
 from datetime import date, datetime
 
@@ -12,8 +11,9 @@ from six import string_types
 from six.moves.urllib import request
 
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
-    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
+from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
+    ExchangeJSONDecoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
@@ -62,6 +62,13 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
+def is_blacklist(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'blacklist.txt')
+
+    return os.path.exists(filename)
+
+
 def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
@@ -101,20 +108,6 @@ def download_exchange_symbols(exchange_name, environ=None):
     return response
 
 
-def symbols_parser(asset_def):
-    for key, value in asset_def.items():
-        match = isinstance(value, string_types) \
-                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
-
-        if match:
-            try:
-                asset_def[key] = pd.to_datetime(value, utc=True)
-            except ValueError:
-                pass
-
-    return asset_def
-
-
 def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     """
     The de-serialized content of the exchange's symbols.json.
@@ -134,13 +127,16 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
 
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
                 pd.Timestamp('now', tz='UTC') - last_modified_time(
-                    filename)).days > 1):
-        download_exchange_symbols(exchange_name, environ)
+                filename)).days > 1):
+        try:
+            download_exchange_symbols(exchange_name, environ)
+        except Exception as e:
+            pass
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
             try:
-                data = json.load(data_file, object_hook=symbols_parser)
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
                 return data
 
             except ValueError:
@@ -195,7 +191,7 @@ def get_symbols_string(assets):
     return ', '.join([asset.symbol for asset in array])
 
 
-def get_exchange_auth(exchange_name, environ=None):
+def get_exchange_auth(exchange_name, alias=None, environ=None):
     """
     The de-serialized contend of the exchange's auth.json file.
 
@@ -210,7 +206,8 @@ def get_exchange_auth(exchange_name, environ=None):
 
     """
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'auth.json')
+    name = 'auth' if alias is None else alias
+    filename = os.path.join(exchange_folder, '{}.json'.format(name))
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
@@ -266,7 +263,7 @@ def get_algo_folder(algo_name, environ=None):
     return algo_folder
 
 
-def get_algo_object(algo_name, key, environ=None, rel_path=None):
+def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     """
     The de-serialized object of the algo name and key.
 
@@ -290,19 +287,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
 
-    filename = os.path.join(folder, key + '.p')
+    name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
+    filename = os.path.join(folder, name)
 
     if os.path.isfile(filename):
-        try:
+        if how == 'pickle':
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception:
-            return None
+
+        else:
+            with open(filename) as data_file:
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
+                return data
+
     else:
         return None
 
 
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
+                     how='pickle'):
     """
     Serialize and save an object by algo name and key.
 
@@ -321,10 +324,15 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
-    filename = os.path.join(folder, key + '.p')
+    if how == 'json':
+        filename = os.path.join(folder, '{}.json'.format(key))
+        with open(filename, 'wt') as handle:
+            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
 
-    with open(filename, 'wb') as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
+    else:
+        filename = os.path.join(folder, '{}.p'.format(key))
+        with open(filename, 'wb') as handle:
+            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
@@ -428,6 +436,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
+def has_bundle(exchange_name, data_frequency, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+
+    folder_name = '{}_bundle'.format(data_frequency.lower())
+    folder = os.path.join(exchange_folder, folder_name)
+
+    return os.path.isdir(folder)
+
+
 def symbols_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -495,67 +512,6 @@ def get_common_assets(exchanges):
     return assets
 
 
-def get_frequency(freq, data_frequency):
-    """
-    Get the frequency parameters.
-
-    Notes
-    -----
-    We're trying to use Pandas convention for frequency aliases.
-
-    Parameters
-    ----------
-    freq: str
-    data_frequency: str
-
-    Returns
-    -------
-    str, int, str, str
-
-    """
-    if freq == 'minute':
-        unit = 'T'
-        candle_size = 1
-
-    elif freq == 'daily':
-        unit = 'D'
-        candle_size = 1
-
-    else:
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
-                else 1
-            unit = freq_match.group(2)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-    if unit.lower() == 'd':
-        alias = '{}D'.format(candle_size)
-
-        if data_frequency == 'minute':
-            data_frequency = 'daily'
-
-    elif unit.lower() == 'm' or unit == 'T':
-        alias = '{}T'.format(candle_size)
-
-        if data_frequency == 'daily':
-            data_frequency = 'minute'
-
-    # elif unit.lower() == 'h':
-    #     candle_size = candle_size * 60
-    #
-    #     alias = '{}T'.format(candle_size)
-    #     if data_frequency == 'daily':
-    #         data_frequency = 'minute'
-
-    else:
-        raise InvalidHistoryFrequencyAlias(freq=freq)
-
-    return alias, candle_size, unit, data_frequency
-
-
 def resample_history_df(df, freq, field):
     """
     Resample the OHCLV DataFrame using the specified frequency.
@@ -629,14 +585,6 @@ def mixin_market_params(exchange_name, params, market):
                 params['lot'] = params['min_trade_size']
 
 
-def from_ms_timestamp(ms):
-    return pd.to_datetime(ms, unit='ms', utc=True)
-
-
-def get_epoch():
-    return pd.to_datetime('1970-1-1', utc=True)
-
-
 def group_assets_by_exchange(assets):
     exchange_assets = dict()
     for asset in assets:
@@ -646,3 +594,70 @@ def group_assets_by_exchange(assets):
         exchange_assets[asset.exchange].append(asset)
 
     return exchange_assets
+
+
+def get_catalyst_symbol(market_or_symbol):
+    """
+    The Catalyst symbol.
+
+    Parameters
+    ----------
+    market_or_symbol
+
+    Returns
+    -------
+
+    """
+    if isinstance(market_or_symbol, string_types):
+        parts = market_or_symbol.split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+    else:
+        return '{}_{}'.format(
+            market_or_symbol['base'].lower(),
+            market_or_symbol['quote'].lower(),
+        )
+
+
+def save_asset_data(folder, df, decimals=8):
+    symbols = df.index.get_level_values('symbol')
+    for symbol in symbols:
+        symbol_df = df.loc[(symbols == symbol)]  # Type: pd.DataFrame
+
+        filename = os.path.join(folder, '{}.csv'.format(symbol))
+        if os.path.exists(filename):
+            print_headers = False
+
+        else:
+            print_headers = True
+
+        with open(filename, 'a') as f:
+            symbol_df.to_csv(
+                path_or_buf=f,
+                header=print_headers,
+                float_format='%.{}f'.format(decimals),
+            )
+
+
+def get_candles_df(candles, field, freq, bar_count, end_dt,
+                   previous_value=None):
+    all_series = dict()
+    for asset in candles:
+        periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
+
+        dates = [candle['last_traded'] for candle in candles[asset]]
+        values = [candle[field] for candle in candles[asset]]
+        series = pd.Series(values, index=dates)
+
+        series = series.reindex(
+            periods,
+            method='ffill',
+            fill_value=previous_value,
+        )
+        series.sort_index(inplace=True)
+        all_series[asset] = series
+
+    df = pd.DataFrame(all_series)
+    df.dropna(inplace=True)
+
+    return df

catalyst/exchange/utils/factory.py

@@ -0,0 +1,97 @@
+import os
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
+    get_exchange_folder, is_blacklist
+from logbook import Logger
+
+log = Logger('factory', level=LOG_LEVEL)
+exchange_cache = dict()
+
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
+                 skip_init=False, auth_alias=None):
+    key = (exchange_name, base_currency)
+    if key in exchange_cache:
+        return exchange_cache[key]
+
+    exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
+        )
+
+    exchange = CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+    )
+    exchange_cache[key] = exchange
+
+    if not skip_init:
+        exchange.init()
+
+    return exchange
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges
+
+
+def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
+                   base_currency=None):
+    """
+    Find exchanges filtered by a list of feature.
+
+    Parameters
+    ----------
+    features: str
+        The list of features.
+
+    skip_blacklist: bool
+    is_authenticated: bool
+    base_currency: bool
+
+    Returns
+    -------
+    list[Exchange]
+
+    """
+    exchange_names = CCXT.find_exchanges(features, is_authenticated)
+
+    exchanges = []
+    for exchange_name in exchange_names:
+        if skip_blacklist and is_blacklist(exchange_name):
+            continue
+
+        exchange = get_exchange(
+            exchange_name=exchange_name,
+            skip_init=True,
+            base_currency=base_currency,
+        )
+
+        if features is not None:
+            if 'dailyBundle' in features \
+                    and not exchange.has_bundle('daily'):
+                continue
+
+            elif 'minuteBundle' in features \
+                    and not exchange.has_bundle('minute'):
+                continue
+
+        exchanges.append(exchange)
+
+    return exchanges

catalyst/exchange/utils/live_chart_utils.py

@@ -0,0 +1,131 @@
+import matplotlib.dates as mdates
+import pandas as pd
+
+from catalyst.exchange.exchange_errors import \
+    MismatchingBaseCurrenciesExchanges
+
+fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+
+
+def format_ax(ax):
+    """
+    Trying to assign reasonable parameters to the time axis.
+
+    Parameters
+    ----------
+    ax:
+
+    """
+    # TODO: room for improvement
+    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
+    ax.xaxis.set_major_formatter(fmt)
+
+    locator = mdates.HourLocator(interval=4)
+    locator.MAXTICKS = 5000
+    ax.xaxis.set_minor_locator(locator)
+
+    datemin = pd.Timestamp.utcnow()
+    ax.set_xlim(datemin)
+
+    ax.grid(True)
+
+
+def set_legend(ax):
+    """
+    Set legend on the chart.
+
+    Parameters
+    ----------
+    ax
+
+    """
+    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
+
+
+def draw_pnl(ax, df):
+    """
+    Draw p&l line on the chart.
+
+    """
+    ax.clear()
+    ax.set_title('Performance')
+    index = df.index.unique()
+    dt = index.get_level_values(level=0)
+    pnl = index.get_level_values(level=4)
+    ax.plot(
+        dt, pnl, '-',
+        color='green',
+        linewidth=1.0,
+        label='Performance'
+    )
+
+    def perc(val):
+        return '{:2f}'.format(val)
+
+    ax.format_ydata = perc
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_custom_signals(ax, df):
+    """
+    Draw custom signals on the chart.
+
+    """
+    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
+
+    ax.clear()
+    ax.set_title('Custom Signals')
+    for index, column in enumerate(df.columns.values.tolist()):
+        ax.plot(df.index, df[column], '-',
+                color=colors[index],
+                linewidth=1.0,
+                label=column
+                )
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_exposure(ax, df, context):
+    """
+    Draw exposure line on the chart.
+
+    """
+    # TODO: list exchanges in graph
+    base_currency = None
+    positions = []
+    for exchange_name in context.exchanges:
+        exchange = context.exchanges[exchange_name]
+
+        if not base_currency:
+            base_currency = exchange.base_currency
+        elif base_currency != exchange.base_currency:
+            raise MismatchingBaseCurrenciesExchanges(
+                base_currency=base_currency,
+                exchange_name=exchange.name,
+                exchange_currency=exchange.base_currency
+            )
+
+        positions += exchange.portfolio.positions
+
+    ax.clear()
+    ax.set_title('Exposure')
+    ax.plot(df.index, df['base_currency'], '-',
+            color='green',
+            linewidth=1.0,
+            label='Base Currency: {}'.format(base_currency.upper())
+            )
+
+    symbols = []
+    for position in positions:
+        symbols.append(position.symbol)
+
+    ax.plot(df.index, df['long_exposure'], '-',
+            color='blue',
+            linewidth=1.0,
+            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
+
+    set_legend(ax)
+    format_ax(ax)

catalyst/exchange/utils/serialization_utils.py

@@ -0,0 +1,69 @@
+import json
+import re
+from json import JSONEncoder
+
+import pandas as pd
+from catalyst.constants import DATE_TIME_FORMAT
+from six import string_types
+
+
+class ExchangeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pd.Timestamp):
+            return obj.strftime(DATE_TIME_FORMAT)
+
+        # Let the base class default method raise the TypeError
+        return JSONEncoder.default(self, obj)
+
+
+class ExchangeJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        json.JSONDecoder.__init__(
+            self, object_hook=self.object_hook, *args, **kwargs
+        )
+
+    def recursive_iter(self, obj):
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                match = isinstance(value, string_types) and re.search(
+                    r'(\d{4}-\d{2}-\d{2}).*', value
+                )
+                if match:
+                    try:
+                        obj[key] = pd.to_datetime(value, utc=True)
+                    except ValueError:
+                        pass
+
+        elif any(isinstance(obj, t) for t in (list, tuple)):
+            for item in obj:
+                self.recursive_iter(item)
+
+    def object_hook(self, obj):
+        self.recursive_iter(obj)
+        return obj
+
+
+def portfolio_to_dict(portfolio):
+    positions = []
+    for asset in portfolio.positions:
+        p = portfolio.positions[asset]  # Type: Position
+
+        position = dict(
+            symbol=asset.symbol,
+            exchange=asset.exchange,
+            amount=p.amount,
+            cost_basis=p.cost_basis,
+            last_sale_price=p.last_sale_price,
+            last_sale_date=p.last_sale_date,
+        )
+        positions.append(position)
+
+    portfolio_dict = vars(portfolio)
+    portfolio_dict['positions'] = positions
+
+    return portfolio_dict
+
+
+def portfolio_from_dict(self, portfolio_data):
+    from catalyst.protocol import Portfolio
+    return Portfolio()

catalyst/exchange/utils/stats_utils.py

@@ -1,18 +1,19 @@
-import csv
-import numbers
-
 import copy
-import numpy as np
+import csv
+import json
+import numbers
 import os
-import pandas as pd
-import boto3
 import time
 
+import numpy as np
+import pandas as pd
 from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_algo_folder
+from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
 
-from catalyst.exchange.exchange_utils import get_algo_folder
-
-s3 = boto3.resource('s3')
+s3_conn = []
+mailgun = []
 
 
 def trend_direction(series):
@@ -195,6 +196,9 @@ def prepare_stats(stats, recorded_cols=list()):
         if recorded_cols is not None:
             for column in recorded_cols[:]:
                 value = row_data[column]
+                if isinstance(value, pd.Series):
+                    value = value.to_dict()
+
                 if type(value) is dict:
                     for asset in value:
                         if not isinstance(asset, TradingPair):
@@ -257,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()):
     return df, columns
 
 
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
@@ -276,23 +287,19 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
 
     """
     if isinstance(stats, pd.DataFrame):
-        stats = stats.T.to_dict().values()
+        stats = list(stats.T.to_dict().values())
+        stats.sort(key=itemgetter('period_close'))
 
-    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
 
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-    formatters = {
-        'returns': lambda returns: "{0:.4f}".format(returns),
-    }
-
-    return df.tail(num_rows).to_string(
-        columns=columns,
-        formatters=formatters
+    df, columns = prepare_stats(
+        display_stats, recorded_cols=recorded_cols
     )
+    set_print_settings()
+    return df.to_string(columns=columns)
 
 
 def get_csv_stats(stats, recorded_cols=None):
@@ -338,6 +345,12 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     -------
 
     """
+    if not s3_conn:
+        import boto3
+        s3_conn.append(boto3.resource('s3'))
+
+    s3 = s3_conn[0]
+
     if bytes_to_write is None:
         bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
 
@@ -346,12 +359,45 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     pid = os.getpid()
 
     parts = uri.split('//')
-    obj = s3.Object(parts[1], '{}/{}-{}-{}.csv'.format(
-        folder, timestr, algo_namespace, pid
-    ))
+    path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
+        folder=folder,
+        algo=algo_namespace,
+        time=timestr,
+        pid=pid,
+    )
+    obj = s3.Object(parts[1], path)
     obj.put(Body=bytes_to_write)
 
 
+def email_error(algo_name, dt, e, environ=None):
+    import requests
+    import traceback
+
+    if not mailgun:
+        root = data_root(environ)
+        filename = os.path.join(root, 'mailgun.json')
+        if not os.path.exists(filename):
+            raise ValueError(
+                'mailgun.json not found in the catalyst data folder'
+            )
+
+        with open(filename) as data_file:
+            mailgun.append(json.load(data_file))
+
+    mg = mailgun[0]
+
+    return requests.post(
+        mg['url'],
+        auth=("api", mg['api']),
+        data={
+            "from": mg['from'],
+            "to": mg['to'],
+            "subject": 'Error: {}'.format(algo_name),
+            "text": '{}\n\n{}\n{}'.format(
+                dt, e, traceback.format_exc()
+            )})
+
+
 def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     """
     Saves the performance stats to the algo local folder.
@@ -372,7 +418,10 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     timestr = time.strftime('%Y%m%d')
     folder = get_algo_folder(algo_namespace)
 
-    filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
+    stats_folder = os.path.join(folder, 'stats')
+    ensure_directory(stats_folder)
+
+    filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
 
     with open(filename, 'wb') as handle:
         handle.write(bytes_to_write)
@@ -401,6 +450,17 @@ def df_to_string(df):
     return df.to_string()
 
 
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
 def extract_transactions(perf):
     """
     Compute indexes for buy and sell transactions

catalyst/exchange/utils/test_utils.py

@@ -0,0 +1,82 @@
+import os
+import random
+import tempfile
+
+from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import find_exchanges
+from catalyst.utils.paths import ensure_directory
+
+
+def handle_exchange_error(exchange, e):
+    try:
+        message = '{}: {}'.format(
+            e.__class__, e.message.decode('ascii', 'ignore')
+        )
+    except Exception:
+        message = 'unexpected error'
+
+    folder = get_exchange_folder(exchange.name)
+    filename = os.path.join(folder, 'blacklist.txt')
+    with open(filename, 'wt') as handle:
+        handle.write(message)
+
+
+def select_random_exchanges(population=3, features=None,
+                            is_authenticated=False, base_currency=None):
+    all_exchanges = find_exchanges(
+        features=features,
+        is_authenticated=is_authenticated,
+        base_currency=base_currency,
+    )
+
+    if population is not None:
+        if len(all_exchanges) < population:
+            population = len(all_exchanges)
+
+        exchanges = random.sample(all_exchanges, population)
+
+    else:
+        exchanges = all_exchanges
+
+    return exchanges
+
+
+def select_random_assets(all_assets, population=3):
+    assets = random.sample(all_assets, population)
+    return assets
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
+    else:
+        asset_folder = ','.join(
+            ['{}_{}'.format(a.exchange, a.symbol) for a in assets]
+        )
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path, folder

catalyst/exchange/validator.py

@@ -1,142 +0,0 @@
-import os
-import tempfile
-
-import pandas as pd
-import six
-from catalyst.assets._assets import TradingPair, get_calendar
-from logbook import Logger
-from pandas.util.testing import assert_frame_equal
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
-from catalyst.exchange.factory import get_exchanges
-from catalyst.utils.paths import ensure_directory
-
-log = Logger('Validator', level=LOG_LEVEL)
-
-
-def output_df(df, assets, name=None):
-    """
-    Outputs a price DataFrame to a temp folder.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-    assets
-    name
-
-    Returns
-    -------
-
-    """
-    if isinstance(assets, TradingPair):
-        exchange_folder = assets.exchange
-        asset_folder = assets.symbol
-    else:
-        exchange_folder = ','.join([asset.exchange for asset in assets])
-        asset_folder = ','.join([asset.symbol for asset in assets])
-
-    folder = os.path.join(
-        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
-    )
-    ensure_directory(folder)
-
-    if name is None:
-        name = 'output'
-
-    path = os.path.join(folder, '{}.csv'.format(name))
-    df.to_csv(path)
-
-    return path
-
-
-class Validator(object):
-    def __init__(self, data_portal):
-        self.data_portal = data_portal
-
-    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     sample_minutes):
-        """
-        Creates DataFrames from the bundle and exchange for the specified
-        data set.
-
-        Parameters
-        ----------
-        exchange: Exchange
-        assets
-        end_dt
-        bar_count
-        sample_minutes
-
-        Returns
-        -------
-
-        """
-        freq = '{}T'.format(sample_minutes)
-
-        log.info('creating data sample from bundle')
-        df1 = self.data_portal.get_history_window(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=bar_count,
-            frequency=freq,
-            field='close',
-            data_frequency='minute'
-        )
-        path = output_df(df1, assets, '{}_resampled'.format(freq))
-        log.info('saved resampled bundle candles: {}\n{}'.format(
-            path, df1.tail(10))
-        )
-
-        log.info('creating data sample from exchange api')
-        candles = exchange.get_candles(
-            end_dt=end_dt,
-            freq='{}T'.format(sample_minutes),
-            assets=assets,
-            bar_count=bar_count
-        )
-
-        series = dict()
-        for asset in assets:
-            series[asset] = pd.Series(
-                data=[candle['close'] for candle in candles[asset]],
-                index=[candle['last_traded'] for candle in candles[asset]]
-            )
-
-        df2 = pd.DataFrame(series)
-        path = output_df(df2, assets, '{}_api'.format(freq))
-        log.info('saved exchange api candles: {}\n{}'.format(
-            path, df2.tail(10))
-        )
-
-        try:
-            assert_frame_equal(df1, df2)
-            return True
-        except:
-            log.warn('differences found in dataframes')
-            return False
-
-
-if __name__ == '__main__':
-    exchanges = get_exchanges(['poloniex'])
-    exchange = six.next(six.itervalues(exchanges))
-    assets = exchange.get_assets(symbols=['eth_btc'])
-
-    open_calendar = get_calendar('OPEN')
-    asset_finder = AssetFinderExchange()
-    data_portal = DataPortalExchangeBacktest(
-        exchanges=exchanges,
-        asset_finder=asset_finder,
-        trading_calendar=open_calendar,
-        first_trading_day=None  # will set dynamically based on assets
-    )
-    validator = Validator(data_portal=data_portal)
-
-    validator.compare_bundle_with_exchange(
-        exchange=exchange,
-        assets=assets,
-        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
-        bar_count=200,
-        sample_minutes=30
-    )

catalyst/finance/risk/cumulative.py

@@ -27,15 +27,15 @@ from .risk import (
     choose_treasury
 )
 
-from empyrical import (
+from catalyst.patches.stats import (
     alpha_beta_aligned,
     annual_volatility,
-    cum_returns,
     downside_risk,
     information_ratio,
     max_drawdown,
     sharpe_ratio,
     sortino_ratio,
+    cum_returns,
 )
 import warnings
 from catalyst.constants import LOG_LEVEL
@@ -161,9 +161,13 @@ class RiskMetricsCumulative(object):
             if len(self.algorithm_returns) == 1:
                 self.algorithm_returns = np.append(0.0, self.algorithm_returns)
 
-        self.algorithm_cumulative_returns[dt_loc] = cum_returns(
-            self.algorithm_returns
-        )[-1]
+        try:
+            self.algorithm_cumulative_returns[dt_loc] = cum_returns(
+                self.algorithm_returns
+            )[-1]
+        except Exception as e:
+            log.debug('unable to calculate cum returns: {}'.format(e))
+            self.algorithm_cumulative_returns[dt_loc] = np.nan
 
         algo_cumulative_returns_to_date = \
             self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -196,8 +200,11 @@ class RiskMetricsCumulative(object):
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
-        except Exception:
-            self.benchmark_cumulative_returns[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.benchmark_cumulative_returns[dt_loc] = np.nan
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -269,9 +276,16 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.sharpe[dt_loc] = sharpe_ratio(
             self.algorithm_returns,
         )
-        self.downside_risk[dt_loc] = downside_risk(
-            self.algorithm_returns
-        )
+
+        try:
+            self.downside_risk[dt_loc] = downside_risk(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate downside risk returns: {}'.format(e)
+            )
+            self.downside_risk[dt_loc] = np.nan
 
         try:
             risk = self.downside_risk[dt_loc]
@@ -279,17 +293,26 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
                 self.algorithm_returns,
                 _downside_risk=risk
             )
-        except Exception:
-            # TODO: what causes it to error out?
-            self.sortino[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.sortino[dt_loc] = np.nan
 
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
         )
-        self.max_drawdown = max_drawdown(
-            self.algorithm_returns
-        )
+        try:
+            self.max_drawdown = max_drawdown(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate max drawdown: {}'.format(e)
+            )
+            self.max_drawdown = np.nan
+
         self.max_drawdowns[dt_loc] = self.max_drawdown
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage

catalyst/pipeline/engine.py

@@ -7,6 +7,7 @@ from abc import (
 )
 from uuid import uuid4
 
+import six
 from six import (
     iteritems,
     with_metaclass,
@@ -33,7 +34,6 @@ from catalyst.utils.sharedoc import copydoc
 
 
 class PipelineEngine(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def run_pipeline(self, pipeline, start_date, end_date):
         """
@@ -118,6 +118,7 @@ class ExplodingPipelineEngine(PipelineEngine):
     """
     A PipelineEngine that doesn't do anything.
     """
+
     def run_pipeline(self, pipeline, start_date, end_date):
         raise NoEngineRegistered(
             "Attempted to run a pipeline but no pipeline "
@@ -484,8 +485,10 @@ class SimplePipelineEngine(PipelineEngine):
             )
 
             if isinstance(term, LoadableTerm):
+                term_key = loader_group_key(term)
+                # TODO: temp workaround
                 to_load = sorted(
-                    loader_groups[loader_group_key(term)],
+                    six.next(six.itervalues(loader_groups)),
                     key=lambda t: t.dataset
                 )
                 loader = get_loader(term)
@@ -565,9 +568,10 @@ class SimplePipelineEngine(PipelineEngine):
                 index=MultiIndex.from_arrays([empty_dates, empty_assets]),
             )
 
-        resolved_assets = array(self._finder.retrieve_all(assets))
+        # TODO: not sure what's wrong with the resolved_assets
+        # resolved_assets = array(self._finder.retrieve_all(assets))
         dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
-        assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
+        assets_kept = repeat_first_axis(assets, len(dates))[mask]
 
         final_columns = {}
         for name in data:

catalyst/pipeline/graph.py

@@ -142,7 +142,7 @@ class TermGraph(object):
         at the end of execution.
         """
         refcounts = self.graph.out_degree()
-        for t in self.outputs.values():
+        for t in list(self.outputs.values()):
             refcounts[t] += 1
 
         for t in initial_terms:
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
                  min_extra_rows=0):
         super(ExecutionPlan, self).__init__(terms)
 
-        for term in terms.values():
+        for term in list(terms.values()):
             self.set_extra_rows(
                 term,
                 all_dates,

catalyst/sources/test_source.py

@@ -144,7 +144,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
             for event in self.event_list:
                 event.sid = assets_by_identifier[event.sid].sid
 
@@ -167,7 +167,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
 
         # Hash_value for downstream sorting.
         self.arg_string = hash_args(*args, **kwargs)

catalyst/support/binance_history.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from catalyst import run_algorithm
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = 'cryptopia'
+    context.base_currency = 'btc'
+    context.coins = context.exchanges[context.exchange].assets
+    context.coins = [c for c in context.coins if
+                     c.quote_currency == context.base_currency]
+
+
+def handle_data(context, data):
+    # current date formatted into a string
+    today = data.current_dt
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.coins = context.exchanges[context.exchange].assets
+        context.coins = [c for c in context.coins if
+                         c.quote_currency == context.base_currency]
+
+    # get data every 30 minutes
+    minutes = 1
+    if not context.i % minutes:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            price = data.current(coin, 'price')
+            print(today, pair, price)
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2018-01-17', utc=True)
+    end_date = pd.to_datetime('2018-01-18', utc=True)
+
+    performance = run_algorithm(
+        capital_base=1.0,
+        # amount of base_currency, not always in dollars unless usd
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='cryptopia',
+        data_frequency='minute',
+        base_currency='btc',
+        live=True,
+        live_graph=False,
+        simulate_orders=True,
+        algo_namespace='simple_universe'
+    )

catalyst/support/ccxt_issue_1358.py

@@ -0,0 +1,8 @@
+import ccxt
+
+bitfinex = ccxt.bitfinex()
+bitfinex.verbose = True
+ohlcvs = bitfinex.fetch_ohlcv('ETH/BTC', '30m', 1504224000000)
+
+dt = bitfinex.iso8601(ohlcvs[0][0])
+print(dt)  # should print '2017-09-01T00:00:00.000Z'

catalyst/support/history_multiple_assets.py

@@ -0,0 +1,50 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset1 = symbol('fct_btc')
+    context.asset2 = symbol('btc_usdt')
+    context.coins = [context.asset1, context.asset2]
+
+
+def handle_data(context, data):
+    df = data.history(context.coins,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+    print(df)
+    print(data.current(context.asset1, 'close'))
+    print(data.current(context.asset2, 'close'))
+    exit(0)
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_111.py

@@ -0,0 +1,44 @@
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import order_target_percent
+
+NAMESPACE = 'goose7'
+log = Logger(NAMESPACE)
+
+from catalyst.api import record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('trx_btc')
+
+
+def handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    record(btc=price)
+
+    # Only ordering if it does not have any position to avoid trying some
+    # tiny orders with the leftover btc
+    pos_amount = context.portfolio.positions[context.asset].amount
+    if pos_amount > 0:
+        return
+
+    # Adding a limit price to workaround an issue with performance
+    # calculations of market orders
+    order_target_percent(
+        context.asset, 1, limit_price=price * 1.01
+    )
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.003,
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=NAMESPACE,
+        base_currency='btc',
+        live_graph=False,
+        simulate_orders=False,
+    )

catalyst/support/issue_112.py

@@ -0,0 +1,44 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    df = data.history(context.asset,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_126.py

@@ -0,0 +1,52 @@
+from catalyst import run_algorithm
+from catalyst.api import order, record, symbol
+import pandas as pd
+
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+
+
+def initialize(context):
+    context.assets = [symbol('eth_btc'), symbol('eth_usdt')]
+
+
+def handle_data(context, data):
+    order(context.assets[0], 1)
+
+    prices = data.current(context.assets, 'price')
+    record(price=prices)
+    pass
+
+
+def analyze(context, perf):
+    stats = get_pretty_stats(perf)
+    print(stats)
+    pass
+
+
+if __name__ == '__main__':
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='btc',
+            analyze=analyze,
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='usd',
+            analyze=analyze,
+            start=pd.to_datetime('2017-01-01', utc=True),
+            end=pd.to_datetime('2017-12-25', utc=True),
+        )

catalyst/support/issue_169.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from catalyst.utils.run_algo import run_algorithm
+from catalyst.api import symbol
+from exchange.utils.stats_utils import set_print_settings
+
+
+def initialize(context):
+    context.i = 0
+    context.data = []
+
+
+def handle_data(context, data):
+    prices = data.history(
+        symbol('xlm_eth'),
+        fields=['open', 'high', 'low', 'close'],
+        bar_count=50,
+        frequency='1T'
+    )
+    set_print_settings()
+    print(prices.tail(10))
+    context.data.append(prices)
+
+    context.i = context.i + 1
+    if context.i == 3:
+        context.interrupt_algorithm()
+
+
+def analyze(context, prefs):
+    for dataset in context.data:
+        print(dataset[-2:])
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.1,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='binance',
+        algo_namespace='Test candles',
+        base_currency='eth',
+        data_frequency='minute',
+        live=True,
+        simulate_orders=True)

catalyst/support/issue_30.py

@@ -0,0 +1,28 @@
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+
+def initialize(context):
+    context.asset = symbol('bcc_usdt')
+
+
+def handle_data(context, data):
+    data.history(context.asset, ['close'], bar_count=100, frequency='5T')
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=100,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bittrex',
+        algo_namespace="bittrex_is_broken",
+        base_currency='usdt',
+        data_frequency='minute',
+        simulate_orders=True,
+        live=True)

catalyst/utils/run_algo.py

@@ -8,11 +8,14 @@ from time import sleep
 
 import click
 import pandas as pd
-from logbook import Logger
+from six import string_types
 
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
+    TradingPairPricing
+from catalyst.exchange.utils.factory import get_exchange
+from logbook import Logger
 
 try:
     from pygments import highlight
@@ -37,10 +40,7 @@ from catalyst.exchange.exchange_algorithm import (
 )
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, NotEnoughCapitalError)
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
 
 from catalyst.constants import LOG_LEVEL
 
@@ -91,12 +91,15 @@ def _run(handle_data,
          algo_namespace,
          base_currency,
          live_graph,
+         analyze_live,
          simulate_orders,
+         auth_aliases,
          stats_output):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
     """
+    # TODO: refactor for more granularity
     if algotext is not None:
         if local_namespace:
             ip = get_ipython()  # noqa
@@ -141,22 +144,53 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
-    mode = 'paper-trading' if simulate_orders else 'live-trading' \
-        if live else 'backtest'
+    log.warn(
+        'Catalyst is currently in ALPHA. It is going through rapid '
+        'development and it is subject to errors. Please use carefully. '
+        'We encourage you to report any issue on GitHub: '
+        'https://github.com/enigmampc/catalyst/issues'
+    )
+    sleep(3)
+
+    if live:
+        if simulate_orders:
+            mode = 'paper-trading'
+        else:
+            mode = 'live-trading'
+    else:
+        mode = 'backtest'
+
     log.info('running algo in {mode} mode'.format(mode=mode))
 
     exchange_name = exchange
     if exchange_name is None:
         raise ValueError('Please specify at least one exchange.')
 
-    exchange_list = [x.strip().lower() for x in exchange.split(',')]
+    if isinstance(auth_aliases, string_types):
+        aliases = auth_aliases.split(',')
+        if len(aliases) < 2 or len(aliases) % 2 != 0:
+            raise ValueError(
+                'the `auth_aliases` parameter must contain an even list '
+                'of comma-delimited values. For example, '
+                '"binance,auth2" or "binance,auth2,bittrex,auth2".'
+            )
 
+        auth_aliases = dict(zip(aliases[::2], aliases[1::2]))
+
+    exchange_list = [x.strip().lower() for x in exchange.split(',')]
     exchanges = dict()
-    for exchange_name in exchange_list:
-        exchanges[exchange_name] = get_exchange(
-            exchange_name=exchange_name,
+    for name in exchange_list:
+        if auth_aliases is not None and name in auth_aliases:
+            auth_alias = auth_aliases[name]
+        else:
+            auth_alias = None
+
+        exchanges[name] = get_exchange(
+            exchange_name=name,
             base_currency=base_currency,
             must_authenticate=(live and not simulate_orders),
+            skip_init=True,
+            auth_alias=auth_alias,
         )
 
     open_calendar = get_calendar('OPEN')
@@ -172,14 +206,22 @@ def _run(handle_data,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
     )
-    env.asset_finder = AssetFinderExchange()
-    choose_loader = None  # TODO: use the DataPortal in the algo class for this
+    env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
+
+    def choose_loader(column):
+        bound_cols = TradingPairPricing.columns
+        if column in bound_cols:
+            return ExchangePricingLoader(data_frequency)
+        raise ValueError(
+            "No PipelineLoader registered for column %s." % column
+        )
 
     if live:
         start = pd.Timestamp.utcnow()
 
         # TODO: fix the end data.
-        end = start + timedelta(hours=8760)
+        if end is None:
+            end = start + timedelta(hours=8760)
 
         data = DataPortalExchangeLive(
             exchanges=exchanges,
@@ -188,66 +230,6 @@ def _run(handle_data,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
 
-        def fetch_capital_base(exchange, attempt_index=0):
-            """
-            Fetch the base currency amount required to bootstrap
-            the algorithm against the exchange.
-
-            The algorithm cannot continue without this value.
-
-            :param exchange: the targeted exchange
-            :param attempt_index:
-            :return capital_base: the amount of base currency available for
-            trading
-            """
-            try:
-                log.debug('retrieving capital base in {} to bootstrap '
-                          'exchange {}'.format(base_currency, exchange_name))
-                balances = exchange.get_balances()
-            except ExchangeRequestError as e:
-                if attempt_index < 20:
-                    log.warn(
-                        'could not retrieve balances on {}: {}'.format(
-                            exchange.name, e
-                        )
-                    )
-                    sleep(5)
-                    return fetch_capital_base(exchange, attempt_index + 1)
-
-                else:
-                    raise ExchangeRequestErrorTooManyAttempts(
-                        attempts=attempt_index,
-                        error=e
-                    )
-
-            if base_currency in balances:
-                base_currency_available = balances[base_currency]['free']
-                log.info(
-                    'base currency available in the account: {} {}'.format(
-                        base_currency_available, base_currency
-                    )
-                )
-
-                return base_currency_available
-            else:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=base_currency,
-                    exchange=exchange_name
-                )
-
-        if not simulate_orders:
-            for exchange_name in exchanges:
-                exchange = exchanges[exchange_name]
-                balance = fetch_capital_base(exchange)
-
-                if balance < capital_base:
-                    raise NotEnoughCapitalError(
-                        exchange=exchange_name,
-                        base_currency=base_currency,
-                        balance=balance,
-                        capital_base=capital_base,
-                    )
-
         sim_params = create_simulation_parameters(
             start=start,
             end=end,
@@ -266,6 +248,8 @@ def _run(handle_data,
             live_graph=live_graph,
             simulate_orders=simulate_orders,
             stats_output=stats_output,
+            analyze_live=analyze_live,
+            end=end,
         )
     elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
@@ -427,7 +411,9 @@ def run_algorithm(initialize,
                   base_currency=None,
                   algo_namespace=None,
                   live_graph=False,
+                  analyze_live=None,
                   simulate_orders=True,
+                  auth_aliases=None,
                   stats_output=None,
                   output=os.devnull):
     """Run a trading algorithm.
@@ -559,6 +545,8 @@ def run_algorithm(initialize,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=analyze_live,
         simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
         stats_output=stats_output
     )

docs/Makefile

@@ -23,7 +23,7 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  build      to build the C and Cython extensions for zipline"
+	@echo "  build      to build the C and Cython extensions for catalyst"
 	@echo "  html       to make standalone HTML files"
 	@echo "  livehtml   to run a persistent process that rebuilds the docs"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
@@ -96,9 +96,9 @@ qthelp: build
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/zipline.qhcp"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/catalyst.qhcp"
 	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/zipline.qhc"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/catalyst.qhc"
 
 applehelp: build
 	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@@ -113,8 +113,8 @@ devhelp: build
 	@echo
 	@echo "Build finished."
 	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/zipline"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/zipline"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/catalyst"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/catalyst"
 	@echo "# devhelp"
 
 epub: build

docs/deploy.py

@@ -8,8 +8,8 @@ from shutil import move, rmtree
 from subprocess import check_call
 
 HERE = dirname(abspath(__file__))
-ZIPLINE_ROOT = dirname(HERE)
-TEMP_LOCATION = '/tmp/zipline-doc'
+CATALYST_ROOT = dirname(HERE)
+TEMP_LOCATION = '/tmp/catalyst-doc'
 TEMP_LOCATION_GLOB = TEMP_LOCATION + '/*'
 
 
@@ -46,8 +46,8 @@ def main():
             print("Copying built files to temp location.")
             move('build/html', TEMP_LOCATION)
 
-            print("Moving to '%s'" % ZIPLINE_ROOT)
-            os.chdir(ZIPLINE_ROOT)
+            print("Moving to '%s'" % CATALYST_ROOT)
+            os.chdir(CATALYST_ROOT)
 
             print("Checking out gh-pages branch.")
             check_call(
@@ -70,7 +70,7 @@ def main():
         os.chdir(old_dir)
 
     print()
-    print("Updated documentation branch in directory %s" % ZIPLINE_ROOT)
+    print("Updated documentation branch in directory %s" % CATALYST_ROOT)
     print("If you are happy with these changes, commit and push to gh-pages.")
 
 if __name__ == '__main__':

docs/make.bat

@@ -127,9 +127,9 @@ if "%1" == "qthelp" (
 	echo.
 	echo.Build finished; now you can run "qcollectiongenerator" with the ^
 .qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\zipline.qhcp
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\catalyst.qhcp
 	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\zipline.ghc
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\catalyst.ghc
 	goto end
 )
 

docs/source/beginner-tutorial.rst

@@ -483,7 +483,7 @@ bitcoin price.
 
 Now we will run the simulation again, but this time we extend our original 
 algorithm with the addition of the ``analyze()`` function. Somewhat analogously 
-as how ``initialize()`` gets called once before the start of the algorith, 
+as how ``initialize()`` gets called once before the start of the algorithm, 
 ``analyze()`` gets called once at the end of the algorithm, and receives two 
 variables: ``context``, which we discussed at the very beginning, and ``perf``, 
 which is the pandas dataframe containing the performance data for our algorithm 
@@ -589,7 +589,7 @@ the ``examples`` directory:
     from catalyst import run_algorithm
     from catalyst.api import (order, record, symbol, order_target_percent, 
             get_open_orders)
-    from catalyst.exchange.stats_utils import extract_transactions
+    from catalyst.exchange.utils.stats_utils import extract_transactions
 
     NAMESPACE = 'dual_moving_average'
     log = Logger(NAMESPACE)
@@ -660,7 +660,8 @@ the ``examples`` directory:
     def analyze(context, perf):
 
         # Get the base_currency that was passed as a parameter to the simulation
-        base_currency = context.exchanges.values()[0].base_currency.upper()
+        exchange = list(context.exchanges.values())[0]
+        base_currency = exchange.base_currency.upper()
 
         # First chart: Plot portfolio value using base_currency
         ax1 = plt.subplot(411)
@@ -928,7 +929,6 @@ functions.
                 context.asset,
                 target_hodl_value,
                 limit_price=price*1.1,
-                stop_price=price*0.9,
             )
 
         record(

docs/source/conf.py

@@ -41,11 +41,11 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC, Inc.'
+copyright = u'2018, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
-release = '0.3'
+release = '0.4'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

docs/source/development-guidelines.rst

@@ -84,6 +84,25 @@ To build and view the docs locally, run:
    $ {BROWSER} build/html/index.html
 
 
+There is a `documented issue <https://github.com/sphinx-doc/sphinx/issues/3212>`_ 
+with ``sphinx`` and ``docutils`` that causes the error below when trying to build 
+the docs.
+
+.. code-block:: text
+
+   Exception occurred:
+     File "(...)/env-c/lib/python2.7/site-packages/docutils/writers/_html_base.py", line 671, in depart_document
+       assert not self.context, 'len(context) = %s' % len(self.context)
+   AssertionError: len(context) = 3
+
+If you get this error, you need to downgrade your version of ``docutils`` as 
+follows, and build the docs again:
+
+.. code-block:: bash
+
+   $ pip install docutils==0.12
+
+
 Commit messages
 ---------------
 

docs/source/example-algos.rst

@@ -184,7 +184,6 @@ one day prior to the current date.
               context.asset,
               target_hodl_value,
               limit_price=price * 1.1,
-              stop_price=price * 0.9,
           )
 
       record(

docs/source/features.rst

@@ -44,11 +44,11 @@ For additional details on the functionality added on recent releases, see the
 Upcoming features
 ~~~~~~~~~~~~~~~~~
 
-* Additional datasets beyond pricing data (Dec. 2017)
-* API documentation (Jan. 2017)
-* Support for decentralized exchanges (Jan. 2017)
-* Support for data ingestion of community-contributed data sets (Jan. 2017)
-* Pipeline support (Jan. 2018)
+* Additional datasets beyond pricing data (Q1 2018)
+* API documentation (Q1 2018)
+* Support for decentralized exchanges (Q1 2018)
+* Support for data ingestion of community-contributed data sets (Q1 2018)
+* Pipeline support (Q1 2018)
 * Web UI (Q2 2018)
 
 

docs/source/install.rst

@@ -15,6 +15,9 @@ as an alternative installation method for MacOS and Linux, you can install
 Catalyst directly with ``pip`` (we recommend in combination with a virtual 
 environemnt). See :ref:`Installing with pip <pip>`.
 
+Alternatively you can install Catalyst using ``pipenv`` which is a mix of pip
+and virtualenv. See :ref:`Installing with pipenv <pipenv>`.
+
 Regardless of the method, each operating system (OS), has its own 
 prerequisites, make sure to review the corresponding sections for your system:
 :ref:`Linux <linux>`, :ref:`MacOS <macos>` and :ref:`Windows <windows>`.
@@ -177,20 +180,6 @@ use a single tool to install Python and non-Python dependencies, or if you're
 already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
 distribution, refer to the :ref:`Installing with Conda <conda>` section.
 
-Once you've installed the necessary additional dependencies for your system 
-(see below for your particular platform: :ref:`Linux`, :ref:`MacOS` or 
-:ref:`Windows`), you should be able to simply run
-
-.. code-block:: bash
-
-   $ pip install enigma-catalyst matplotlib
-
-Note that in the command above we install two different packages. The second 
-one, ``matplotlib`` is a visualization library. While it's not strictly 
-required to run catalyst simulations or live trading, it comes in very handy
-to visualize the performance of your algorithms, and for this reason we 
-recommend you install it, as well.
-
 If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
@@ -203,8 +192,21 @@ summarized version:
    $ pip install virtualenv
    $ virtualenv catalyst-venv
    $ source ./catalyst-venv/bin/activate
+
+Once you've installed the necessary additional dependencies for your system 
+(:ref:`Linux`, :ref:`MacOS` or :ref:`Windows`) **and have activated your virtualenv**, you should be able to simply run
+
+.. code-block:: bash
+
    $ pip install enigma-catalyst matplotlib
 
+Note that in the command above we install two different packages. The second 
+one, ``matplotlib`` is a visualization library. While it's not strictly 
+required to run catalyst simulations or live trading, it comes in very handy
+to visualize the performance of your algorithms, and for this reason we 
+recommend you install it, as well.
+
+
 Troubleshooting ``pip`` Install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -216,13 +218,13 @@ Troubleshooting ``pip`` Install
 
    .. code-block:: bash
 
-      pip install --upgrade pip
+      $ pip install --upgrade pip
 
    On Windows, the recommended command is:
 
    .. code-block:: bash
 
-      python -m pip install --upgrade pip
+      $ python -m pip install --upgrade pip
 
 ----
 
@@ -248,7 +250,7 @@ Troubleshooting ``pip`` Install
 
    .. code-block:: bash
 
-      pip install --pre enigma-catalyst
+      $ pip install --pre enigma-catalyst
 
 ----
 
@@ -260,7 +262,7 @@ Troubleshooting ``pip`` Install
 
    .. code-block:: bash
 
-      pip install --upgrade pip setuptools
+      $ pip install --upgrade pip setuptools
 
 ----
 
@@ -275,7 +277,7 @@ Troubleshooting ``pip`` Install
 
    .. code-block:: bash
 
-      pip install -r requirements.txt
+      $ pip install -r requirements.txt
 
 ----
 
@@ -291,8 +293,41 @@ Troubleshooting ``pip`` Install
 
    .. code-block:: bash
 
-      sudo apt-get install python-dev
+      $ sudo apt-get install python-dev
 
+.. _pipenv:
+
+Installing with ``pipenv``
+-------------------------
+
+Installing Catalyst via ``pipenv`` is perhaps easier that installing it via
+``pip`` itself but you need to install ``pipenv`` first via ``pip``.
+
+.. code-block:: bash
+
+   $ pip install pipenv
+
+Once ``pipenv`` is installed you can proceed by creating a project folder and
+installing Catalyst on that project automagically as follows:
+
+.. code-block:: bash
+
+   $ mkdir project
+   $ cd project
+   $ pipenv --two
+   $ pipenv install enigma-catalyst matplotlib
+
+Until now the workflow compared to ``pip`` is almost identical, the difference
+is that you don't need to load manually any virtualenv however you need to use
+the `pipenv run` prefix to run the `catalyst` command as follows:
+
+.. code-block:: bash
+
+   $ pipenv run catalyst --version
+
+If you want to know more about ``pipenv`` go to the `pipenv github repo`_
+
+.. _`pipenv github repo`: https://github.com/pypa/pipenv
 
 .. _linux:
 
@@ -340,14 +375,14 @@ outdated. Thus, you first need to run:
 
 .. code-block:: bash
 
-   pip install --upgrade pip setuptools
+   $ pip install --upgrade pip setuptools
 
 The default installation is also missing the C and C++ compilers, which you 
 install by:
 
 .. code-block:: bash
 
-   sudo yum install gcc gcc-c++
+   $ sudo yum install gcc gcc-c++
 
 Then you should follow the regular installation instructions outlined at the 
 beginning of this page.
@@ -372,20 +407,34 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
-MacOS + virtualenv + matplotlib
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+MacOS + virtualenv/conda + matplotlib
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A note about using matplotlib in virtual enviroments on MacOS: it may be 
-necessary to run
+The first time that you try to run an algorithm that loads the ``matplotlib`` 
+library, you may get the following error:
+
+.. code-block:: text
+
+  RuntimeError: Python is not installed as a framework. The Mac OS X backend 
+  will not be able to function correctly if Python is not installed as a 
+  framework. See the Python documentation for more information on installing 
+  Python as a framework on Mac OS X. Please either reinstall Python as a 
+  framework, or try one of the other backends. If you are using (Ana)Conda 
+  please install python.app and replace the use of 'python' with 'pythonw'. 
+  See 'Working with Matplotlib on OSX' in the Matplotlib FAQ for more 
+  information.
+
+This is a ``matplotlib``-specific error, that will go away once you run the 
+following command:
 
 .. code-block:: bash
 
-   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+   $ echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
 
 in order to override the default ``MacOS`` backend for your system, which 
-may not be accessible from inside the virtual environment. This will allow 
-Catalyst to open matplotlib charts from within a virtual environment, which 
-is useful for displaying the performance of your backtests.  To learn more 
+may not be accessible from inside the virtual or conda environment. This will 
+allow Catalyst to open matplotlib charts from within a virtual environment, 
+which is useful for displaying the performance of your backtests.  To learn more 
 about matplotlib backends, please refer to the
 `matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
 
@@ -439,6 +488,33 @@ mentioned above are as follows:
   - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
   - Run ``msiexec /i VCForPython27.msi``
 
+Updating Catalyst
+-----------------
+
+Catalyst is currently in alpha and in under very active development. We release
+new minor versions every few days in response to the thorough battle testing 
+that our user community puts Catalyst in. As a result, you should expect to 
+update Catalyst frequently. Once installed, Catalyst can easily be updated as a 
+``pip`` package regardless of the environemnt used for installation. Make sure 
+you activate your environment first as you did in your first install, and then 
+execute:
+
+.. code-block:: bash
+
+   $ pip uninstall enigma-catalyst
+   $ pip install enigma-catalyst
+
+Alternatively, you could update Catalyst issuing the following command:
+
+.. code-block:: bash
+
+   $ pip install -U enigma-catalyst
+
+but this command will also upgrade all the Catalyst dependencies to the latest 
+versions available, and may have unexpected side effects if a newer version of a 
+dependency inadvertently breaks some functionality that Catalyst relies on. 
+Thus, the first method is the recommended one.
+
 Getting Help
 ------------
 

docs/source/live-trading.rst

@@ -4,11 +4,63 @@ This document explains how to get started with live trading.
 
 Supported Exchanges
 ^^^^^^^^^^^^^^^^^^^
-Catalyst can trade against these exchanges:
 
-- Bitfinex, id= ``bitfinex``
-- Bittrex, id= ``bittrex``
-- Poloniex, id= ``poloniex``
+Since version 0.4, Catalyst integrated with `CCXT <https://github.com/ccxt/ccxt>`_,
+a cryptocurrency trading library with support for more than 90 exchanges. The 
+range of CCXT and Catalyst support for each of those exchanges varies greatly. 
+The most supported exchanges are as follows:
+
+The exchanges available for backtesting are fully supported in live mode:
+
+- Bitfinex, id = ``bitfinex``
+- Bittrex, id = ``bittrex``
+- Poloniex, id = ``poloniex``
+
+Additionally, we have successfully tested the following exchanges:
+
+- Binance, id = ``binance``
+- Bitmex, id = ``bitmex``
+- GDAX, id = ``gdax``
+
+As Catalyst is currently in Alpha and in under active development, you are 
+encouraged to throughly test any exchange in *paper trading* mode before trading
+*live* with it.
+
+Paper Trading vs Live Trading modes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Catalyst currently supports three different modes in which you can execute your
+trading algorithm. The first is backtesting, which is covered extensively in the 
+tutorial, and uses historical data to run your algorithm. There is no 
+interaction with the exchange in backtesting mode, and this is the first mode 
+that you should test any new algorithm.
+
+Once you are confident with the simulations that you have obtained with your
+algorithm in backtesting, you may switch to live trading, where you have two
+different modes:
+* *Paper Trading*: The simulated algorithm runs in real time, and fetches 
+pricing data in real time from the exchange, but the orders never reach the 
+exchange, and are instead kept within Catalyst and simulated. No real currency
+is bought or sold. Think of it as a `backtesting happening in real time`.
+* *Live Trading*: This is the proper live trading mode in which an algorithm
+runs in real time, fetching pricing data from live exchanges and placing orders 
+against the exchange. Real currency is transacted on the exchange driven by the 
+algorithm.
+
+These three modes are controlled by the following variables:
+
++---------------+-------------------------+
+|    Mode       |       Parameters        |
++               +-------+-----------------+
+|               | live  | simulate_orders |
++---------------+-------+-----------------+
+| backtesting   | False | True (default)  |
++---------------+-------+-----------------+
+| paper trading | True  | True            |
++---------------+-------+-----------------+
+| live trading  | True  | False           |
++---------------+-------+-----------------+
+
 
 Authentication
 ^^^^^^^^^^^^^^
@@ -75,7 +127,8 @@ Note that the trading pairs are always referenced in the same manner.
 However, not all trading pairs are available on all exchanges. An
 error will occur if the specified trading pair is not trading
 on the exchange. To check which currency pairs are available on each 
-of the supported exchanges, see `Catalyst Market Coverage <https://www.enigma.co/catalyst/status`_.
+of the supported exchanges, see 
+`Catalyst Market Coverage <https://www.enigma.co/catalyst/status>`_.
 
 Trading an Algorithm
 ^^^^^^^^^^^^^^^^^^^^
@@ -105,20 +158,22 @@ What differs are the arguments provided to the catalyst client or
 
 Here is the breakdown of the new arguments:
 
-- ``live``: Boolean flag which enables live trading.
+- ``live``: Boolean flag which enables live trading. It defaults to ``False``.
 - ``capital_base``: The amount of base_currency assigned to the strategy.
   It has to be lower or equal to the amount of base currency available for
   trading on the exchange. For illustration, order_target_percent(asset, 1)
   will order the capital_base amount specified here of the specified asset.
-- ``exchange_name``: The name of the targeted exchange
-  (supported values: *bitfinex*, *bittrex*).
+- ``exchange_name``: The name of the targeted exchange. See the 
+  `CCXT Supported Exchanges <https://github.com/ccxt/ccxt/wiki/Exchange-Markets>`_ 
+  for the full list.
 - ``algo_namespace``: A arbitrary label assigned to your algorithm for
   data storage purposes.
 - ``base_currency``: The base currency used to calculate the
   statistics of your algorithm. Currently, the base currency of all
   trading pairs of your algorithm must match this value.
 - ``simulate_orders``: Enables the paper trading mode, in which orders are
-  simulated in Catalyst instead of processed on the exchange.
+  simulated in Catalyst instead of processed on the exchange. It defaults to 
+  ``True``.
 
 Here is a complete algorithm for reference:
 `Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_

docs/source/releases.rst

@@ -2,23 +2,92 @@
 Release Notes
 =============
 
-Version 0.3.10
+Version 0.4.7
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-19
+
+Bug Fixes
+~~~~~~~~~
+- Fixing issue :issue:`137` impacting the CLI
+
+Build
+~~~~~
+- Implemented authentication aliases (:issue:`60`)
+
+Version 0.4.6
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-18
+
+Bug Fixes
+~~~~~~~~~
+- Fixed some Python3 issues
+- Reading the trade log to get executed order prices on exchanges like Binance (:issue:`151`)
+- Fixed issue with market order executing price (:issue:`150` and :issue:`111`)
+- Implemented standardized symbol mapping (:issue:`157`)
+- Improved error handling for unsupported timeframes (:issue:`159`)
+- Using Bitfinex instead of Poloniex to fetch btc_usdt benchmark (:issue:`161`)
+
+
+Build
+~~~~~
+- Added a `context.state` dict to keep arbitrary state values between runs
+- Added ability to stop live algo at specified end date
+
+Version 0.4.5
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-12
+
+Bug Fixes
+~~~~~~~~~
+- Improved order execution for exchanges supporting trade lists (:issue:`151`)
+- Fixed an issue where requesting history of multiple assets repeats values
+- Raising an error for order amounts smaller than exchange lots
+- Handling multiple req errors with tickers more gracefully (:issue:`160`)
+
+Version 0.4.4
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-09
+
+Bug Fixes
+~~~~~~~~~
+- Removed redundant capital_base validation (:issue:`142`)
+- Fixed portfolio update issue with restored state (:issue:`111`)
+- Skipping cash validation where there are open orders (:issue:`144`)
+
+Version 0.4.3
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-05
+
+Bug Fixes
+~~~~~~~~~
+- Fixed CLI issue (:issue:`137`)
+- Upgraded CCXT
+
+Version 0.4.2
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-03
+
+Bug Fixes
+~~~~~~~~~
+- Fixed cash synchronization issue (:issue:`133`)
+- Fixed positions synchronization issue (:issue:`132`)
+- Patched empyrical to resolve a np.log1p issue (:issue:`126`)
+- Fixed a paper trading issue (:issue:`124`)
+- Fixed a commission issue (:issue:`104`)
+- Fixed a poloniex specific issue in live trading (:issue:`103`)
+
+Build
+~~~~~
+- Caching CCXT market info to limit round-trips (:issue:`99`)
+- Tentative support for Pipeline (:issue:`96`)
+
+Version 0.4.0
 ^^^^^^^^^^^^^
 **Release Date**: 2017-12-12
 
 Bug Fixes
 ~~~~~~~~~
 
-- Fixed issue with fetching assets with daily frequency
-
-Version 0.3.10
-^^^^^^^^^^^^^
-**Release Date**: 2017-11-28
-
-Bug Fixes
-~~~~~~~~~
-
-- Fixed issue with fetching assets with daily frequency
 - Changed Poloniex interface (should solve :issue:`95` and :issue:`94`)
 - Solved issue with overriding commission and slippage (:issue:`87`)
 - Fixed inefficiency with Bittrex current prices (:issue:`76`)
@@ -30,6 +99,15 @@ Build
 - More granular commissions (:issue:`82`)
 - Added market orders in live mode (:issue:`81`)
 
+Version 0.3.10
+~~~~~~~~~~~~~~
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with fetching assets with daily frequency
+
 Version 0.3.9
 ^^^^^^^^^^^^^
 **Release Date**: 2017-11-28

docs/source/unit-tests.rst

@@ -0,0 +1,88 @@
+==========
+Unit Tests
+==========
+
+Exchanges
+~~~~~~~~~
+
+Markets
+-------
+Sample:
+    All markets in 3 random exchanges
+Test:
+    Fetch all TradingPair instances
+Assert:
+    No error
+
+Current Ticker
+------------------
+Sample:
+    3 random markets in each of the 3 random exchanges
+Test:
+    Fetch current price and volume
+Assert:
+    Not null and no error
+
+Historical Price Data
+---------------------
+Sample:
+    - 3 random markets for each of the 3 random exchanges supporting historical data
+    - For each market, randomly select one supported frequency
+Test:
+    Fetch historical data for each market using the selected frequency
+Assert:
+    - No error and not blank
+    - Date of each candle is consistent with the Catalyst desired pattern,
+        - All candle start at fix intervals
+        - Last candle partial and forward looking from the end date
+
+Authentication and Orders
+-------------------------
+Sample:
+    1 random market for each of 3 random authenticated exchanges
+Test:
+    - Create one limit order randomly buying or selling at least 10% out from the current price
+    - Retrieve the open order from the exchange
+    - Cancel the open order
+Assert:
+    No error
+
+
+Bundles
+~~~~~~~
+
+Validate Bundle Data
+--------------------
+Sample:
+    - 3 random market in bundles for exchanges supporting historical data
+    - For each market, randomly selected data range available in the exchange historical data
+Test:
+    - Clean the target exchange bundle
+    - Ingest the selected market data for the selected data range
+    - Retrieve the bundle data into a dataframe
+    - Retrieve the equivalent OHLCV data from the exchange into a dataframe
+Assert:
+    Matching data for the bundle and exchange
+
+
+Algo Stats
+----------
+Sample:
+    - 2 sample algorithms with built-in stats calculator
+    - 2 KPIs both calculated by each algo and by Catalyst
+Test:
+    - Run each algorithm
+    - Compare the results of the two methods or calculating stats
+Assert:
+    - Matching stats
+
+CSV Ingestion
+-------------
+Sample:
+    3 random CSV files containing price data
+Test:
+    - Ingest each CSV files
+    - Validate with the exchange like in the 'Validate Bundle Data' test
+Assert:
+    Matching data between the bundle and the exchange
+

etc/python2.7-environment.yml

@@ -20,7 +20,7 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
-  - ccxt==1.10.319
+  - ccxt==1.10.774
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0
@@ -45,6 +45,7 @@ dependencies:
   - python-dateutil==2.6.1
   - python-editor==1.0.3
   - pytz==2017.2
+  - redo==1.6
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1

etc/requirements.txt

@@ -81,5 +81,6 @@ empyrical==0.2.1
 tables==3.3.0
 
 #Catalyst dependencies
-ccxt==1.10.283
+ccxt==1.10.774
 boto3==1.4.8
+redo==1.6

etc/requirements_docs.txt

@@ -1,3 +1,4 @@
 Sphinx>=1.3.2
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
+docutils==0.12

tests/exchange/test_bcolz.py

@@ -1,15 +1,14 @@
-import shutil
 import random
+import shutil
 import tempfile
-import pandas as pd
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
+import pandas as pd
+from nose.tools import assert_equals
+
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
     BcolzExchangeBarReader
-
-from catalyst.exchange.bundle_utils import get_df_from_arrays
-
-from nose.tools import assert_equals
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.utils.bundle_utils import get_df_from_arrays
 
 
 class TestBcolzWriter(object):

tests/exchange/test_bitfinex.py

@@ -1,78 +0,0 @@
-from logbook import Logger
-
-from base import BaseExchangeTestCase
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.finance.execution import (LimitOrder)
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('test_bitfinex')
-
-
-@deprecated
-class TestBitfinex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        log.info('creating bitfinex object')
-        auth = get_exchange_auth('bitfinex')
-        self.exchange = Bitfinex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency='usd'
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('eth_usd')
-        order_id = self.exchange.order(
-            asset=asset,
-            style=LimitOrder(limit_price=200),
-            limit_price=200,
-            amount=0.5,
-            stop_price=None
-        )
-        log.info('order created {}'.format(order_id))
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # orders = self.exchange.get_open_orders()
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        pass
-
-    def test_cancel_order(self):
-        log.info('cancel order')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        # ohlcv_neo = self.exchange.get_candles(
-        #    freq='1T',
-        #    assets=self.exchange.get_asset('neo_btc'))
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        # tickers = self.exchange.tickers([
-        #     self.exchange.get_asset('eth_btc'),
-        #     self.exchange.get_asset('etc_btc')
-        # ])
-        pass
-
-    def test_get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def test_get_balances(self):
-        log.info('testing exchange balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for bitfinex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_bittrex.py

@@ -1,95 +0,0 @@
-# import pandas as pd
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.finance.order import Order
-from base import BaseExchangeTestCase
-from logbook import Logger
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('test_bittrex')
-
-
-@deprecated
-class TestBittrex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        auth = get_exchange_auth('bittrex')
-        self.exchange = Bittrex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency=None,
-            portfolio=None
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('neo_btc')
-        order_id = self.exchange.order(
-            asset=asset,
-            limit_price=0.0005,
-            amount=1,
-        )
-        log.info('order created {}'.format(order_id))
-        assert order_id is not None
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # asset = self.exchange.get_asset('neo_btc')
-        # orders = self.exchange.get_open_orders(asset)
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        order = self.exchange.get_order(
-            u'2c584020-9caf-4af5-bde0-332c0bba17e2')
-        assert isinstance(order, Order)
-        pass
-
-    def test_cancel_order(self, ):
-        log.info('cancel order')
-        self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        # ohlcv_neo = self.exchange.get_candles(
-        #     freq='5T',
-        #     assets=self.exchange.get_asset('neo_btc'),
-        #     bar_count=20,
-        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
-        # )
-        # ohlcv_neo_ubq = self.exchange.get_candles(
-        #     freq='1D',
-        #     assets=[
-        #         self.exchange.get_asset('neo_btc'),
-        #         self.exchange.get_asset('ubq_btc')
-        #     ],
-        #     bar_count=14,
-        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
-        # )
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
-        assert len(tickers) == 2
-        pass
-
-    def test_get_balances(self):
-        log.info('testing wallet balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_get_account(self):
-        log.info('testing account data')
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for bittrex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_bundle.py

@@ -5,15 +5,16 @@ from logging import getLogger
 
 import pandas as pd
 
-from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
-    get_start_dt, get_df_from_arrays
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
-from catalyst.exchange.exchange_utils import get_exchange_folder
-from catalyst.exchange.factory import get_exchange
-from catalyst.exchange.stats_utils import df_to_string
+from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
+    get_df_from_arrays
+from exchange.utils.datetime_utils import get_start_dt
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.exchange.utils.stats_utils import df_to_string
 from catalyst.utils.paths import ensure_directory
 
 log = getLogger('test_exchange_bundle')

tests/exchange/test_ccxt.py

@@ -1,10 +1,13 @@
 import pandas as pd
 from logbook import Logger
-from base import BaseExchangeTestCase
 
+from catalyst.testing import ZiplineTestCase
+from catalyst.testing.fixtures import WithLogger
+from .base import BaseExchangeTestCase
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth
 from catalyst.finance.order import Order
-from catalyst.exchange.exchange_utils import get_exchange_auth
 
 log = Logger('test_ccxt')
 
@@ -12,22 +15,22 @@ log = Logger('test_ccxt')
 class TestCCXT(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        exchange_name = 'gdax'
+        exchange_name = 'bitfinex'
         auth = get_exchange_auth(exchange_name)
         self.exchange = CCXT(
             exchange_name=exchange_name,
             key=auth['key'],
             secret=auth['secret'],
-            base_currency='eth',
-            portfolio=None
+            base_currency='bnb',
         )
+        self.exchange.init()
 
     def test_order(self):
         log.info('creating order')
-        asset = self.exchange.get_asset('neo_eth')
+        asset = self.exchange.get_asset('neo_bnb')
         order_id = self.exchange.order(
             asset=asset,
-            limit_price=0.07,
+            style=ExchangeLimitOrder(limit_price=10),
             amount=1,
         )
         log.info('order created {}'.format(order_id))
@@ -55,10 +58,10 @@ class TestCCXT(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         candles = self.exchange.get_candles(
-            freq='5T',
+            freq='30T',
             assets=[self.exchange.get_asset('eth_btc')],
             bar_count=200,
-            start_dt=pd.to_datetime('2017-01-01', utc=True)
+            start_dt=pd.to_datetime('2017-09-01', utc=True)
         )
 
         for asset in candles:
@@ -68,12 +71,29 @@ class TestCCXT(BaseExchangeTestCase):
 
     def test_tickers(self):
         log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-        ])
+        assets = [
+            self.exchange.get_asset('iot_usd'),
+        ]
+        tickers = self.exchange.tickers(assets)
         assert len(tickers) == 1
         pass
 
+    def test_my_trades(self):
+        asset = self.exchange.get_asset('dsh_btc')
+
+        trades = self.exchange.get_trades(asset)
+        assert trades
+        pass
+
+    def test_get_executed_order(self):
+        log.info('retrieving executed order')
+        asset = self.exchange.get_asset('eng_eth')
+
+        order = self.exchange.get_order('165784', asset)
+        transactions = self.exchange.process_order(order)
+        assert transactions
+        pass
+
     def test_get_balances(self):
         log.info('testing wallet balances')
         # balances = self.exchange.get_balances()

tests/exchange/test_data_portal.py

@@ -2,13 +2,13 @@ import pandas as pd
 from logbook import Logger
 
 from catalyst import get_calendar
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
 from catalyst.exchange.exchange_data_portal import (
     DataPortalExchangeBacktest,
     DataPortalExchangeLive
 )
-from catalyst.exchange.exchange_utils import get_common_assets
-from catalyst.exchange.factory import get_exchanges
+from catalyst.exchange.utils.exchange_utils import get_common_assets
+from catalyst.exchange.utils.factory import get_exchanges
 from test_utils import rnd_history_date_days, rnd_bar_count
 
 log = Logger('test_bitfinex')
@@ -20,7 +20,7 @@ class TestExchangeDataPortal:
         log.info('creating bitfinex exchange')
         exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
         open_calendar = get_calendar('OPEN')
-        asset_finder = AssetFinderExchange()
+        asset_finder = ExchangeAssetFinder()
 
         self.data_portal_live = DataPortalExchangeLive(
             exchanges=exchanges,

tests/exchange/test_poloniex.py

@@ -1,96 +0,0 @@
-from catalyst.exchange.poloniex.poloniex import Poloniex
-from catalyst.finance.order import Order
-from base import BaseExchangeTestCase
-from logbook import Logger
-from catalyst.exchange.exchange_utils import get_exchange_auth
-import pandas as pd
-
-from catalyst.utils.deprecate import deprecated
-from test_utils import output_df
-
-log = Logger('test_poloniex')
-
-
-@deprecated
-class TestPoloniex(BaseExchangeTestCase):
-    @classmethod
-    def setup(self):
-        print ('creating poloniex object')
-        auth = get_exchange_auth('poloniex')
-        self.exchange = Poloniex(
-            key=auth['key'],
-            secret=auth['secret'],
-            base_currency='btc'
-        )
-
-    def test_order(self):
-        log.info('creating order')
-        asset = self.exchange.get_asset('neos_btc')
-        order_id = self.exchange.order(
-            asset=asset,
-            limit_price=0.0005,
-            amount=1,
-        )
-        log.info('order created {}'.format(order_id))
-        assert order_id is not None
-        pass
-
-    def test_open_orders(self):
-        log.info('retrieving open orders')
-        # asset = self.exchange.get_asset('neos_btc')
-        # orders = self.exchange.get_open_orders(asset)
-        pass
-
-    def test_get_order(self):
-        log.info('retrieving order')
-        order = self.exchange.get_order(
-            u'2c584020-9caf-4af5-bde0-332c0bba17e2')
-        assert isinstance(order, Order)
-        pass
-
-    def test_cancel_order(self, ):
-        log.info('cancel order')
-        self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9')
-        pass
-
-    def test_get_candles(self):
-        log.info('retrieving candles')
-        assets = self.exchange.get_asset('eth_btc')
-        ohlcv = self.exchange.get_candles(
-            # end_dt=pd.to_datetime('2017-11-01', utc=True),
-            end_dt=None,
-            freq='5T',
-            assets=assets,
-            bar_count=200
-        )
-        df = pd.DataFrame(ohlcv)
-        df.set_index('last_traded', drop=True, inplace=True)
-        log.info(df.tail(25))
-
-        path = output_df(df, assets, '5min_candles')
-        log.info('saved candles: {}'.format(path))
-        pass
-
-    def test_tickers(self):
-        log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
-        assert len(tickers) == 2
-        pass
-
-    def test_get_balances(self):
-        log.info('testing wallet balances')
-        # balances = self.exchange.get_balances()
-        pass
-
-    def test_get_account(self):
-        log.info('testing account data')
-        pass
-
-    def test_orderbook(self):
-        log.info('testing order book for poloniex')
-        # asset = self.exchange.get_asset('eth_btc')
-        # orderbook = self.exchange.get_orderbook(asset)
-        pass

tests/exchange/test_server_bundle.py

@@ -1,17 +1,18 @@
-import os
 import importlib
+import os
 
-import pandas as pd
 import matplotlib
 import matplotlib.pyplot as plt
-from matplotlib.finance import candlestick2_ohlc
 # from matplotlib.finance import volume_overlay
 import matplotlib.ticker as ticker
+import pandas as pd
+from matplotlib.finance import candlestick2_ohlc
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
-from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
-from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.utils.bundle_utils import get_df_from_arrays, \
+    get_bcolz_chunk
+from catalyst.exchange.utils.factory import get_exchange
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 exchanges = dict((e, getattr(importlib.import_module(

tests/exchange/test_suites/test_suite_algo.py

@@ -0,0 +1,79 @@
+import importlib
+from os.path import join, isfile
+
+import pandas as pd
+import os
+
+from catalyst import run_algorithm
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
+    extract_transactions, set_print_settings, extract_orders
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+from logbook import TestHandler, WARNING
+from pathtools.path import listdir
+
+filter_algos = [
+    'buy_and_hodl.py',
+    'buy_btc_simple.py',
+    'buy_low_sell_high.py',
+    'mean_reversion_simple.py',
+    'rsi_profit_target.py',
+    'simple_loop.py',
+    'simple_universe.py',
+]
+
+
+class TestSuiteAlgo(WithLogger, ZiplineTestCase):
+    @staticmethod
+    def analyze(context, perf):
+        set_print_settings()
+
+        transaction_df = extract_transactions(perf)
+        print('the transactions:\n{}'.format(transaction_df))
+
+        orders_df = extract_orders(perf)
+        print('the orders:\n{}'.format(orders_df))
+
+        stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
+        print('the stats:\n{}'.format(stats))
+        pass
+
+    def test_run_examples(self):
+        folder = join('..', '..', '..', 'catalyst', 'examples')
+        files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+        algo_list = []
+        for filename in files:
+            name = os.path.basename(filename)
+            if filter_algos and name not in filter_algos:
+                continue
+
+            module_name = 'catalyst.examples.{}'.format(
+                name.replace('.py', '')
+            )
+            algo_list.append(module_name)
+
+        for module_name in algo_list:
+            algo = importlib.import_module(module_name)
+            namespace = module_name.replace('.', '_')
+
+            log_catcher = TestHandler()
+            with log_catcher:
+                run_algorithm(
+                    capital_base=0.1,
+                    data_frequency='minute',
+                    initialize=algo.initialize,
+                    handle_data=algo.handle_data,
+                    analyze=TestSuiteAlgo.analyze,
+                    exchange_name='poloniex',
+                    algo_namespace='test_{}'.format(namespace),
+                    base_currency='eth',
+                    start=pd.to_datetime('2017-10-01', utc=True),
+                    end=pd.to_datetime('2017-10-02', utc=True),
+                    # output=out
+                )
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+
+                if len(warnings) > 0:
+                    print('WARNINGS:\n{}'.format(warnings))
+            pass

tests/exchange/test_suites/test_suite_bundle.py

@@ -0,0 +1,154 @@
+import random
+
+import os
+import pandas as pd
+from logbook import TestHandler
+from pandas.util.testing import assert_frame_equal
+
+from catalyst import get_calendar
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
+from catalyst.exchange.utils.exchange_utils import get_candles_df
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.exchange.utils.test_utils import output_df, \
+    select_random_assets
+
+pd.set_option('display.expand_frame_repr', False)
+pd.set_option('precision', 8)
+pd.set_option('display.width', 1000)
+pd.set_option('display.max_colwidth', 1000)
+
+
+class TestSuiteBundle:
+    @staticmethod
+    def get_data_portal(exchanges):
+        open_calendar = get_calendar('OPEN')
+        asset_finder = ExchangeAssetFinder(exchanges)
+
+        exchange_names = [exchange.name for exchange in exchanges]
+        data_portal = DataPortalExchangeBacktest(
+            exchange_names=exchange_names,
+            asset_finder=asset_finder,
+            trading_calendar=open_calendar,
+            first_trading_day=None  # will set dynamically based on assets
+        )
+        return data_portal
+
+    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
+                                     freq, data_frequency, data_portal):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        freq
+        data_frequency
+        data_portal
+
+        Returns
+        -------
+
+        """
+        data = dict()
+
+        log_catcher = TestHandler()
+        with log_catcher:
+            data['bundle'] = data_portal.get_history_window(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                frequency=freq,
+                field='close',
+                data_frequency=data_frequency,
+            )
+            candles = exchange.get_candles(
+                end_dt=end_dt,
+                freq=freq,
+                assets=assets,
+                bar_count=bar_count,
+            )
+            data['exchange'] = get_candles_df(
+                candles=candles,
+                field='close',
+                freq=freq,
+                bar_count=bar_count,
+                end_dt=end_dt,
+            )
+            for source in data:
+                df = data[source]
+                path, folder = output_df(
+                    df, assets, '{}_{}'.format(freq, source)
+                )
+
+            print('saved {} test results: {}'.format(end_dt, folder))
+
+            assert_frame_equal(
+                right=data['bundle'],
+                left=data['exchange'],
+                check_less_precise=1,
+            )
+            try:
+                assert_frame_equal(
+                    right=data['bundle'],
+                    left=data['exchange'],
+                    check_less_precise=min([a.decimals for a in assets]),
+                )
+            except Exception as e:
+                print('Some differences were found within a 1 decimal point '
+                      'interval of confidence: {}'.format(e))
+                with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
+                    handle.write(e.args[0])
+
+            pass
+
+    def test_validate_bundles(self):
+        # exchange_population = 3
+        asset_population = 3
+        data_frequency = random.choice(['minute'])
+
+        # bundle = 'dailyBundle' if data_frequency
+        #  == 'daily' else 'minuteBundle'
+        # exchanges = select_random_exchanges(
+        #     population=exchange_population,
+        #     features=[bundle],
+        # )  # Type: list[Exchange]
+        exchanges = [get_exchange('poloniex', skip_init=True)]
+
+        data_portal = TestSuiteBundle.get_data_portal(exchanges)
+        for exchange in exchanges:
+            exchange.init()
+
+            frequencies = exchange.get_candle_frequencies(data_frequency)
+            freq = random.sample(frequencies, 1)[0]
+
+            bar_count = random.randint(1, 10)
+
+            assets = select_random_assets(
+                exchange.assets, asset_population
+            )
+            end_dt = None
+            for asset in assets:
+                attribute = 'end_{}'.format(data_frequency)
+                asset_end_dt = getattr(asset, attribute)
+
+                if end_dt is None or asset_end_dt < end_dt:
+                    end_dt = asset_end_dt
+
+            dt_range = pd.date_range(
+                end=end_dt, periods=bar_count, freq=freq
+            )
+            self.compare_bundle_with_exchange(
+                exchange=exchange,
+                assets=assets,
+                end_dt=dt_range[-1],
+                bar_count=bar_count,
+                freq=freq,
+                data_frequency=data_frequency,
+                data_portal=data_portal,
+            )
+        pass

tests/exchange/test_suites/test_suite_exchange.py

@@ -0,0 +1,208 @@
+import json
+import os
+import random
+from logging import Logger, WARNING
+from time import sleep
+
+import pandas as pd
+from catalyst.assets._assets import TradingPair
+from logbook import TestHandler
+
+from catalyst.exchange.exchange_errors import ExchangeRequestError
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.test_utils import select_random_exchanges, \
+    handle_exchange_error, select_random_assets
+from catalyst.testing import ZiplineTestCase
+from catalyst.testing.fixtures import WithLogger
+from exchange.utils.factory import get_exchanges
+
+log = Logger('TestSuiteExchange')
+
+
+class TestSuiteExchange(WithLogger, ZiplineTestCase):
+    def _test_markets_exchange(self, exchange, attempts=0):
+        assets = None
+        try:
+            exchange.init()
+
+            # Verify that the assets and markets are populated
+            if not exchange.markets:
+                raise ValueError(
+                    'no markets found'
+                )
+            if not exchange.assets:
+                raise ValueError(
+                    'no assets derived from markets'
+                )
+            assets = exchange.assets
+
+        except ExchangeRequestError as e:
+            sleep(5)
+
+            if attempts > 5:
+                handle_exchange_error(exchange, e)
+
+            else:
+                print(
+                    're-trying an exchange request {} {}'.format(
+                        exchange.name, attempts
+                    )
+                )
+                self._test_markets_exchange(exchange, attempts + 1)
+
+        except Exception as e:
+            handle_exchange_error(exchange, e)
+
+        return assets
+
+    def test_markets(self):
+        population = 3
+        results = dict()
+
+        exchanges = select_random_exchanges(population)  # Type: list[Exchange]
+        for exchange in exchanges:
+            assets = self._test_markets_exchange(exchange)
+
+            if assets is not None:
+                results[exchange.name] = len(assets)
+
+                folder = get_exchange_folder(exchange.name)
+                filename = os.path.join(folder, 'whitelist.json')
+
+                symbols = [asset.symbol for asset in assets]
+                with open(filename, 'wt') as handle:
+                    json.dump(symbols, handle, indent=4)
+
+        series = pd.Series(results)
+        print('the tested markets\n{}'.format(series))
+
+        if population is not None:
+            assert (len(results) == population)
+
+        pass
+
+    def test_tickers(self):
+        exchange_population = 3
+        asset_population = 15
+
+        # exchanges = select_random_exchanges(
+        #     exchange_population,
+        #     features=['fetchTickers'],
+        # )  # Type: list[Exchange]
+        exchanges = list(get_exchanges(['bitfinex']).values())
+        for exchange in exchanges:
+            exchange.init()
+
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+                tickers = exchange.tickers(assets)
+
+                assert len(tickers) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
+        pass
+
+    def test_candles(self):
+        exchange_population = 3
+        asset_population = 3
+
+        exchanges = select_random_exchanges(
+            population=exchange_population,
+            features=['fetchOHLCV'],
+        )  # Type: list[Exchange]
+        for exchange in exchanges:
+            exchange.init()
+
+            if exchange.assets and len(exchange.assets) >= asset_population:
+                frequencies = exchange.get_candle_frequencies()
+                freq = random.sample(frequencies, 1)[0]
+
+                bar_count = random.randint(1, 10)
+                end_dt = pd.Timestamp.utcnow().floor('1T')
+                dt_range = pd.date_range(
+                    end=end_dt, periods=bar_count, freq=freq
+                )
+                assets = select_random_assets(
+                    exchange.assets, asset_population
+                )
+
+                candles = exchange.get_candles(
+                    freq=freq,
+                    assets=assets,
+                    bar_count=bar_count,
+                    start_dt=dt_range[0],
+                    end_dt=dt_range[-1],
+                )
+
+                assert len(candles) == asset_population
+
+            else:
+                print(
+                    'skipping exchange without assets {}'.format(exchange.name)
+                )
+                exchange_population -= 1
+        pass
+
+    def test_orders(self):
+        population = 3
+        quote_currency = 'eth'
+        order_amount = 0.1
+
+        exchanges = select_random_exchanges(
+            population=population,
+            features=['fetchOrder'],
+            is_authenticated=True,
+            base_currency=quote_currency,
+        )  # Type: list[Exchange]
+
+        log_catcher = TestHandler()
+        with log_catcher:
+            for exchange in exchanges:
+                exchange.init()
+
+                assets = exchange.get_assets(quote_currency=quote_currency)
+                asset = select_random_assets(assets, 1)[0]
+                self.assertIsInstance(asset, TradingPair)
+
+                tickers = exchange.tickers([asset])
+                price = tickers[asset]['last_price']
+
+                amount = order_amount / price
+
+                limit_price = price * 0.8
+                style = ExchangeLimitOrder(limit_price=limit_price)
+
+                order = exchange.order(
+                    asset=asset,
+                    amount=amount,
+                    style=style,
+                )
+                sleep(1)
+
+                open_order = exchange.get_order(order.id, asset)
+                self.assertEqual(0, open_order.status)
+
+                exchange.cancel_order(open_order, asset)
+                sleep(1)
+
+                canceled_order = exchange.get_order(open_order.id, asset)
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+
+                self.assertEqual(0, len(warnings))
+                self.assertEqual(2, canceled_order.status)
+                print(
+                    'tested {exchange} / {symbol}, order: {order}'.format(
+                        exchange=exchange.name,
+                        symbol=asset.symbol,
+                        order=order.id,
+                    )
+                )
+        pass