Merge branch 'develop'

catalyst/__main__.py

@@ -282,6 +282,7 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
+        analyze_live=None,
         live_graph=False,
         simulate_orders=True,
         stats_output=None,
@@ -312,11 +313,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                # these options are set when running in line magic mode
-                # set a non None algo text to use the ipython user_ns
-                '--algotext', '',
-                '--local-namespace',
-            ] if cell is None else []) + line.split(),
+                     # these options are set when running in line magic mode
+                     # set a non None algo text to use the ipython user_ns
+                     '--algotext', '',
+                     '--local-namespace',
+                 ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,
@@ -470,6 +471,7 @@ def live(ctx,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=None,
         simulate_orders=simulate_orders,
         stats_output=None,
     )

catalyst/data/loader.py

@@ -146,6 +146,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         exchange = get_exchange(
             exchange_name='poloniex', base_currency='usdt'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 

catalyst/examples/mean_reversion_simple.py

@@ -37,14 +37,14 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 50
-    context.RSI_OVERBOUGHT = 65
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 60
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.1, taker=0.2)
-    context.set_slippage(spread=0.0001)
+    context.set_commission(maker=0.001, taker=0.002)
+    context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -248,7 +248,7 @@ if __name__ == '__main__':
 
     if live:
         run_algorithm(
-            capital_base=0.03,
+            capital_base=0.1,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
@@ -280,7 +280,7 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -6,12 +6,8 @@ from collections import defaultdict
 import ccxt
 import pandas as pd
 import six
-from catalyst.assets._assets import TradingPair
-from ccxt import ExchangeNotAvailable, InvalidOrder
-from logbook import Logger
-from six import string_types
-
 from catalyst.algorithm import MarketOrder
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -23,6 +19,10 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
     from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
     get_exchange_auth
 from catalyst.finance.order import Order, ORDER_STATUS
+from ccxt import InvalidOrder, NetworkError, \
+    ExchangeError
+from logbook import Logger
+from six import string_types
 
 log = Logger('CCXT', level=LOG_LEVEL)
 
@@ -105,7 +105,12 @@ class CCXT(Exchange):
                 with open(filename, 'w+') as f:
                     json.dump(self.markets, f, indent=4)
 
-            except ExchangeNotAvailable as e:
+            except (ExchangeError, NetworkError) as e:
+                log.warn(
+                    'unable to fetch markets {}: {}'.format(
+                        self.name, e
+                    )
+                )
                 raise ExchangeRequestError(error=e)
 
         self.load_assets()
@@ -408,6 +413,7 @@ class CCXT(Exchange):
     def _fetch_symbol_map(self, is_local):
         try:
             return self.fetch_symbol_map(is_local)
+
         except ExchangeSymbolsNotFound:
             return None
 
@@ -559,8 +565,12 @@ class CCXT(Exchange):
             for key in balances:
                 balances_lower[key.lower()] = balances[key]
 
-        except Exception as e:
-            log.debug('error retrieving balances: {}', e)
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch balance {}: {}'.format(
+                    self.name, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return balances_lower
@@ -703,14 +713,18 @@ class CCXT(Exchange):
                 amount=adj_amount,
                 price=price
             )
-        except ExchangeNotAvailable as e:
-            log.debug('unable to create order: {}'.format(e))
-            raise ExchangeRequestError(error=e)
-
         except InvalidOrder as e:
             log.warn('the exchange rejected the order: {}'.format(e))
             raise CreateOrderError(exchange=self.name, error=e)
 
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to create order {} / {}: {}'.format(
+                    self.name, symbol, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
         if 'info' not in result:
             raise ValueError('cannot use order without info attribute')
 
@@ -735,7 +749,12 @@ class CCXT(Exchange):
                 limit=None,
                 params=dict()
             )
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch open orders {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         orders = []
@@ -758,7 +777,12 @@ class CCXT(Exchange):
             order_status = self.api.fetch_order(id=order_id, symbol=symbol)
             order, executed_price = self._create_order(order_status)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return order, executed_price
@@ -777,7 +801,12 @@ class CCXT(Exchange):
                 if asset_or_symbol is not None else None
             self.api.cancel_order(id=order_id, symbol=symbol)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to cancel order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
     def tickers(self, assets):
@@ -828,10 +857,10 @@ class CCXT(Exchange):
 
                 tickers[asset] = ticker
 
-        except ExchangeNotAvailable as e:
+        except (ExchangeError, NetworkError) as e:
             log.warn(
-                'unable to fetch ticker: {} {}'.format(
-                    self.name, asset.symbol
+                'unable to fetch ticker {} / {}: {}'.format(
+                    self.name, asset.symbol, e
                 )
             )
             raise ExchangeRequestError(error=e)

catalyst/exchange/exchange.py

@@ -5,8 +5,6 @@ from time import sleep
 
 import numpy as np
 import pandas as pd
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -19,6 +17,7 @@ from catalyst.exchange.utils.bundle_utils import get_start_dt, \
     get_delta, get_periods, get_periods_range
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
     get_frequency, resample_history_df, has_bundle
+from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
 

catalyst/exchange/exchange_algorithm.py

@@ -18,11 +18,9 @@ from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
 
+import catalyst.protocol as zp
 import logbook
 import pandas as pd
-from redo import retry
-
-import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
@@ -49,6 +47,7 @@ from catalyst.utils.api_support import api_method
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
+from redo import retry
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
@@ -130,7 +129,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_commission(self, maker=None, taker=None):
-        key = self.blotter.commission_models.keys()[0]
+        key = list(self.blotter.commission_models.keys())[0]
         if maker is not None:
             self.blotter.commission_models[key].maker = maker
 
@@ -139,7 +138,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_slippage(self, spread=None):
-        key = self.blotter.slippage_models.keys()[0]
+        key = list(self.blotter.slippage_models.keys())[0]
         if spread is not None:
             self.blotter.slippage_models[key].spread = spread
 
@@ -271,9 +270,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
-        stats['positions'] = cum.position_tracker.get_positions_list()
-
         period = tracker.todays_performance
+        stats['positions'] = period.position_tracker.get_positions_list()
+
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
         stats['transactions'] = []
@@ -391,16 +390,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      'before exiting the algorithm.')
 
             algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
+            folder = join(algo_folder, 'daily_performance')
             files = [f for f in listdir(folder) if isfile(join(folder, f))]
 
             daily_perf_list = []
             for item in files:
                 filename = join(folder, item)
+
                 with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
+                    perf_period = pickle.load(handle)
+                    perf_period_dict = perf_period.to_dict()
+                    daily_perf_list.append(perf_period_dict)
 
             stats = pd.DataFrame(daily_perf_list)
+            stats.set_index('period_close', drop=False, inplace=True)
 
             self.analyze(stats)
 
@@ -460,43 +463,62 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def get_generator(self):
-        if self.trading_client is not None:
-            return self.trading_client.transform()
+    def _init_trading_client(self):
+        """
+        This replaces Ziplines `_create_generator` method. The main difference
+        is that we are restoring performance tracker objects if available.
+        This allows us to stop/start algos without loosing their state.
 
-        perf = None
+        """
         if self.perf_tracker is None:
+            # Note from the Zipline dev:
+            # HACK: When running with the `run` method, we set perf_tracker to
+            # None so that it will be overwritten here.
             tracker = self.perf_tracker = PerformanceTracker(
                 sim_params=self.sim_params,
                 trading_calendar=self.trading_calendar,
                 env=self.trading_environment,
             )
-
             # Set the dt initially to the period start by forcing it to change.
             self.on_dt_changed(self.sim_params.start_session)
 
+            new_position_tracker = tracker.position_tracker
+            tracker.position_tracker = None
+
             # Unpacking the perf_tracker and positions if available
-            perf = get_algo_object(
+            cum_perf = get_algo_object(
                 algo_name=self.algo_namespace,
                 key='cumulative_performance',
             )
+            if cum_perf is not None:
+                tracker.cumulative_performance = cum_perf
+                # Ensure single common position tracker
+                tracker.position_tracker = cum_perf.position_tracker
+
+            today = pd.Timestamp.utcnow().floor('1D')
+            todays_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                rel_path='daily_performance',
+            )
+            if todays_perf is not None:
+                # Ensure single common position tracker
+                if tracker.position_tracker is not None:
+                    todays_perf.position_tracker = tracker.position_tracker
+                else:
+                    tracker.position_tracker = todays_perf.position_tracker
+
+                tracker.todays_performance = todays_perf
+
+            if tracker.position_tracker is None:
+                # Use a new position_tracker if not is found in the state
+                tracker.position_tracker = new_position_tracker
 
         if not self.initialized:
+            # Calls the initialize function of the algorithm
             self.initialize(*self.initialize_args, **self.initialize_kwargs)
             self.initialized = True
 
-        # Call the simulation trading algorithm for side-effects:
-        # it creates the perf tracker
-        # TradingAlgorithm._create_generator(self, self.sim_params)
-        if perf is not None:
-            tracker.cumulative_performance = perf
-
-            period = self.perf_tracker.todays_performance
-            period.starting_cash = perf.ending_cash
-            period.starting_exposure = perf.ending_exposure
-            period.starting_value = perf.ending_value
-            period.position_tracker = perf.position_tracker
-
         self.trading_client = ExchangeAlgorithmExecutor(
             algo=self,
             sim_params=self.sim_params,
@@ -506,6 +528,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             restrictions=self.restrictions,
             universe_func=self._calculate_universe,
         )
+
+    def get_generator(self):
+        if self.trading_client is None:
+            self._init_trading_client()
+
         return self.trading_client.transform()
 
     def updated_portfolio(self):
@@ -523,10 +550,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         positions, returning the available cash, and raising error
         if the data goes out of sync.
 
-        Parameters
-        ----------
-        attempt_index: int
-
         Returns
         -------
         float
@@ -559,10 +582,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             if base_currency is None:
                 base_currency = exchange.base_currency
 
+            # Don't check the cash if there are open orders. This could
+            # results in false positives.
+            orders = []
+            for asset in self.blotter.open_orders:
+                asset_orders = self.blotter.open_orders[asset]
+                if asset_orders:
+                    orders += asset_orders
+
+            required_cash = self.portfolio.cash if not orders else None
             cash, positions_value = exchange.sync_positions(
                 positions=exchange_positions,
                 check_balances=check_balances,
-                cash=self.portfolio.cash,
+                cash=required_cash,
             )
             total_cash += cash
             total_positions_value += positions_value
@@ -676,11 +708,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.frame_stats = list()
 
         self.performance_needs_update = False
-        new_orders = self.perf_tracker.todays_performance.orders_by_id.keys()
-        if new_orders != self._last_orders:
+        orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
+        if orders != self._last_orders:
             self.performance_needs_update = True
 
-        self._last_orders = new_orders
+        # Saving current orders to detect changes in the next frame
+        self._last_orders = copy.deepcopy(orders)
 
         if self.performance_needs_update:
             self.perf_tracker.update_performance()
@@ -697,7 +730,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.portfolio_needs_update = False
 
         log.info(
-            'got totals from exchanges, cash: {} positions: {}'.format(
+            'portfolio balances, cash: {}, positions: {}'.format(
                 cash, positions_value
             )
         )
@@ -709,18 +742,29 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        self._save_algo_state(data)
+        self.current_day = data.current_dt.floor('1D')
+
+    def _save_algo_state(self, data):
+        today = data.current_dt.floor('1D')
         try:
             self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
+        log.debug('saving cumulative performance object')
         save_algo_object(
             algo_name=self.algo_namespace,
             key='cumulative_performance',
             obj=self.perf_tracker.cumulative_performance,
         )
-
-        self.current_day = data.current_dt.floor('1D')
+        log.debug('saving todays performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=self.perf_tracker.todays_performance,
+            rel_path='daily_performance'
+        )
 
     def _process_stats(self, data):
         today = data.current_dt.floor('1D')
@@ -763,12 +807,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             start_dt=today,
             end_dt=data.current_dt
         )
-        save_algo_object(
-            algo_name=self.algo_namespace,
-            key=today.strftime('%Y-%m-%d'),
-            obj=daily_stats,
-            rel_path='daily_perf'
-        )
 
         return recorded_cols
 

catalyst/exchange/exchange_asset_finder.py

@@ -1,8 +1,7 @@
 import pandas as pd
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.utils.factory import find_exchanges
+from logbook import Logger
 
 log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
 

catalyst/exchange/exchange_blotter.py

@@ -1,8 +1,5 @@
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from redo import retry
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
@@ -11,6 +8,8 @@ from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
+from logbook import Logger
+from redo import retry
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
@@ -42,6 +41,11 @@ class TradingPairFeeSchedule(CommissionModel):
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
@@ -55,8 +59,7 @@ class TradingPairFeeSchedule(CommissionModel):
         cost = abs(transaction.amount) * transaction.price
 
         asset = order.asset
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
+        maker, taker = self.get_maker_taker(asset)
 
         multiplier = taker
         if order.limit is not None:
@@ -251,6 +254,7 @@ class ExchangeBlotter(Blotter):
         for order, txn in self.check_open_orders():
             order.dt = txn.dt
 
+            # TODO: is the commission already on the order object?
             transactions.append(txn)
 
             if not order.open:

catalyst/exchange/exchange_bundle.py

@@ -8,12 +8,8 @@ from operator import is_not
 import numpy as np
 import pandas as pd
 import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from pytz import UTC
-from six import itervalues
-
 from catalyst import get_calendar
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
@@ -32,6 +28,9 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
+from logbook import Logger
+from pytz import UTC
+from six import itervalues
 
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 

catalyst/exchange/exchange_data_portal.py

@@ -3,9 +3,6 @@ import abc
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from redo import retry
-
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -14,6 +11,8 @@ from catalyst.exchange.exchange_errors import (
     PricingDataNotLoadedError)
 from catalyst.exchange.utils.exchange_utils import get_frequency, \
     resample_history_df, group_assets_by_exchange
+from logbook import Logger
+from redo import retry
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 

catalyst/exchange/exchange_portfolio.py

@@ -1,8 +1,7 @@
 import numpy as np
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from logbook import Logger
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 

catalyst/exchange/exchange_pricing_loader.py

@@ -11,12 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from logbook import Logger
-from numpy import (
-    iinfo,
-    uint32,
-)
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.us_equity_pricing import BcolzDailyBarReader
 from catalyst.errors import NoFurtherDataError
@@ -26,6 +20,11 @@ from catalyst.pipeline.data import DataSet, Column
 from catalyst.pipeline.loaders.base import PipelineLoader
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.numpy_utils import float64_dtype
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
 
 UINT32_MAX = iinfo(uint32).max
 

catalyst/exchange/live_graph_clock.py

@@ -1,13 +1,12 @@
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.stats_utils import prepare_stats
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.utils.stats_utils import prepare_stats
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 

catalyst/exchange/simple_clock.py

@@ -14,14 +14,13 @@
 from time import sleep
 
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 

catalyst/exchange/utils/bundle_utils.py

@@ -6,7 +6,6 @@ from datetime import timedelta, datetime, date
 import numpy as np
 import pandas as pd
 import pytz
-
 from catalyst.data.bundles.core import download_without_progress
 from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
 

catalyst/exchange/utils/exchange_utils.py

@@ -8,9 +8,6 @@ from datetime import date, datetime
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from six import string_types
-from six.moves.urllib import request
-
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
 from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
     InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
@@ -18,6 +15,8 @@ from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
     ExchangeJSONDecoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
+from six import string_types
+from six.moves.urllib import request
 
 
 def get_sid(symbol):

catalyst/exchange/utils/factory.py

@@ -1,13 +1,12 @@
 import os
 
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
 from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
     get_exchange_folder, is_blacklist
+from logbook import Logger
 
 log = Logger('factory', level=LOG_LEVEL)
 exchange_cache = dict()

catalyst/exchange/utils/serialization_utils.py

@@ -3,9 +3,8 @@ import re
 from json import JSONEncoder
 
 import pandas as pd
-from six import string_types
-
 from catalyst.constants import DATE_TIME_FORMAT
+from six import string_types
 
 
 class ExchangeJSONEncoder(json.JSONEncoder):

catalyst/exchange/utils/stats_utils.py

@@ -8,9 +8,9 @@ import time
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-
 from catalyst.exchange.utils.exchange_utils import get_algo_folder
 from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
 
 s3_conn = []
 mailgun = []
@@ -261,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()):
     return df, columns
 
 
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
@@ -281,17 +288,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     if isinstance(stats, pd.DataFrame):
         stats = stats.T.to_dict().values()
+        stats.sort(key=itemgetter('period_close'))
+
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
 
-    display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
     df, columns = prepare_stats(
         display_stats, recorded_cols=recorded_cols
     )
-
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
+    set_print_settings()
     return df.to_string(columns=columns)
 
 
@@ -439,6 +446,17 @@ def df_to_string(df):
     return df.to_string()
 
 
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
 def extract_transactions(perf):
     """
     Compute indexes for buy and sell transactions

catalyst/exchange/utils/test_utils.py

@@ -3,7 +3,6 @@ import random
 import tempfile
 
 from catalyst.assets._assets import TradingPair
-
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.factory import find_exchanges
 from catalyst.utils.paths import ensure_directory

catalyst/utils/run_algo.py

@@ -8,13 +8,12 @@ from time import sleep
 
 import click
 import pandas as pd
-from logbook import Logger
-
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
     TradingPairPricing
 from catalyst.exchange.utils.factory import get_exchange
+from logbook import Logger
 
 try:
     from pygments import highlight
@@ -40,9 +39,6 @@ from catalyst.exchange.exchange_algorithm import (
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, NotEnoughCapitalError)
 
 from catalyst.constants import LOG_LEVEL
 
@@ -70,7 +66,36 @@ class _RunAlgoError(click.ClickException, ValueError):
         return self.pyfunc_msg
 
 
-def _build_namespace(algotext, local_namespace, defines):
+def _run(handle_data,
+         initialize,
+         before_trading_start,
+         analyze,
+         algofile,
+         algotext,
+         defines,
+         data_frequency,
+         capital_base,
+         data,
+         bundle,
+         bundle_timestamp,
+         start,
+         end,
+         output,
+         print_algo,
+         local_namespace,
+         environ,
+         live,
+         exchange,
+         algo_namespace,
+         base_currency,
+         live_graph,
+         analyze_live,
+         simulate_orders,
+         stats_output):
+    """Run a backtest for the given algorithm.
+
+    This is shared between the cli and :func:`catalyst.run_algo`.
+    """
     if algotext is not None:
         if local_namespace:
             ip = get_ipython()  # noqa
@@ -84,197 +109,127 @@ def _build_namespace(algotext, local_namespace, defines):
             except ValueError:
                 raise ValueError(
                     'invalid define %r, should be of the form name=value' %
-                    assign)
+                    assign,
+                )
             try:
                 # evaluate in the same namespace so names may refer to
                 # eachother
                 namespace[name] = eval(value, namespace)
             except Exception as e:
                 raise ValueError(
-                    'failed to execute definition for name %r: %s' % (name, e))
+                    'failed to execute definition for name %r: %s' % (name, e),
+                )
     elif defines:
         raise _RunAlgoError(
             'cannot pass define without `algotext`',
-            "cannot pass '-D' / '--define' without '-t' / '--algotext'")
+            "cannot pass '-D' / '--define' without '-t' / '--algotext'",
+        )
     else:
         namespace = {}
+        if algofile is not None:
+            algotext = algofile.read()
 
-    return namespace
+    if print_algo:
+        if PYGMENTS:
+            highlight(
+                algotext,
+                PythonLexer(),
+                TerminalFormatter(),
+                outfile=sys.stdout,
+            )
+        else:
+            click.echo(algotext)
 
+    log.warn(
+        'Catalyst is currently in ALPHA. It is going through rapid '
+        'development and it is subject to errors. Please use carefully. '
+        'We encourage you to report any issue on GitHub: '
+        'https://github.com/enigmampc/catalyst/issues'
+    )
+    sleep(3)
 
-def _mode(simulate_orders, live):
-    if not live:
-        return 'backtest'
-    elif simulate_orders:
-        return 'paper-trading'
+    if live:
+        if simulate_orders:
+            mode = 'paper-trading'
+        else:
+            mode = 'live-trading'
     else:
-        return 'live-trading'
+        mode = 'backtest'
 
+    log.info('running algo in {mode} mode'.format(mode=mode))
 
-def _build_exchanges_dict(exchange, live, simulate_orders, base_currency):
     exchange_name = exchange
     if exchange_name is None:
         raise ValueError('Please specify at least one exchange.')
 
     exchange_list = [x.strip().lower() for x in exchange.split(',')]
 
-    exchanges = {exchange_name: get_exchange(
-                                    exchange_name=exchange_name,
-                                    base_currency=base_currency,
-                                    must_authenticate=(live and not simulate_orders))
-                 for exchange_name in exchange_list}
-
-    return exchanges
-
-
-def _pretty_print_code(algotext):
-    if PYGMENTS:
-        highlight(
-            algotext,
-            PythonLexer(),
-            TerminalFormatter(),
-            outfile=sys.stdout)
-    else:
-        click.echo(algotext)
-
-
-def _choose_loader(data_frequency, column):
-    bound_cols = TradingPairPricing.columns
-    if column in bound_cols:
-        return ExchangePricingLoader(data_frequency)
-    raise ValueError(
-        "No PipelineLoader registered for column %s." % column)
-
-
-def _get_live_time_range():
-    start = pd.Timestamp.utcnow()
-    # TODO: fix the end data.
-    end = start + timedelta(hours=8760)
-    return start, end
-
-
-def _data_for_live_trading(sim_params, exchanges, env, open_calendar):
-    data = DataPortalExchangeLive(
-        exchanges=exchanges,
-        asset_finder=env.asset_finder,
-        trading_calendar=open_calendar,
-        first_trading_day=pd.to_datetime('today', utc=True))
-
-    return data
-
-
-# TODO use proper retry here
-def _fetch_capital_base(base_currency, exchange_name, exchange,
-                        attempt_index=0):
-    """
-    Fetch the base currency amount required to bootstrap
-    the algorithm against the exchange.
-
-    The algorithm cannot continue without this value.
-
-    :param exchange: the targeted exchange
-    :param attempt_index:
-    :return capital_base: the amount of base currency available for
-    trading
-    """
-    try:
-        log.debug('retrieving capital base in {} to bootstrap '
-                  'exchange {}'.format(base_currency, exchange_name))
-        balances = exchange.get_balances()
-    except ExchangeRequestError as e:
-        if attempt_index < 20:
-            log.warn(
-                'could not retrieve balances on {}: {}'.format(
-                    exchange.name, e))
-            sleep(5)
-            return _fetch_capital_base(base_currency, exchange_name, exchange,
-                                       attempt_index + 1)
-
-        else:
-            raise ExchangeRequestErrorTooManyAttempts(
-                attempts=attempt_index,
-                error=e)
-
-    if base_currency in balances:
-        base_currency_available = balances[base_currency]['free']
-        log.info(
-            'base currency available in the account: {} {}'.format(
-                base_currency_available, base_currency))
-
-        return base_currency_available
-    else:
-        raise BaseCurrencyNotFoundError(
+    exchanges = dict()
+    for exchange_name in exchange_list:
+        exchanges[exchange_name] = get_exchange(
+            exchange_name=exchange_name,
             base_currency=base_currency,
-            exchange=exchange_name)
+            must_authenticate=(live and not simulate_orders),
+            skip_init=True,
+        )
 
+    open_calendar = get_calendar('OPEN')
 
-def _algorithm_class_for_live(algo_namespace, live_graph, stats_output,
-                              analyze_live, base_currency, simulate_orders,
-                              exchanges, capital_base):
-    if not simulate_orders:
-        for exchange_name in exchanges:
-            exchange = exchanges[exchange_name]
-            balance = _fetch_capital_base(base_currency, exchange_name,
-                                          exchange)
+    env = TradingEnvironment(
+        load=partial(
+            load_crypto_market_data,
+            environ=environ,
+            start_dt=start,
+            end_dt=end
+        ),
+        environ=environ,
+        exchange_tz='UTC',
+        asset_db_path=None  # We don't need an asset db, we have exchanges
+    )
+    env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
 
-            if balance < capital_base:
-                raise NotEnoughCapitalError(
-                    exchange=exchange_name,
-                    base_currency=base_currency,
-                    balance=balance,
-                    capital_base=capital_base)
-
-    algorithm_class = partial(
-        ExchangeTradingAlgorithmLive,
-        exchanges=exchanges,
-        algo_namespace=algo_namespace,
-        live_graph=live_graph,
-        simulate_orders=simulate_orders,
-        stats_output=stats_output,
-        analyze_live=analyze_live,)
-
-    return algorithm_class
-
-
-def _bundle_trading_environment(bundle_data, environ):
-    prefix, connstr = re.split(
-        r'sqlite:///',
-        str(bundle_data.asset_finder.engine.url),
-        maxsplit=1)
-    if prefix:
+    def choose_loader(column):
+        bound_cols = TradingPairPricing.columns
+        if column in bound_cols:
+            return ExchangePricingLoader(data_frequency)
         raise ValueError(
-            "invalid url %r, must begin with 'sqlite:///'" %
-            str(bundle_data.asset_finder.engine.url))
+            "No PipelineLoader registered for column %s." % column
+        )
 
-    return TradingEnvironment(asset_db_path=connstr, environ=environ)
+    if live:
+        start = pd.Timestamp.utcnow()
 
+        # TODO: fix the end data.
+        end = start + timedelta(hours=8760)
 
-def _build_live_algo_and_data(sim_params, exchanges, env, open_calendar,
-                              simulate_orders, algo_namespace, capital_base,
-                              live_graph, stats_output, analyze_live,
-                              base_currency, namespace, choose_loader,
-                              algorithm_class_kwargs):
-    sim_params._arena = 'live'  # TODO: use the constructor instead
+        data = DataPortalExchangeLive(
+            exchanges=exchanges,
+            asset_finder=env.asset_finder,
+            trading_calendar=open_calendar,
+            first_trading_day=pd.to_datetime('today', utc=True)
+        )
 
-    data = _data_for_live_trading(sim_params, exchanges, env, open_calendar)
+        sim_params = create_simulation_parameters(
+            start=start,
+            end=end,
+            capital_base=capital_base,
+            emission_rate='minute',
+            data_frequency='minute'
+        )
 
-    algorithm_class = _algorithm_class_for_live(
-        algo_namespace, live_graph, stats_output, analyze_live,
-        base_currency, simulate_orders, exchanges, capital_base)
+        # TODO: use the constructor instead
+        sim_params._arena = 'live'
 
-    return data, algorithm_class(
-        namespace=namespace,
-        env=env,
-        get_pipeline_loader=choose_loader,
-        sim_params=sim_params,
-        **algorithm_class_kwargs)
-
-
-def _build_backtest_algo_and_data(
-        exchanges, bundle, env, environ, bundle_timestamp, open_calendar,
-        start, end, namespace, choose_loader, sim_params,
-        algorithm_class_kwargs):
-    if exchanges:
+        algorithm_class = partial(
+            ExchangeTradingAlgorithmLive,
+            exchanges=exchanges,
+            algo_namespace=algo_namespace,
+            live_graph=live_graph,
+            simulate_orders=simulate_orders,
+            stats_output=stats_output,
+            analyze_live=analyze_live,
+        )
+    elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
         # We can add back the complexity if required.
 
@@ -288,19 +243,41 @@ def _build_backtest_algo_and_data(
             asset_finder=None,
             trading_calendar=open_calendar,
             first_trading_day=start,
-            last_available_session=end)
+            last_available_session=end
+        )
+
+        sim_params = create_simulation_parameters(
+            start=start,
+            end=end,
+            capital_base=capital_base,
+            data_frequency=data_frequency,
+            emission_rate=data_frequency,
+        )
 
         algorithm_class = partial(
             ExchangeTradingAlgorithmBacktest,
-            exchanges=exchanges)
+            exchanges=exchanges
+        )
+
     elif bundle is not None:
-        # TODO This branch should probably be removed or fixed: it doesn't even
-        # build `algorithm_class`, so it will break when trying to instantiate
-        # it.
-        bundle_data = load(bundle, environ, bundle_timestamp)
+        bundle_data = load(
+            bundle,
+            environ,
+            bundle_timestamp,
+        )
 
-        env = _bundle_trading_environment(bundle_data, environ)
+        prefix, connstr = re.split(
+            r'sqlite:///',
+            str(bundle_data.asset_finder.engine.url),
+            maxsplit=1,
+        )
+        if prefix:
+            raise ValueError(
+                "invalid url %r, must begin with 'sqlite:///'" %
+                str(bundle_data.asset_finder.engine.url),
+            )
 
+        env = TradingEnvironment(asset_db_path=connstr, environ=environ)
         first_trading_day = \
             bundle_data.equity_minute_bar_reader.first_trading_day
 
@@ -309,103 +286,27 @@ def _build_backtest_algo_and_data(
             first_trading_day=first_trading_day,
             equity_minute_reader=bundle_data.equity_minute_bar_reader,
             equity_daily_reader=bundle_data.equity_daily_bar_reader,
-            adjustment_reader=bundle_data.adjustment_reader)
+            adjustment_reader=bundle_data.adjustment_reader,
+        )
 
-    return data, algorithm_class(
+    perf = algorithm_class(
         namespace=namespace,
         env=env,
         get_pipeline_loader=choose_loader,
         sim_params=sim_params,
-        **algorithm_class_kwargs)
-
-
-def _build_algo_and_data(handle_data, initialize, before_trading_start,
-                         analyze, algofile, algotext, defines, data_frequency,
-                         capital_base, data, bundle, bundle_timestamp, start,
-                         end, output, print_algo, local_namespace, environ,
-                         live, exchange, algo_namespace, base_currency,
-                         live_graph, analyze_live, simulate_orders,
-                         stats_output):
-    namespace = _build_namespace(algotext, local_namespace, defines)
-    if algotext is not None:
-        algotext = algofile.read()
-
-    if print_algo:
-        _pretty_print_code(algotext)
-
-    mode = _mode(simulate_orders, live)
-    log.info('running algo in {mode} mode'.format(mode=mode))
-
-    exchanges = _build_exchanges_dict(exchange, live, simulate_orders,
-                                      base_currency)
-
-    open_calendar = get_calendar('OPEN')
-
-    env = TradingEnvironment(
-        load=partial(load_crypto_market_data, environ=environ, start_dt=start,
-                     end_dt=end),
-        environ=environ,
-        exchange_tz='UTC',
-        asset_db_path=None)  # We don't need an asset db, we have exchanges
-
-    env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
-
-    choose_loader = partial(_choose_loader, data_frequency)
-
-    if live:
-        start, end = _get_live_time_range()
-        data_frequency = 'minute'  # TODO double check if this is the desired behavior
-
-    sim_params = create_simulation_parameters(
-        start=start,
-        end=end,
-        capital_base=capital_base,
-        emission_rate=data_frequency,
-        data_frequency=data_frequency)
-
-    if algotext is None:
-        algorithm_class_kwargs = {'initialize': initialize,
-                                  'handle_data': handle_data,
-                                  'before_trading_start': before_trading_start,
-                                  'analyze': analyze}
-    else:
-        algorithm_class_kwargs = {'algo_filename': getattr(algofile, 'name',
-                                                           '<algorithm>'),
-                                  'script': algotext}
-
-    if live:
-        return _build_live_algo_and_data(
-            sim_params, exchanges, env, open_calendar, simulate_orders,
-            algo_namespace, capital_base, live_graph, stats_output,
-            analyze_live, base_currency, namespace, choose_loader,
-            algorithm_class_kwargs)
-    else:
-        return _build_backtest_algo_and_data(
-            exchanges, bundle, env, environ, bundle_timestamp, open_calendar,
-            start, end, namespace, choose_loader, sim_params,
-            algorithm_class_kwargs)
-
-
-def _run(handle_data, initialize, before_trading_start, analyze, algofile,
-         algotext, defines, data_frequency, capital_base, data, bundle,
-         bundle_timestamp, start, end, output, print_algo, local_namespace,
-         environ, live, exchange, algo_namespace, base_currency, live_graph,
-         analyze_live, simulate_orders, stats_output):
-    """Run an algorithm in backtest,
-    paper-trading or live-trading mode.
-
-    This is shared between the cli and :func:`catalyst.run_algo`.
-    """
-
-    data, algorithm = _build_algo_and_data(
-        handle_data, initialize, before_trading_start, analyze, algofile,
-        algotext, defines, data_frequency, capital_base, data, bundle,
-        bundle_timestamp, start, end, output, print_algo, local_namespace,
-        environ, live, exchange, algo_namespace, base_currency, live_graph,
-        analyze_live, simulate_orders, stats_output)
-    perf = algorithm.run(
+        **{
+            'initialize': initialize,
+            'handle_data': handle_data,
+            'before_trading_start': before_trading_start,
+            'analyze': analyze,
+        } if algotext is None else {
+            'algo_filename': getattr(algofile, 'name', '<algorithm>'),
+            'script': algotext,
+        }
+    ).run(
         data,
-        overwrite_sim_params=False)
+        overwrite_sim_params=False,
+    )
 
     if output == '-':
         click.echo(str(perf))
@@ -462,7 +363,8 @@ def load_extensions(default, extensions, strict, environ, reload=False):
             # without `strict` we should just log the failure
             warnings.warn(
                 'Failed to load extension: %r\n%s' % (ext, e),
-                stacklevel=2)
+                stacklevel=2
+            )
         else:
             _loaded_extensions.add(ext)
 
@@ -561,7 +463,8 @@ def run_algorithm(initialize,
     catalyst.data.bundles.bundles : The available data bundles.
     """
     load_extensions(
-        default_extension, extensions, strict_extensions, environ)
+        default_extension, extensions, strict_extensions, environ
+    )
 
     if capital_base is None:
         raise ValueError(
@@ -569,7 +472,8 @@ def run_algorithm(initialize,
             'amount of base currency available for trading. For example, '
             'if the `capital_base` is 5ETH, the '
             '`order_target_percent(asset, 1)` command will order 5ETH worth '
-            'of the specified asset.')
+            'of the specified asset.'
+        )
     # I'm not sure that we need this since the modified DataPortal
     # does not require extensions to be explicitly loaded.
 
@@ -587,11 +491,13 @@ def run_algorithm(initialize,
         elif len(non_none_data) != 1:
             raise ValueError(
                 'must specify one of `data`, `data_portal`, or `bundle`,'
-                ' got: %r' % non_none_data)
+                ' got: %r' % non_none_data,
+            )
 
         elif 'bundle' not in non_none_data and bundle_timestamp is not None:
             raise ValueError(
-                'cannot specify `bundle_timestamp` without passing `bundle`')
+                'cannot specify `bundle_timestamp` without passing `bundle`',
+            )
     return _run(
         handle_data=handle_data,
         initialize=initialize,
@@ -618,4 +524,5 @@ def run_algorithm(initialize,
         live_graph=live_graph,
         analyze_live=analyze_live,
         simulate_orders=simulate_orders,
-        stats_output=stats_output)
+        stats_output=stats_output
+    )

docs/source/releases.rst

@@ -2,9 +2,28 @@
 Release Notes
 =============
 
-Version 0.4.1
+Version 0.4.4
 ^^^^^^^^^^^^^
-**Release Date**: 2017-01-03
+**Release Date**: 2018-01-09
+
+Bug Fixes
+~~~~~~~~~
+- Removed redundant capital_base validation (:issue:`142`)
+- Fixed portfolio update issue with restored state (:issue:`111`)
+- Skipping cash validation where there are open orders (:issue:`144`)
+
+Version 0.4.3
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-05
+
+Bug Fixes
+~~~~~~~~~
+- Fixed CLI issue (:issue:`137`)
+- Upgraded CCXT
+
+Version 0.4.2
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-03
 
 Bug Fixes
 ~~~~~~~~~
@@ -39,7 +58,7 @@ Build
 - Added market orders in live mode (:issue:`81`)
 
 Version 0.3.10
-^^^^^^^^^^^^^
+~~~~~~~~~~~~~~
 **Release Date**: 2017-11-28
 
 Bug Fixes

etc/python2.7-environment.yml

@@ -20,7 +20,7 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
-  - ccxt==1.10.283
+  - ccxt==1.10.565
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0

etc/requirements.txt

@@ -81,6 +81,6 @@ empyrical==0.2.1
 tables==3.3.0
 
 #Catalyst dependencies
-ccxt==1.10.283
+ccxt==1.10.565
 boto3==1.4.8
 redo==1.6

tests/exchange/test_ccxt.py

@@ -19,16 +19,16 @@ class TestCCXT(BaseExchangeTestCase):
             exchange_name=exchange_name,
             key=auth['key'],
             secret=auth['secret'],
-            base_currency='eth',
+            base_currency='bnb',
         )
         self.exchange.init()
 
     def test_order(self):
         log.info('creating order')
-        asset = self.exchange.get_asset('neo_eth')
+        asset = self.exchange.get_asset('neo_bnb')
         order_id = self.exchange.order(
             asset=asset,
-            style=ExchangeLimitOrder(limit_price=0.7),
+            style=ExchangeLimitOrder(limit_price=10),
             amount=1,
         )
         log.info('order created {}'.format(order_id))

tests/exchange/test_suites/test_suite_algo.py

@@ -0,0 +1,72 @@
+import importlib
+from os.path import join, isfile
+
+import pandas as pd
+import os
+
+from catalyst import run_algorithm
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
+    extract_transactions, set_print_settings, extract_orders
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+from logbook import TestHandler, WARNING
+from pathtools.path import listdir
+
+filter_algos = [
+    'mean_reversion_simple_custom_fees.py',
+]
+
+
+class TestSuiteAlgo(WithLogger, ZiplineTestCase):
+    @staticmethod
+    def analyze(context, perf):
+        set_print_settings()
+
+        transaction_df = extract_transactions(perf)
+        print('the transactions:\n{}'.format(transaction_df))
+
+        orders_df = extract_orders(perf)
+        print('the orders:\n{}'.format(orders_df))
+
+        stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
+        print('the stats:\n{}'.format(stats))
+        pass
+
+    def test_run_examples(self):
+        folder = join('..', '..', '..', 'catalyst', 'examples')
+        files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+        algo_list = []
+        for filename in files:
+            name = os.path.basename(filename)
+            if filter_algos and name not in filter_algos:
+                continue
+
+            module_name = 'catalyst.examples.{}'.format(
+                name.replace('.py', '')
+            )
+            algo_list.append(module_name)
+
+        for module_name in algo_list:
+            algo = importlib.import_module(module_name)
+            namespace = module_name.replace('.', '_')
+
+            log_catcher = TestHandler()
+            with log_catcher:
+                run_algorithm(
+                    capital_base=0.1,
+                    data_frequency='minute',
+                    initialize=algo.initialize,
+                    handle_data=algo.handle_data,
+                    analyze=TestSuiteAlgo.analyze,
+                    exchange_name='bitfinex',
+                    algo_namespace='test_{}'.format(namespace),
+                    base_currency='eth',
+                    start=pd.to_datetime('2017-10-01', utc=True),
+                    end=pd.to_datetime('2017-10-02', utc=True),
+                    # output=out
+                )
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+                self.assertEqual(0, len(warnings))
+
+            pass

tests/exchange/test_suites/test_suite_bundle.py

@@ -1,7 +1,7 @@
 import random
 
 import pandas as pd
-from logbook import Logger
+from logbook import TestHandler
 from pandas.util.testing import assert_frame_equal
 
 from catalyst import get_calendar
@@ -11,8 +11,7 @@ from catalyst.exchange.utils.exchange_utils import get_candles_df
 from catalyst.exchange.utils.factory import get_exchange
 from catalyst.exchange.utils.test_utils import output_df, \
     select_random_assets
-
-log = Logger('TestSuiteExchange')
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
 
 pd.set_option('display.expand_frame_repr', False)
 pd.set_option('precision', 8)
@@ -20,7 +19,7 @@ pd.set_option('display.width', 1000)
 pd.set_option('display.max_colwidth', 1000)
 
 
-class TestSuiteBundle:
+class TestSuiteBundle(WithLogger, ZiplineTestCase):
     @staticmethod
     def get_data_portal(exchange_names):
         open_calendar = get_calendar('OPEN')
@@ -54,46 +53,46 @@ class TestSuiteBundle:
         """
         data = dict()
 
-        log.info('creating data sample from bundle')
-        data['bundle'] = data_portal.get_history_window(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=bar_count,
-            frequency=freq,
-            field='close',
-            data_frequency=data_frequency,
-        )
-        log.info('bundle data:\n{}'.format(
-            data['bundle'].tail(10))
-        )
+        log_catcher = TestHandler()
+        with log_catcher:
+            data['bundle'] = data_portal.get_history_window(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                frequency=freq,
+                field='close',
+                data_frequency=data_frequency,
+            )
+            print('bundle data:\n{}'.format(
+                data['bundle'].tail(10))
+            )
 
-        log.info('creating data sample from exchange api')
-        candles = exchange.get_candles(
-            end_dt=end_dt,
-            freq=freq,
-            assets=assets,
-            bar_count=bar_count,
-        )
-        data['exchange'] = get_candles_df(
-            candles=candles,
-            field='close',
-            freq=freq,
-            bar_count=bar_count,
-            end_dt=end_dt,
-        )
-        log.info('exchange data:\n{}'.format(
-            data['exchange'].tail(10))
-        )
-        for source in data:
-            df = data[source]
-            path = output_df(df, assets, '{}_{}'.format(freq, source))
-            log.info('saved {}:\n{}'.format(source, path))
+            candles = exchange.get_candles(
+                end_dt=end_dt,
+                freq=freq,
+                assets=assets,
+                bar_count=bar_count,
+            )
+            data['exchange'] = get_candles_df(
+                candles=candles,
+                field='close',
+                freq=freq,
+                bar_count=bar_count,
+                end_dt=end_dt,
+            )
+            print('exchange data:\n{}'.format(
+                data['exchange'].tail(10))
+            )
+            for source in data:
+                df = data[source]
+                path = output_df(df, assets, '{}_{}'.format(freq, source))
+                print('saved {}:\n{}'.format(source, path))
 
-        assert_frame_equal(
-            right=data['bundle'],
-            left=data['exchange'],
-            check_less_precise=True,
-        )
+            assert_frame_equal(
+                right=data['bundle'],
+                left=data['exchange'],
+                check_less_precise=True,
+            )
 
     def test_validate_bundles(self):
         # exchange_population = 3

tests/exchange/test_suites/test_suite_exchange.py

@@ -1,21 +1,25 @@
 import json
 import os
 import random
-from logging import Logger
+from logging import Logger, WARNING
 from time import sleep
 
 import pandas as pd
+from catalyst.assets._assets import TradingPair
+from logbook import TestHandler
 
 from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.test_utils import select_random_exchanges, \
     handle_exchange_error, select_random_assets
+from catalyst.testing import ZiplineTestCase
+from catalyst.testing.fixtures import WithLogger
 
 log = Logger('TestSuiteExchange')
 
 
-class TestSuiteExchange:
+class TestSuiteExchange(WithLogger, ZiplineTestCase):
     def _test_markets_exchange(self, exchange, attempts=0):
         assets = None
         try:
@@ -156,34 +160,47 @@ class TestSuiteExchange:
             base_currency=quote_currency,
         )  # Type: list[Exchange]
 
-        for exchange in exchanges:
-            exchange.init()
+        log_catcher = TestHandler()
+        with log_catcher:
+            for exchange in exchanges:
+                exchange.init()
 
-            assets = exchange.get_assets(quote_currency=quote_currency)
-            asset = select_random_assets(assets, 1)[0]
-            assert asset
+                assets = exchange.get_assets(quote_currency=quote_currency)
+                asset = select_random_assets(assets, 1)[0]
+                self.assertIsInstance(asset, TradingPair)
 
-            tickers = exchange.tickers([asset])
-            price = tickers[asset]['last_price']
+                tickers = exchange.tickers([asset])
+                price = tickers[asset]['last_price']
 
-            amount = order_amount / price
+                amount = order_amount / price
 
-            limit_price = price * 0.8
-            style = ExchangeLimitOrder(limit_price=limit_price)
+                limit_price = price * 0.8
+                style = ExchangeLimitOrder(limit_price=limit_price)
 
-            order = exchange.order(
-                asset=asset,
-                amount=amount,
-                style=style,
-            )
-            sleep(1)
+                order = exchange.order(
+                    asset=asset,
+                    amount=amount,
+                    style=style,
+                )
+                sleep(1)
 
-            open_order, _ = exchange.get_order(order.id, asset)
-            assert open_order.status == 0
+                open_order, _ = exchange.get_order(order.id, asset)
+                self.assertEqual(0, open_order.status)
 
-            exchange.cancel_order(open_order, asset)
-            sleep(1)
+                exchange.cancel_order(open_order, asset)
+                sleep(1)
 
-            canceled_order, _ = exchange.get_order(open_order.id, asset)
-            assert canceled_order.status == 2
+                canceled_order, _ = exchange.get_order(open_order.id, asset)
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+
+                self.assertEqual(0, len(warnings))
+                self.assertEqual(2, canceled_order.status)
+                print(
+                    'tested {exchange} / {symbol}, order: {order}'.format(
+                        exchange=exchange.name,
+                        symbol=asset.symbol,
+                        order=order.id,
+                    )
+                )
         pass