From 215de33c35c79241de37c80d77fd299ba9ba9a14 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Fri, 5 Jan 2018 02:46:11 -0500
Subject: [PATCH 01/22] BUG: Fixed issue with updating positions after
 restoring the state of an algo

---
 catalyst/exchange/exchange_algorithm.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index df30b368..9365d11d 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -271,9 +271,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
-        stats['positions'] = cum.position_tracker.get_positions_list()
-
         period = tracker.todays_performance
+        stats['positions'] = period.position_tracker.get_positions_list()
+
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
         stats['transactions'] = []
@@ -495,6 +495,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             period.starting_cash = perf.ending_cash
             period.starting_exposure = perf.ending_exposure
             period.starting_value = perf.ending_value
+            # This does not seem to get updated correctly
             period.position_tracker = perf.position_tracker
 
         self.trading_client = ExchangeAlgorithmExecutor(

From 13de3e69efdde22e1001b01019d07bfbe07453ba Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Fri, 5 Jan 2018 22:22:04 -0500
Subject: [PATCH 02/22] BLD: refined CCXT error handlers

---
 catalyst/exchange/ccxt/ccxt_exchange.py | 65 ++++++++++++++++++-------
 1 file changed, 47 insertions(+), 18 deletions(-)

diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py
index f1107ade..ed3b2b24 100644
--- a/catalyst/exchange/ccxt/ccxt_exchange.py
+++ b/catalyst/exchange/ccxt/ccxt_exchange.py
@@ -6,12 +6,8 @@ from collections import defaultdict
 import ccxt
 import pandas as pd
 import six
-from catalyst.assets._assets import TradingPair
-from ccxt import ExchangeNotAvailable, InvalidOrder
-from logbook import Logger
-from six import string_types
-
 from catalyst.algorithm import MarketOrder
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -23,6 +19,10 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
     from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
     get_exchange_auth
 from catalyst.finance.order import Order, ORDER_STATUS
+from ccxt import InvalidOrder, NetworkError, \
+    ExchangeError
+from logbook import Logger
+from six import string_types
 
 log = Logger('CCXT', level=LOG_LEVEL)
 
@@ -105,7 +105,12 @@ class CCXT(Exchange):
                 with open(filename, 'w+') as f:
                     json.dump(self.markets, f, indent=4)
 
-            except ExchangeNotAvailable as e:
+            except (ExchangeError, NetworkError) as e:
+                log.warn(
+                    'unable to fetch markets {}: {}'.format(
+                        self.name, e
+                    )
+                )
                 raise ExchangeRequestError(error=e)
 
         self.load_assets()
@@ -408,6 +413,7 @@ class CCXT(Exchange):
     def _fetch_symbol_map(self, is_local):
         try:
             return self.fetch_symbol_map(is_local)
+
         except ExchangeSymbolsNotFound:
             return None
 
@@ -559,8 +565,12 @@ class CCXT(Exchange):
             for key in balances:
                 balances_lower[key.lower()] = balances[key]
 
-        except Exception as e:
-            log.debug('error retrieving balances: {}', e)
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch balance {}: {}'.format(
+                    self.name, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return balances_lower
@@ -703,14 +713,18 @@ class CCXT(Exchange):
                 amount=adj_amount,
                 price=price
             )
-        except ExchangeNotAvailable as e:
-            log.debug('unable to create order: {}'.format(e))
-            raise ExchangeRequestError(error=e)
-
         except InvalidOrder as e:
             log.warn('the exchange rejected the order: {}'.format(e))
             raise CreateOrderError(exchange=self.name, error=e)
 
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to create order {} / {}: {}'.format(
+                    self.name, symbol, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
         if 'info' not in result:
             raise ValueError('cannot use order without info attribute')
 
@@ -735,7 +749,12 @@ class CCXT(Exchange):
                 limit=None,
                 params=dict()
             )
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch open orders {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         orders = []
@@ -758,7 +777,12 @@ class CCXT(Exchange):
             order_status = self.api.fetch_order(id=order_id, symbol=symbol)
             order, executed_price = self._create_order(order_status)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return order, executed_price
@@ -777,7 +801,12 @@ class CCXT(Exchange):
                 if asset_or_symbol is not None else None
             self.api.cancel_order(id=order_id, symbol=symbol)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to cancel order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
     def tickers(self, assets):
@@ -828,10 +857,10 @@ class CCXT(Exchange):
 
                 tickers[asset] = ticker
 
-        except ExchangeNotAvailable as e:
+        except (ExchangeError, NetworkError) as e:
             log.warn(
-                'unable to fetch ticker: {} {}'.format(
-                    self.name, asset.symbol
+                'unable to fetch ticker {} / {}: {}'.format(
+                    self.name, asset.symbol, e
                 )
             )
             raise ExchangeRequestError(error=e)

From 96b36c6614ae669805c871f26224afcbccd10f75 Mon Sep 17 00:00:00 2001
From: danim7 <danim7@users.noreply.github.com>
Date: Sat, 6 Jan 2018 23:55:07 +0100
Subject: [PATCH 03/22] DOC: wrong year: 2017 --> 2018

---
 docs/source/releases.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/releases.rst b/docs/source/releases.rst
index e095d0b4..5f642dfe 100644
--- a/docs/source/releases.rst
+++ b/docs/source/releases.rst
@@ -4,7 +4,7 @@ Release Notes
 
 Version 0.4.3
 ^^^^^^^^^^^^^
-**Release Date**: 2017-01-05
+**Release Date**: 2018-01-05
 
 Bug Fixes
 ~~~~~~~~~
@@ -13,7 +13,7 @@ Bug Fixes
 
 Version 0.4.2
 ^^^^^^^^^^^^^
-**Release Date**: 2017-01-03
+**Release Date**: 2018-01-03
 
 Bug Fixes
 ~~~~~~~~~

From 5d9708901d7b964919f02f7fd7939d19b5ed7a86 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sat, 6 Jan 2018 19:08:47 -0500
Subject: [PATCH 04/22] BUG: fixed issue #111 related to positions update after
 restoring algo state

---
 catalyst/examples/mean_reversion_simple.py |  6 +-
 catalyst/exchange/exchange_algorithm.py    | 99 ++++++++++++++--------
 2 files changed, 69 insertions(+), 36 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 14f7ec99..e46b8df6 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -37,8 +37,8 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 55
-    context.RSI_OVERBOUGHT = 65
+    context.RSI_OVERSOLD = 35
+    context.RSI_OVERBOUGHT = 50
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
@@ -248,7 +248,7 @@ if __name__ == '__main__':
 
     if live:
         run_algorithm(
-            capital_base=0.03,
+            capital_base=0.025,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 9365d11d..8e369f8e 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -391,16 +391,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      'before exiting the algorithm.')
 
             algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
+            folder = join(algo_folder, 'daily_performance')
             files = [f for f in listdir(folder) if isfile(join(folder, f))]
 
             daily_perf_list = []
             for item in files:
                 filename = join(folder, item)
+
                 with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
+                    perf_period = pickle.load(handle)
+                    perf_period_dict = perf_period.to_dict()
+                    daily_perf_list.append(perf_period_dict)
 
             stats = pd.DataFrame(daily_perf_list)
+            stats.set_index('period_close', drop=False, inplace=True)
 
             self.analyze(stats)
 
@@ -460,44 +464,62 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def get_generator(self):
-        if self.trading_client is not None:
-            return self.trading_client.transform()
+    def _init_trading_client(self):
+        """
+        This replaces Ziplines `_create_generator` method. The main difference
+        is that we are restoring performance tracker objects if available.
+        This allows us to stop/start algos without loosing their state.
 
-        perf = None
+        """
         if self.perf_tracker is None:
+            # Note from the Zipline dev:
+            # HACK: When running with the `run` method, we set perf_tracker to
+            # None so that it will be overwritten here.
             tracker = self.perf_tracker = PerformanceTracker(
                 sim_params=self.sim_params,
                 trading_calendar=self.trading_calendar,
                 env=self.trading_environment,
             )
-
             # Set the dt initially to the period start by forcing it to change.
             self.on_dt_changed(self.sim_params.start_session)
 
+            new_position_tracker = tracker.position_tracker
+            tracker.position_tracker = None
+
             # Unpacking the perf_tracker and positions if available
-            perf = get_algo_object(
+            cum_perf = get_algo_object(
                 algo_name=self.algo_namespace,
                 key='cumulative_performance',
             )
+            if cum_perf is not None:
+                tracker.cumulative_performance = cum_perf
+                # Ensure single common position tracker
+                tracker.position_tracker = cum_perf.position_tracker
+
+            today = pd.Timestamp.utcnow().floor('1D')
+            todays_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                rel_path='daily_performance',
+            )
+            if todays_perf is not None:
+                # Ensure single common position tracker
+                if tracker.position_tracker is not None:
+                    todays_perf.position_tracker = tracker.position_tracker
+                else:
+                    tracker.position_tracker = todays_perf.position_tracker
+
+                tracker.todays_performance = todays_perf
+
+            if tracker.position_tracker is None:
+                # Use a new position_tracker if not is found in the state
+                tracker.position_tracker = new_position_tracker
 
         if not self.initialized:
+            # Calls the initialize function of the algorithm
             self.initialize(*self.initialize_args, **self.initialize_kwargs)
             self.initialized = True
 
-        # Call the simulation trading algorithm for side-effects:
-        # it creates the perf tracker
-        # TradingAlgorithm._create_generator(self, self.sim_params)
-        if perf is not None:
-            tracker.cumulative_performance = perf
-
-            period = self.perf_tracker.todays_performance
-            period.starting_cash = perf.ending_cash
-            period.starting_exposure = perf.ending_exposure
-            period.starting_value = perf.ending_value
-            # This does not seem to get updated correctly
-            period.position_tracker = perf.position_tracker
-
         self.trading_client = ExchangeAlgorithmExecutor(
             algo=self,
             sim_params=self.sim_params,
@@ -507,6 +529,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             restrictions=self.restrictions,
             universe_func=self._calculate_universe,
         )
+
+    def get_generator(self):
+        if self.trading_client is None:
+            self._init_trading_client()
+
         return self.trading_client.transform()
 
     def updated_portfolio(self):
@@ -677,11 +704,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.frame_stats = list()
 
         self.performance_needs_update = False
-        new_orders = self.perf_tracker.todays_performance.orders_by_id.keys()
-        if new_orders != self._last_orders:
+        orders = self.perf_tracker.todays_performance.orders_by_id.keys()
+        if orders != self._last_orders:
             self.performance_needs_update = True
 
-        self._last_orders = copy.deepcopy(new_orders)
+        # Saving current orders to detect changes in the next frame
+        self._last_orders = copy.deepcopy(orders)
 
         if self.performance_needs_update:
             self.perf_tracker.update_performance()
@@ -698,7 +726,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.portfolio_needs_update = False
 
         log.info(
-            'got totals from exchanges, cash: {} positions: {}'.format(
+            'portfolio balances, cash: {}, positions: {}'.format(
                 cash, positions_value
             )
         )
@@ -710,18 +738,29 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        self._save_algo_state(data)
+        self.current_day = data.current_dt.floor('1D')
+
+    def _save_algo_state(self, data):
+        today = data.current_dt.floor('1D')
         try:
             self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
+        log.debug('saving cumulative performance object')
         save_algo_object(
             algo_name=self.algo_namespace,
             key='cumulative_performance',
             obj=self.perf_tracker.cumulative_performance,
         )
-
-        self.current_day = data.current_dt.floor('1D')
+        log.debug('saving todays performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=self.perf_tracker.todays_performance,
+            rel_path='daily_performance'
+        )
 
     def _process_stats(self, data):
         today = data.current_dt.floor('1D')
@@ -764,12 +803,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             start_dt=today,
             end_dt=data.current_dt
         )
-        save_algo_object(
-            algo_name=self.algo_namespace,
-            key=today.strftime('%Y-%m-%d'),
-            obj=daily_stats,
-            rel_path='daily_perf'
-        )
 
         return recorded_cols
 

From ba0208910f59de83c7adbc214888a02e16e25bef Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sat, 6 Jan 2018 19:37:53 -0500
Subject: [PATCH 05/22] BUG: Fixed issue #142 by removing unecessary
 capital_base check since we are now validating positions and cash against the
 exchange

---
 catalyst/exchange/exchange_algorithm.py   |  4 --
 catalyst/exchange/utils/exchange_utils.py | 14 ++++++
 catalyst/utils/run_algo.py                | 61 +----------------------
 3 files changed, 15 insertions(+), 64 deletions(-)

diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 8e369f8e..efd10553 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -551,10 +551,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         positions, returning the available cash, and raising error
         if the data goes out of sync.
 
-        Parameters
-        ----------
-        attempt_index: int
-
         Returns
         -------
         float
diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py
index ab16f71d..ebc678d2 100644
--- a/catalyst/exchange/utils/exchange_utils.py
+++ b/catalyst/exchange/utils/exchange_utils.py
@@ -733,3 +733,17 @@ def get_candles_df(candles, field, freq, bar_count, end_dt,
     df.dropna(inplace=True)
 
     return df
+
+
+def fetch_capital_base(exchange):
+    """
+    Fetch the base currency amount required to bootstrap
+    the algorithm against the exchange.
+
+    The algorithm cannot continue without this value.
+
+    :param exchange: the targeted exchange
+    :param attempt_index:
+    :return capital_base: the amount of base currency available for
+    trading
+    """
diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py
index 015f91e4..af67c29f 100644
--- a/catalyst/utils/run_algo.py
+++ b/catalyst/utils/run_algo.py
@@ -15,6 +15,7 @@ from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
     TradingPairPricing
 from catalyst.exchange.utils.factory import get_exchange
+from redo import retry
 
 try:
     from pygments import highlight
@@ -199,66 +200,6 @@ def _run(handle_data,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
 
-        def fetch_capital_base(exchange, attempt_index=0):
-            """
-            Fetch the base currency amount required to bootstrap
-            the algorithm against the exchange.
-
-            The algorithm cannot continue without this value.
-
-            :param exchange: the targeted exchange
-            :param attempt_index:
-            :return capital_base: the amount of base currency available for
-            trading
-            """
-            try:
-                log.debug('retrieving capital base in {} to bootstrap '
-                          'exchange {}'.format(base_currency, exchange_name))
-                balances = exchange.get_balances()
-            except ExchangeRequestError as e:
-                if attempt_index < 20:
-                    log.warn(
-                        'could not retrieve balances on {}: {}'.format(
-                            exchange.name, e
-                        )
-                    )
-                    sleep(5)
-                    return fetch_capital_base(exchange, attempt_index + 1)
-
-                else:
-                    raise ExchangeRequestErrorTooManyAttempts(
-                        attempts=attempt_index,
-                        error=e
-                    )
-
-            if base_currency in balances:
-                base_currency_available = balances[base_currency]['free']
-                log.info(
-                    'base currency available in the account: {} {}'.format(
-                        base_currency_available, base_currency
-                    )
-                )
-
-                return base_currency_available
-            else:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=base_currency,
-                    exchange=exchange_name
-                )
-
-        if not simulate_orders:
-            for exchange_name in exchanges:
-                exchange = exchanges[exchange_name]
-                balance = fetch_capital_base(exchange)
-
-                if balance < capital_base:
-                    raise NotEnoughCapitalError(
-                        exchange=exchange_name,
-                        base_currency=base_currency,
-                        balance=balance,
-                        capital_base=capital_base,
-                    )
-
         sim_params = create_simulation_parameters(
             start=start,
             end=end,

From d88710501f765e5d2abb1ea353c91306ffebe1c3 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sat, 6 Jan 2018 19:50:58 -0500
Subject: [PATCH 06/22] DOC: updated release notes in preparation for version
 0.4.4.

---
 docs/source/releases.rst | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/source/releases.rst b/docs/source/releases.rst
index e095d0b4..40594aa0 100644
--- a/docs/source/releases.rst
+++ b/docs/source/releases.rst
@@ -2,9 +2,18 @@
 Release Notes
 =============
 
+Version 0.4.4
+^^^^^^^^^^^^^
+**Release Date**: 2018-01-06
+
+Bug Fixes
+~~~~~~~~~
+- Removed redundant capital_base validation (:issue:`142`)
+- Fixed portfolio update issue with restored state (:issue:`111)
+
 Version 0.4.3
 ^^^^^^^^^^^^^
-**Release Date**: 2017-01-05
+**Release Date**: 2018-01-05
 
 Bug Fixes
 ~~~~~~~~~
@@ -13,7 +22,7 @@ Bug Fixes
 
 Version 0.4.2
 ^^^^^^^^^^^^^
-**Release Date**: 2017-01-03
+**Release Date**: 2018-01-03
 
 Bug Fixes
 ~~~~~~~~~

From 9c33ee123c6e5dba0c782edca9abafd0cee6fb7b Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sat, 6 Jan 2018 19:51:41 -0500
Subject: [PATCH 07/22] BLD: Housekeeping

---
 catalyst/examples/mean_reversion_simple.py    |  4 ++--
 catalyst/exchange/exchange.py                 |  3 +--
 catalyst/exchange/exchange_algorithm.py       |  5 ++---
 catalyst/exchange/exchange_asset_finder.py    |  3 +--
 catalyst/exchange/exchange_blotter.py         |  5 ++---
 catalyst/exchange/exchange_bundle.py          |  9 ++++-----
 catalyst/exchange/exchange_data_portal.py     |  5 ++---
 catalyst/exchange/exchange_portfolio.py       |  3 +--
 catalyst/exchange/exchange_pricing_loader.py  | 11 +++++------
 catalyst/exchange/live_graph_clock.py         |  5 ++---
 catalyst/exchange/simple_clock.py             |  3 +--
 catalyst/exchange/utils/bundle_utils.py       |  1 -
 catalyst/exchange/utils/exchange_utils.py     | 19 ++-----------------
 catalyst/exchange/utils/factory.py            |  3 +--
 .../exchange/utils/serialization_utils.py     |  3 +--
 catalyst/exchange/utils/stats_utils.py        |  1 -
 catalyst/exchange/utils/test_utils.py         |  1 -
 17 files changed, 27 insertions(+), 57 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index e46b8df6..1ba812a2 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -37,8 +37,8 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 35
-    context.RSI_OVERBOUGHT = 50
+    context.RSI_OVERSOLD = 45
+    context.RSI_OVERBOUGHT = 55
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py
index b6b24d20..75f4ec3c 100644
--- a/catalyst/exchange/exchange.py
+++ b/catalyst/exchange/exchange.py
@@ -5,8 +5,6 @@ from time import sleep
 
 import numpy as np
 import pandas as pd
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -19,6 +17,7 @@ from catalyst.exchange.utils.bundle_utils import get_start_dt, \
     get_delta, get_periods, get_periods_range
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
     get_frequency, resample_history_df, has_bundle
+from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index efd10553..87bff941 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -18,11 +18,9 @@ from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
 
+import catalyst.protocol as zp
 import logbook
 import pandas as pd
-from redo import retry
-
-import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
@@ -49,6 +47,7 @@ from catalyst.utils.api_support import api_method
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
+from redo import retry
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_asset_finder.py b/catalyst/exchange/exchange_asset_finder.py
index 0a887986..5b41cc7e 100644
--- a/catalyst/exchange/exchange_asset_finder.py
+++ b/catalyst/exchange/exchange_asset_finder.py
@@ -1,8 +1,7 @@
 import pandas as pd
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.utils.factory import find_exchanges
+from logbook import Logger
 
 log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py
index 41a673e1..cb48f939 100644
--- a/catalyst/exchange/exchange_blotter.py
+++ b/catalyst/exchange/exchange_blotter.py
@@ -1,8 +1,5 @@
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from redo import retry
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
@@ -11,6 +8,8 @@ from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
+from logbook import Logger
+from redo import retry
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py
index c3d032d2..5ceb73db 100644
--- a/catalyst/exchange/exchange_bundle.py
+++ b/catalyst/exchange/exchange_bundle.py
@@ -8,12 +8,8 @@ from operator import is_not
 import numpy as np
 import pandas as pd
 import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from pytz import UTC
-from six import itervalues
-
 from catalyst import get_calendar
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
@@ -32,6 +28,9 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
+from logbook import Logger
+from pytz import UTC
+from six import itervalues
 
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py
index a37d1025..6f57b7e7 100644
--- a/catalyst/exchange/exchange_data_portal.py
+++ b/catalyst/exchange/exchange_data_portal.py
@@ -3,9 +3,6 @@ import abc
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from redo import retry
-
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -14,6 +11,8 @@ from catalyst.exchange.exchange_errors import (
     PricingDataNotLoadedError)
 from catalyst.exchange.utils.exchange_utils import get_frequency, \
     resample_history_df, group_assets_by_exchange
+from logbook import Logger
+from redo import retry
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py
index 71ed9a35..894fea0f 100644
--- a/catalyst/exchange/exchange_portfolio.py
+++ b/catalyst/exchange/exchange_portfolio.py
@@ -1,8 +1,7 @@
 import numpy as np
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from logbook import Logger
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
diff --git a/catalyst/exchange/exchange_pricing_loader.py b/catalyst/exchange/exchange_pricing_loader.py
index 38663962..3c59f467 100644
--- a/catalyst/exchange/exchange_pricing_loader.py
+++ b/catalyst/exchange/exchange_pricing_loader.py
@@ -11,12 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from logbook import Logger
-from numpy import (
-    iinfo,
-    uint32,
-)
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.us_equity_pricing import BcolzDailyBarReader
 from catalyst.errors import NoFurtherDataError
@@ -26,6 +20,11 @@ from catalyst.pipeline.data import DataSet, Column
 from catalyst.pipeline.loaders.base import PipelineLoader
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.numpy_utils import float64_dtype
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
 
 UINT32_MAX = iinfo(uint32).max
 
diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py
index 40ad9c24..8a04bef0 100644
--- a/catalyst/exchange/live_graph_clock.py
+++ b/catalyst/exchange/live_graph_clock.py
@@ -1,13 +1,12 @@
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.stats_utils import prepare_stats
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.utils.stats_utils import prepare_stats
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
diff --git a/catalyst/exchange/simple_clock.py b/catalyst/exchange/simple_clock.py
index cede9429..14f52bc9 100644
--- a/catalyst/exchange/simple_clock.py
+++ b/catalyst/exchange/simple_clock.py
@@ -14,14 +14,13 @@
 from time import sleep
 
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
diff --git a/catalyst/exchange/utils/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py
index 6107bf79..e9207511 100644
--- a/catalyst/exchange/utils/bundle_utils.py
+++ b/catalyst/exchange/utils/bundle_utils.py
@@ -6,7 +6,6 @@ from datetime import timedelta, datetime, date
 import numpy as np
 import pandas as pd
 import pytz
-
 from catalyst.data.bundles.core import download_without_progress
 from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
 
diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py
index ebc678d2..f6669c4b 100644
--- a/catalyst/exchange/utils/exchange_utils.py
+++ b/catalyst/exchange/utils/exchange_utils.py
@@ -8,9 +8,6 @@ from datetime import date, datetime
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from six import string_types
-from six.moves.urllib import request
-
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
 from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
     InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
@@ -18,6 +15,8 @@ from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
     ExchangeJSONDecoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
+from six import string_types
+from six.moves.urllib import request
 
 
 def get_sid(symbol):
@@ -733,17 +732,3 @@ def get_candles_df(candles, field, freq, bar_count, end_dt,
     df.dropna(inplace=True)
 
     return df
-
-
-def fetch_capital_base(exchange):
-    """
-    Fetch the base currency amount required to bootstrap
-    the algorithm against the exchange.
-
-    The algorithm cannot continue without this value.
-
-    :param exchange: the targeted exchange
-    :param attempt_index:
-    :return capital_base: the amount of base currency available for
-    trading
-    """
diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py
index 17499442..5650c42b 100644
--- a/catalyst/exchange/utils/factory.py
+++ b/catalyst/exchange/utils/factory.py
@@ -1,13 +1,12 @@
 import os
 
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
 from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
     get_exchange_folder, is_blacklist
+from logbook import Logger
 
 log = Logger('factory', level=LOG_LEVEL)
 exchange_cache = dict()
diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py
index 4b098a02..62ab74d5 100644
--- a/catalyst/exchange/utils/serialization_utils.py
+++ b/catalyst/exchange/utils/serialization_utils.py
@@ -3,9 +3,8 @@ import re
 from json import JSONEncoder
 
 import pandas as pd
-from six import string_types
-
 from catalyst.constants import DATE_TIME_FORMAT
+from six import string_types
 
 
 class ExchangeJSONEncoder(json.JSONEncoder):
diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py
index dd2d4899..0957e4a6 100644
--- a/catalyst/exchange/utils/stats_utils.py
+++ b/catalyst/exchange/utils/stats_utils.py
@@ -8,7 +8,6 @@ import time
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-
 from catalyst.exchange.utils.exchange_utils import get_algo_folder
 from catalyst.utils.paths import data_root, ensure_directory
 
diff --git a/catalyst/exchange/utils/test_utils.py b/catalyst/exchange/utils/test_utils.py
index caae1e23..ac5021be 100644
--- a/catalyst/exchange/utils/test_utils.py
+++ b/catalyst/exchange/utils/test_utils.py
@@ -3,7 +3,6 @@ import random
 import tempfile
 
 from catalyst.assets._assets import TradingPair
-
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.factory import find_exchanges
 from catalyst.utils.paths import ensure_directory

From 30448a65c51148628d8596ed09b8363da4a82fb5 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sat, 6 Jan 2018 19:58:11 -0500
Subject: [PATCH 08/22] BLD: Housekeeping

---
 catalyst/utils/run_algo.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py
index af67c29f..405ebdd5 100644
--- a/catalyst/utils/run_algo.py
+++ b/catalyst/utils/run_algo.py
@@ -4,18 +4,15 @@ import sys
 import warnings
 from datetime import timedelta
 from runpy import run_path
-from time import sleep
 
 import click
 import pandas as pd
-from logbook import Logger
-
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
     TradingPairPricing
 from catalyst.exchange.utils.factory import get_exchange
-from redo import retry
+from logbook import Logger
 
 try:
     from pygments import highlight
@@ -41,9 +38,6 @@ from catalyst.exchange.exchange_algorithm import (
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, NotEnoughCapitalError)
 
 from catalyst.constants import LOG_LEVEL
 

From 33f94b3ef935272efcbd2e7bd1459cfc903c005f Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Sun, 7 Jan 2018 02:32:12 -0500
Subject: [PATCH 09/22] BLD: for issue #144, skipped cash verification when
 there are open orders.

---
 catalyst/examples/mean_reversion_simple.py |  4 ++--
 catalyst/exchange/exchange_algorithm.py    | 10 +++++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 1ba812a2..f5b89d73 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -37,8 +37,8 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 45
-    context.RSI_OVERBOUGHT = 55
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 87bff941..1f4063f4 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -582,10 +582,18 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             if base_currency is None:
                 base_currency = exchange.base_currency
 
+            # Don't check the cash if there are open orders. This could
+            # results in false positives.
+            orders = []
+            for asset in self.blotter.open_orders:
+                asset_orders = self.blotter.open_orders[asset]
+                orders += asset_orders
+
+            required_cash = self.portfolio.cash if not orders else None
             cash, positions_value = exchange.sync_positions(
                 positions=exchange_positions,
                 check_balances=check_balances,
-                cash=self.portfolio.cash,
+                cash=required_cash,
             )
             total_cash += cash
             total_positions_value += positions_value

From 55d1fee82d02dca9a4e5c1701e37edeaa37d149c Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Mon, 8 Jan 2018 17:01:58 -0500
Subject: [PATCH 10/22] DOC: added an alpha warning message in response to
 issue #146

---
 catalyst/utils/run_algo.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py
index 405ebdd5..7aee2ff8 100644
--- a/catalyst/utils/run_algo.py
+++ b/catalyst/utils/run_algo.py
@@ -4,6 +4,7 @@ import sys
 import warnings
 from datetime import timedelta
 from runpy import run_path
+from time import sleep
 
 import click
 import pandas as pd
@@ -139,6 +140,14 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
+    log.warn(
+        'Catalyst is currently in ALPHA. It is going through rapid '
+        'development and it is subject to errors. Please use carefully. '
+        'We encourage your to report any issue on GitHub: '
+        'https://github.com/enigmampc/catalyst/issues'
+    )
+    sleep(3)
+
     mode = 'paper-trading' if simulate_orders else 'live-trading' \
         if live else 'backtest'
     log.info('running algo in {mode} mode'.format(mode=mode))

From 141ee65c913395405b0f142df67ca90f776ad57c Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Tue, 9 Jan 2018 01:08:57 -0500
Subject: [PATCH 11/22] BLD: working on unit tests.

---
 catalyst/data/loader.py                       |   1 +
 catalyst/examples/mean_reversion_simple.py    |  10 +-
 .../mean_reversion_simple_custom_fees.py      | 288 ++++++++++++++++++
 catalyst/exchange/exchange_algorithm.py       |   3 +-
 catalyst/exchange/exchange_blotter.py         |  10 +-
 catalyst/exchange/utils/stats_utils.py        |  35 ++-
 catalyst/utils/run_algo.py                    |   2 +-
 tests/exchange/test_suites/test_suite_algo.py |  72 +++++
 8 files changed, 404 insertions(+), 17 deletions(-)
 create mode 100644 catalyst/examples/mean_reversion_simple_custom_fees.py
 create mode 100644 tests/exchange/test_suites/test_suite_algo.py

diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py
index bfe6c701..cdaa26a0 100644
--- a/catalyst/data/loader.py
+++ b/catalyst/data/loader.py
@@ -146,6 +146,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         exchange = get_exchange(
             exchange_name='poloniex', base_currency='usdt'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 
diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index f5b89d73..7bf60ae9 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -37,14 +37,14 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 50
+    context.RSI_OVERSOLD = 55
     context.RSI_OVERBOUGHT = 60
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.1, taker=0.2)
-    context.set_slippage(spread=0.0001)
+    # context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -248,7 +248,7 @@ if __name__ == '__main__':
 
     if live:
         run_algorithm(
-            capital_base=0.025,
+            capital_base=0.1,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
@@ -280,7 +280,7 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out
diff --git a/catalyst/examples/mean_reversion_simple_custom_fees.py b/catalyst/examples/mean_reversion_simple_custom_fees.py
new file mode 100644
index 00000000..fc44c93e
--- /dev/null
+++ b/catalyst/examples/mean_reversion_simple_custom_fees.py
@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))
diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index 1f4063f4..a05bb7a7 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -587,7 +587,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             orders = []
             for asset in self.blotter.open_orders:
                 asset_orders = self.blotter.open_orders[asset]
-                orders += asset_orders
+                if asset_orders:
+                    orders += asset_orders
 
             required_cash = self.portfolio.cash if not orders else None
             cash, positions_value = exchange.sync_positions(
diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py
index cb48f939..26b74457 100644
--- a/catalyst/exchange/exchange_blotter.py
+++ b/catalyst/exchange/exchange_blotter.py
@@ -10,6 +10,7 @@ from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
 from logbook import Logger
 from redo import retry
+from six import iteritems
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
@@ -41,6 +42,11 @@ class TradingPairFeeSchedule(CommissionModel):
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
@@ -54,8 +60,7 @@ class TradingPairFeeSchedule(CommissionModel):
         cost = abs(transaction.amount) * transaction.price
 
         asset = order.asset
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
+        maker, taker = self.get_maker_taker(asset)
 
         multiplier = taker
         if order.limit is not None:
@@ -250,6 +255,7 @@ class ExchangeBlotter(Blotter):
         for order, txn in self.check_open_orders():
             order.dt = txn.dt
 
+            # TODO: is the commission already on the order object?
             transactions.append(txn)
 
             if not order.open:
diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py
index 0957e4a6..82894862 100644
--- a/catalyst/exchange/utils/stats_utils.py
+++ b/catalyst/exchange/utils/stats_utils.py
@@ -10,6 +10,7 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from catalyst.exchange.utils.exchange_utils import get_algo_folder
 from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
 
 s3_conn = []
 mailgun = []
@@ -260,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()):
     return df, columns
 
 
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
@@ -280,17 +288,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     if isinstance(stats, pd.DataFrame):
         stats = stats.T.to_dict().values()
+        stats.sort(key=itemgetter('period_close'))
+
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
 
-    display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
     df, columns = prepare_stats(
         display_stats, recorded_cols=recorded_cols
     )
-
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
+    set_print_settings()
     return df.to_string(columns=columns)
 
 
@@ -438,6 +446,17 @@ def df_to_string(df):
     return df.to_string()
 
 
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
 def extract_transactions(perf):
     """
     Compute indexes for buy and sell transactions
diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py
index 7aee2ff8..d03488ea 100644
--- a/catalyst/utils/run_algo.py
+++ b/catalyst/utils/run_algo.py
@@ -143,7 +143,7 @@ def _run(handle_data,
     log.warn(
         'Catalyst is currently in ALPHA. It is going through rapid '
         'development and it is subject to errors. Please use carefully. '
-        'We encourage your to report any issue on GitHub: '
+        'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
     sleep(3)
diff --git a/tests/exchange/test_suites/test_suite_algo.py b/tests/exchange/test_suites/test_suite_algo.py
new file mode 100644
index 00000000..58152635
--- /dev/null
+++ b/tests/exchange/test_suites/test_suite_algo.py
@@ -0,0 +1,72 @@
+import importlib
+from os.path import join, isfile
+
+import pandas as pd
+import os
+
+from catalyst import run_algorithm
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
+    extract_transactions, set_print_settings, extract_orders
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+from logbook import TestHandler, WARNING
+from pathtools.path import listdir
+
+filter_algos = [
+    'mean_reversion_simple_custom_fees.py',
+]
+
+
+class TestSuiteAlgo(WithLogger, ZiplineTestCase):
+    @staticmethod
+    def analyze(context, perf):
+        set_print_settings()
+
+        transaction_df = extract_transactions(perf)
+        print('the transactions:\n{}'.format(transaction_df))
+
+        orders_df = extract_orders(perf)
+        print('the orders:\n{}'.format(orders_df))
+
+        stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
+        print('the stats:\n{}'.format(stats))
+        pass
+
+    def test_run_examples(self):
+        folder = join('..', '..', '..', 'catalyst', 'examples')
+        files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+        algo_list = []
+        for filename in files:
+            name = os.path.basename(filename)
+            if filter_algos and name not in filter_algos:
+                continue
+
+            module_name = 'catalyst.examples.{}'.format(
+                name.replace('.py', '')
+            )
+            algo_list.append(module_name)
+
+        for module_name in algo_list:
+            algo = importlib.import_module(module_name)
+            namespace = module_name.replace('.', '_')
+
+            log_catcher = TestHandler()
+            with log_catcher:
+                run_algorithm(
+                    capital_base=0.1,
+                    data_frequency='minute',
+                    initialize=algo.initialize,
+                    handle_data=algo.handle_data,
+                    analyze=TestSuiteAlgo.analyze,
+                    exchange_name='bitfinex',
+                    algo_namespace='test_{}'.format(namespace),
+                    base_currency='eth',
+                    start=pd.to_datetime('2017-10-01', utc=True),
+                    end=pd.to_datetime('2017-10-02', utc=True),
+                    # output=out
+                )
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+                self.assertEqual(0, len(warnings))
+
+            pass

From ac15413af8732225c8013fedfc05c568cb9e9170 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Tue, 9 Jan 2018 01:09:23 -0500
Subject: [PATCH 12/22] DOC: updated release notes for version 0.4.4

---
 docs/source/releases.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/source/releases.rst b/docs/source/releases.rst
index 40594aa0..5bcfb7d6 100644
--- a/docs/source/releases.rst
+++ b/docs/source/releases.rst
@@ -4,12 +4,13 @@ Release Notes
 
 Version 0.4.4
 ^^^^^^^^^^^^^
-**Release Date**: 2018-01-06
+**Release Date**: 2018-01-09
 
 Bug Fixes
 ~~~~~~~~~
 - Removed redundant capital_base validation (:issue:`142`)
-- Fixed portfolio update issue with restored state (:issue:`111)
+- Fixed portfolio update issue with restored state (:issue:`111`)
+- Skipping cash validation where there are open orders (:issue:`144`)
 
 Version 0.4.3
 ^^^^^^^^^^^^^

From a49cb558212e08041fd5d7abb2ae7d00f26aa31b Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Tue, 9 Jan 2018 11:32:44 -0500
Subject: [PATCH 13/22] BUG: fixed issue #147 related to python 3 compatibility

---
 catalyst/exchange/exchange_algorithm.py | 6 +++---
 catalyst/exchange/exchange_blotter.py   | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py
index a05bb7a7..782b7ec8 100644
--- a/catalyst/exchange/exchange_algorithm.py
+++ b/catalyst/exchange/exchange_algorithm.py
@@ -129,7 +129,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_commission(self, maker=None, taker=None):
-        key = self.blotter.commission_models.keys()[0]
+        key = list(self.blotter.commission_models.keys())[0]
         if maker is not None:
             self.blotter.commission_models[key].maker = maker
 
@@ -138,7 +138,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_slippage(self, spread=None):
-        key = self.blotter.slippage_models.keys()[0]
+        key = list(self.blotter.slippage_models.keys())[0]
         if spread is not None:
             self.blotter.slippage_models[key].spread = spread
 
@@ -708,7 +708,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.frame_stats = list()
 
         self.performance_needs_update = False
-        orders = self.perf_tracker.todays_performance.orders_by_id.keys()
+        orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
         if orders != self._last_orders:
             self.performance_needs_update = True
 
diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py
index 26b74457..d638e4bd 100644
--- a/catalyst/exchange/exchange_blotter.py
+++ b/catalyst/exchange/exchange_blotter.py
@@ -10,7 +10,6 @@ from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
 from logbook import Logger
 from redo import retry
-from six import iteritems
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 

From db37c9c6a71b98627327341a471c77a8002edd99 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Tue, 9 Jan 2018 16:57:57 -0500
Subject: [PATCH 14/22] BLD: updated sample algo

---
 catalyst/examples/mean_reversion_simple.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 7bf60ae9..4178e0f8 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -43,8 +43,8 @@ def initialize(context):
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.001, taker=0.002)
-    # context.set_slippage(spread=0.001)
+    context.set_commission(maker=0.001, taker=0.002)
+    context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -244,7 +244,7 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    live = True
+    live = False
 
     if live:
         run_algorithm(

From b60b50e99af29dcdfdf3d71c9ebaaa1a06a50019 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Wed, 10 Jan 2018 13:31:54 -0500
Subject: [PATCH 15/22] BUG: fixed python3 issue in run_algo

---
 catalyst/utils/run_algo.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py
index d03488ea..3d83426c 100644
--- a/catalyst/utils/run_algo.py
+++ b/catalyst/utils/run_algo.py
@@ -148,8 +148,14 @@ def _run(handle_data,
     )
     sleep(3)
 
-    mode = 'paper-trading' if simulate_orders else 'live-trading' \
-        if live else 'backtest'
+    if live:
+        if simulate_orders:
+            mode = 'paper-trading'
+        else:
+            mode = 'live-trading'
+    else:
+        mode = 'backtest'
+
     log.info('running algo in {mode} mode'.format(mode=mode))
 
     exchange_name = exchange

From 713d48780853f881ebeec37da9a6d1a901696825 Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Wed, 10 Jan 2018 23:25:27 -0500
Subject: [PATCH 16/22] BUG: troubleshooting and minor fixes

---
 catalyst/examples/mean_reversion_simple.py |  4 +-
 catalyst/exchange/utils/exchange_utils.py  |  5 ++-
 catalyst/support/issue_111.py              | 44 ++++++++++++++++++++++
 catalyst/support/issue_112.py              | 44 ++++++++++++++++++++++
 4 files changed, 94 insertions(+), 3 deletions(-)
 create mode 100644 catalyst/support/issue_111.py
 create mode 100644 catalyst/support/issue_112.py

diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 4178e0f8..b441565f 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -244,7 +244,7 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    live = False
+    live = True
 
     if live:
         run_algorithm(
@@ -257,7 +257,7 @@ if __name__ == '__main__':
             algo_namespace=NAMESPACE,
             base_currency='btc',
             live_graph=False,
-            simulate_orders=False,
+            simulate_orders=True,
             stats_output=None,
         )
 
diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py
index f6669c4b..081da592 100644
--- a/catalyst/exchange/utils/exchange_utils.py
+++ b/catalyst/exchange/utils/exchange_utils.py
@@ -129,7 +129,10 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
                 pd.Timestamp('now', tz='UTC') - last_modified_time(
                 filename)).days > 1):
-        download_exchange_symbols(exchange_name, environ)
+        try:
+            download_exchange_symbols(exchange_name, environ)
+        except Exception as e:
+            pass
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
diff --git a/catalyst/support/issue_111.py b/catalyst/support/issue_111.py
new file mode 100644
index 00000000..d5efdc3a
--- /dev/null
+++ b/catalyst/support/issue_111.py
@@ -0,0 +1,44 @@
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import order_target_percent
+
+NAMESPACE = 'goose7'
+log = Logger(NAMESPACE)
+
+from catalyst.api import record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('trx_btc')
+
+
+def handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    record(btc=price)
+
+    # Only ordering if it does not have any position to avoid trying some
+    # tiny orders with the leftover btc
+    pos_amount = context.portfolio.positions[context.asset].amount
+    if pos_amount > 0:
+        return
+
+    # Adding a limit price to workaround an issue with performance
+    # calculations of market orders
+    order_target_percent(
+        context.asset, 1, limit_price=price * 1.01
+    )
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.003,
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=NAMESPACE,
+        base_currency='btc',
+        live_graph=False,
+        simulate_orders=False,
+    )
diff --git a/catalyst/support/issue_112.py b/catalyst/support/issue_112.py
new file mode 100644
index 00000000..746a6a74
--- /dev/null
+++ b/catalyst/support/issue_112.py
@@ -0,0 +1,44 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    df = data.history(context.asset,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

From 050fda1bdb8146896961ce2f5276bba128700d4e Mon Sep 17 00:00:00 2001
From: Cam Sweeney <cpsweene@gmail.com>
Date: Thu, 11 Jan 2018 10:34:16 -0800
Subject: [PATCH 17/22] MAINT: Convert dictionary .values() to list for python3

---
 catalyst/data/dispatch_bar_reader.py                   | 4 ++--
 catalyst/examples/dual_moving_average.py               | 2 +-
 catalyst/examples/mean_reversion_simple.py             | 2 +-
 catalyst/examples/mean_reversion_simple_custom_fees.py | 2 +-
 catalyst/examples/rsi_profit_target.py                 | 2 +-
 catalyst/examples/simple_loop.py                       | 2 +-
 catalyst/examples/simple_universe.py                   | 2 +-
 catalyst/exchange/utils/stats_utils.py                 | 2 +-
 catalyst/pipeline/graph.py                             | 4 ++--
 catalyst/sources/test_source.py                        | 4 ++--
 10 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/catalyst/data/dispatch_bar_reader.py b/catalyst/data/dispatch_bar_reader.py
index a8e7429b..1b57c463 100644
--- a/catalyst/data/dispatch_bar_reader.py
+++ b/catalyst/data/dispatch_bar_reader.py
@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         if self._last_available_dt is not None:
             return self._last_available_dt
         else:
-            return min(r.last_available_dt for r in self._readers.values())
+            return min(r.last_available_dt for r in list(self._readers.values()))
 
     @lazyval
     def first_trading_day(self):
-        return max(r.first_trading_day for r in self._readers.values())
+        return max(r.first_trading_day for r in list(self._readers.values()))
 
     def get_value(self, sid, dt, field):
         asset = self._asset_finder.retrieve_asset(sid)
diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py
index 3a0a3f1f..d0e6f057 100644
--- a/catalyst/examples/dual_moving_average.py
+++ b/catalyst/examples/dual_moving_average.py
@@ -84,7 +84,7 @@ def handle_data(context, data):
 def analyze(context, perf):
 
     # Get the base_currency that was passed as a parameter to the simulation
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # First chart: Plot portfolio value using base_currency
     ax1 = plt.subplot(411)
diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py
index 4178e0f8..bec9c327 100644
--- a/catalyst/examples/mean_reversion_simple.py
+++ b/catalyst/examples/mean_reversion_simple.py
@@ -161,7 +161,7 @@ def analyze(context=None, perf=None):
 
     import matplotlib.pyplot as plt
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
diff --git a/catalyst/examples/mean_reversion_simple_custom_fees.py b/catalyst/examples/mean_reversion_simple_custom_fees.py
index fc44c93e..3d323e38 100644
--- a/catalyst/examples/mean_reversion_simple_custom_fees.py
+++ b/catalyst/examples/mean_reversion_simple_custom_fees.py
@@ -161,7 +161,7 @@ def analyze(context=None, perf=None):
 
     import matplotlib.pyplot as plt
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
diff --git a/catalyst/examples/rsi_profit_target.py b/catalyst/examples/rsi_profit_target.py
index a07d63f7..1526b035 100644
--- a/catalyst/examples/rsi_profit_target.py
+++ b/catalyst/examples/rsi_profit_target.py
@@ -175,7 +175,7 @@ def handle_data(context, data):
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
 
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results.loc[:, 'portfolio_value'].plot(ax=ax1)
diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py
index 1e639264..0de91d3d 100644
--- a/catalyst/examples/simple_loop.py
+++ b/catalyst/examples/simple_loop.py
@@ -57,7 +57,7 @@ def analyze(context, perf):
     log.info('the stats: {}'.format(get_pretty_stats(perf)))
 
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
diff --git a/catalyst/examples/simple_universe.py b/catalyst/examples/simple_universe.py
index e781281f..f19b2e25 100644
--- a/catalyst/examples/simple_universe.py
+++ b/catalyst/examples/simple_universe.py
@@ -41,7 +41,7 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
 
 def initialize(context):
     context.i = -1  # minute counter
-    context.exchange = context.exchanges.values()[0].name.lower()
+    context.exchange = list(context.exchanges.values())[0].name.lower()
     context.base_currency = context.exchanges.values()[0].base_currency.lower()
 
 
diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py
index 82894862..8cc9b6ca 100644
--- a/catalyst/exchange/utils/stats_utils.py
+++ b/catalyst/exchange/utils/stats_utils.py
@@ -287,7 +287,7 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
 
     """
     if isinstance(stats, pd.DataFrame):
-        stats = stats.T.to_dict().values()
+        stats = list(stats.T.to_dict().values())
         stats.sort(key=itemgetter('period_close'))
 
     if len(stats) > num_rows:
diff --git a/catalyst/pipeline/graph.py b/catalyst/pipeline/graph.py
index 47e349c1..a4f89d14 100644
--- a/catalyst/pipeline/graph.py
+++ b/catalyst/pipeline/graph.py
@@ -142,7 +142,7 @@ class TermGraph(object):
         at the end of execution.
         """
         refcounts = self.graph.out_degree()
-        for t in self.outputs.values():
+        for t in list(self.outputs.values()):
             refcounts[t] += 1
 
         for t in initial_terms:
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
                  min_extra_rows=0):
         super(ExecutionPlan, self).__init__(terms)
 
-        for term in terms.values():
+        for term in list(terms.values()):
             self.set_extra_rows(
                 term,
                 all_dates,
diff --git a/catalyst/sources/test_source.py b/catalyst/sources/test_source.py
index 673e2373..f1503428 100644
--- a/catalyst/sources/test_source.py
+++ b/catalyst/sources/test_source.py
@@ -144,7 +144,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
             for event in self.event_list:
                 event.sid = assets_by_identifier[event.sid].sid
 
@@ -167,7 +167,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
 
         # Hash_value for downstream sorting.
         self.arg_string = hash_args(*args, **kwargs)

From e0827fe4ad0d3165561fc4acc79fd47a0a69aa6f Mon Sep 17 00:00:00 2001
From: Cam Sweeney <cpsweene@gmail.com>
Date: Thu, 11 Jan 2018 12:42:13 -0800
Subject: [PATCH 18/22] DOC: present virtualenv info before pip install command

Creating a virtualenv is now explained before the command for install
via pip. Following the instructions in the current state may cause
users to install using the system python, negating the point of the
virtualenv
---
 docs/source/install.rst | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/docs/source/install.rst b/docs/source/install.rst
index 13f0b115..4ef78df7 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -180,20 +180,6 @@ use a single tool to install Python and non-Python dependencies, or if you're
 already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
 distribution, refer to the :ref:`Installing with Conda <conda>` section.
 
-Once you've installed the necessary additional dependencies for your system 
-(see below for your particular platform: :ref:`Linux`, :ref:`MacOS` or 
-:ref:`Windows`), you should be able to simply run
-
-.. code-block:: bash
-
-   $ pip install enigma-catalyst matplotlib
-
-Note that in the command above we install two different packages. The second 
-one, ``matplotlib`` is a visualization library. While it's not strictly 
-required to run catalyst simulations or live trading, it comes in very handy
-to visualize the performance of your algorithms, and for this reason we 
-recommend you install it, as well.
-
 If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
@@ -206,8 +192,23 @@ summarized version:
    $ pip install virtualenv
    $ virtualenv catalyst-venv
    $ source ./catalyst-venv/bin/activate
+
+
+
+Once you've installed the necessary additional dependencies for your system 
+(:ref:`Linux`, :ref:`MacOS` or :ref:`Windows`) **and have activated your virtualenv**, you should be able to simply run
+
+.. code-block:: bash
+
    $ pip install enigma-catalyst matplotlib
 
+Note that in the command above we install two different packages. The second 
+one, ``matplotlib`` is a visualization library. While it's not strictly 
+required to run catalyst simulations or live trading, it comes in very handy
+to visualize the performance of your algorithms, and for this reason we 
+recommend you install it, as well.
+
+
 Troubleshooting ``pip`` Install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 05c8957c90c40d997bb98c2beec8be51c553d50e Mon Sep 17 00:00:00 2001
From: Frederic Fortier <fredfortier@gmail.com>
Date: Fri, 12 Jan 2018 00:48:53 -0500
Subject: [PATCH 19/22] BUG: fixed issue with history of multiple assets

---
 catalyst/exchange/ccxt/ccxt_exchange.py     |  4 +-
 catalyst/support/history_multiple_assets.py | 50 +++++++++++++++++++++
 2 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 catalyst/support/history_multiple_assets.py

diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py
index ed3b2b24..bade7221 100644
--- a/catalyst/exchange/ccxt/ccxt_exchange.py
+++ b/catalyst/exchange/ccxt/ccxt_exchange.py
@@ -382,9 +382,9 @@ class CCXT(Exchange):
             ms = int(delta.total_seconds()) * 1000
 
         candles = dict()
-        for asset in assets:
+        for index, asset in enumerate(assets):
             ohlcvs = self.api.fetch_ohlcv(
-                symbol=symbols[0],
+                symbol=symbols[index],
                 timeframe=timeframe,
                 since=ms,
                 limit=bar_count,
diff --git a/catalyst/support/history_multiple_assets.py b/catalyst/support/history_multiple_assets.py
new file mode 100644
index 00000000..804ae58f
--- /dev/null
+++ b/catalyst/support/history_multiple_assets.py
@@ -0,0 +1,50 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset1 = symbol('fct_btc')
+    context.asset2 = symbol('btc_usdt')
+    context.coins = [context.asset1, context.asset2]
+
+
+def handle_data(context, data):
+    df = data.history(context.coins,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+    print(df)
+    print(data.current(context.asset1, 'close'))
+    print(data.current(context.asset2, 'close'))
+    exit(0)
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

From 415fceb11cd222647d8773ce7a5dab9229f8b459 Mon Sep 17 00:00:00 2001
From: Victor Grau Serrat <lacabra@gmail.com>
Date: Fri, 12 Jan 2018 08:31:50 -0700
Subject: [PATCH 20/22] DOC: typo - #158

---
 docs/source/beginner-tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst
index 12792ca4..ed6f56ff 100644
--- a/docs/source/beginner-tutorial.rst
+++ b/docs/source/beginner-tutorial.rst
@@ -483,7 +483,7 @@ bitcoin price.
 
 Now we will run the simulation again, but this time we extend our original 
 algorithm with the addition of the ``analyze()`` function. Somewhat analogously 
-as how ``initialize()`` gets called once before the start of the algorith, 
+as how ``initialize()`` gets called once before the start of the algorithm, 
 ``analyze()`` gets called once at the end of the algorithm, and receives two 
 variables: ``context``, which we discussed at the very beginning, and ``perf``, 
 which is the pandas dataframe containing the performance data for our algorithm 

From 14c717015980e984945a00eec6ccfe6a2085ed5e Mon Sep 17 00:00:00 2001
From: Victor Grau Serrat <lacabra@gmail.com>
Date: Fri, 12 Jan 2018 08:57:00 -0700
Subject: [PATCH 21/22] DOC: sphinx/docutils issue when building docs

---
 docs/source/development-guidelines.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/source/development-guidelines.rst b/docs/source/development-guidelines.rst
index 9bc6c7b0..677246ec 100644
--- a/docs/source/development-guidelines.rst
+++ b/docs/source/development-guidelines.rst
@@ -84,6 +84,25 @@ To build and view the docs locally, run:
    $ {BROWSER} build/html/index.html
 
 
+There is a `documented issue <https://github.com/sphinx-doc/sphinx/issues/3212>`_ 
+with ``sphinx`` and ``docutils`` that causes the error below when trying to build 
+the docs.
+
+.. code-block:: text
+
+   Exception occurred:
+     File "(...)/env-c/lib/python2.7/site-packages/docutils/writers/_html_base.py", line 671, in depart_document
+       assert not self.context, 'len(context) = %s' % len(self.context)
+   AssertionError: len(context) = 3
+
+If you get this error, you need to downgrade your version of ``docutils`` as 
+follows, and build the docs again:
+
+.. code-block:: bash
+
+   $ pip install docutils==0.12
+
+
 Commit messages
 ---------------
 

From b7a32656d5ed674657e4950227066fe42753a15d Mon Sep 17 00:00:00 2001
From: Victor Grau Serrat <lacabra@gmail.com>
Date: Fri, 12 Jan 2018 12:15:04 -0700
Subject: [PATCH 22/22] DOC: improved doc of matplotlib error after
 installation

---
 docs/source/install.rst | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/docs/source/install.rst b/docs/source/install.rst
index 13f0b115..a5790372 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -408,20 +408,34 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
-MacOS + virtualenv + matplotlib
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+MacOS + virtualenv/conda + matplotlib
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A note about using matplotlib in virtual enviroments on MacOS: it may be 
-necessary to run
+The first time that you try to run an algorithm that loads the ``matplotlib`` 
+library, you may get the following error:
+
+.. code-block:: text
+
+  RuntimeError: Python is not installed as a framework. The Mac OS X backend 
+  will not be able to function correctly if Python is not installed as a 
+  framework. See the Python documentation for more information on installing 
+  Python as a framework on Mac OS X. Please either reinstall Python as a 
+  framework, or try one of the other backends. If you are using (Ana)Conda 
+  please install python.app and replace the use of 'python' with 'pythonw'. 
+  See 'Working with Matplotlib on OSX' in the Matplotlib FAQ for more 
+  information.
+
+This is a ``matplotlib``-specific error, that will go away once you run the 
+following command:
 
 .. code-block:: bash
 
    echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
 
 in order to override the default ``MacOS`` backend for your system, which 
-may not be accessible from inside the virtual environment. This will allow 
-Catalyst to open matplotlib charts from within a virtual environment, which 
-is useful for displaying the performance of your backtests.  To learn more 
+may not be accessible from inside the virtual or conda environment. This will 
+allow Catalyst to open matplotlib charts from within a virtual environment, 
+which is useful for displaying the performance of your backtests.  To learn more 
 about matplotlib backends, please refer to the
 `matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.