BLD: added a cmd for running on the cloud (WIP)

# Conflicts:
#	catalyst/examples/mean_reversion_simple.py
#	catalyst/exchange/ccxt/ccxt_exchange.py
#	catalyst/exchange/exchange.py
#	catalyst/exchange/utils/serialization_utils.py
#	etc/python2.7-environment.yml
#	etc/requirements.txt
#	tests/exchange/test_suites/test_suite_exchange.py

catalyst/__main__.py

@@ -14,6 +14,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
+from catalyst.utils.run_server import run_server
 
 try:
     __IPYTHON__
@@ -505,6 +506,179 @@ def live(ctx,
     return perf
 
 
+@main.command(name='serve-live')
+@click.option(
+    '-f',
+    '--algofile',
+    default=None,
+    type=click.File('r'),
+    help='The file that contains the algorithm to run.',
+)
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
+@click.option(
+    '-t',
+    '--algotext',
+    help='The algorithm script to run.',
+)
+@click.option(
+    '-D',
+    '--define',
+    multiple=True,
+    help="Define a name to be bound in the namespace before executing"
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
+)
+@click.option(
+    '-o',
+    '--output',
+    default='-',
+    metavar='FILENAME',
+    show_default=True,
+    help="The location to write the perf data. If this is '-' the perf will"
+         " be written to stdout.",
+)
+@click.option(
+    '--print-algo/--no-print-algo',
+    is_flag=True,
+    default=False,
+    help='Print the algorithm to stdout.',
+)
+@ipython_only(click.option(
+    '--local-namespace/--no-local-namespace',
+    is_flag=True,
+    default=None,
+    help='Should the algorithm methods be resolved in the local namespace.'
+))
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the targeted exchange.',
+)
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='A label assigned to the algorithm for data storage purposes.'
+)
+@click.option(
+    '-c',
+    '--base-currency',
+    help='The base currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
+@click.option(
+    '--live-graph/--no-live-graph',
+    is_flag=True,
+    default=False,
+    help='Display live graph.',
+)
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
+@click.pass_context
+def serve_live(ctx,
+               algofile,
+               capital_base,
+               algotext,
+               define,
+               output,
+               print_algo,
+               local_namespace,
+               exchange_name,
+               algo_namespace,
+               base_currency,
+               end,
+               live_graph,
+               auth_aliases,
+               simulate_orders):
+    """Trade live with the given algorithm on the server.
+    """
+    if (algotext is not None) == (algofile is not None):
+        ctx.fail(
+            "must specify exactly one of '-f' / '--algofile' or"
+            " '-t' / '--algotext'",
+        )
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    if algo_namespace is None:
+        ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
+    if base_currency is None:
+        ctx.fail("must specify a base currency '-c' in live execution mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.', sys.stdout)
+
+    else:
+        click.echo('Running in live trading mode.', sys.stdout)
+
+    perf = run_server(
+        initialize=None,
+        handle_data=None,
+        before_trading_start=None,
+        analyze=None,
+        algofile=algofile,
+        algotext=algotext,
+        defines=define,
+        data_frequency=None,
+        capital_base=capital_base,
+        data=None,
+        bundle=None,
+        bundle_timestamp=None,
+        start=None,
+        end=end,
+        output=output,
+        print_algo=print_algo,
+        local_namespace=local_namespace,
+        environ=os.environ,
+        live=True,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency,
+        live_graph=live_graph,
+        analyze_live=None,
+        simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
+        stats_output=None,
+    )
+
+    if output == '-':
+        click.echo(str(perf), sys.stdout)
+    elif output != os.devnull:  # make the catalyst magic not write any data
+        perf.to_pickle(output)
+
+    return perf
+
+
 @main.command(name='ingest-exchange')
 @click.option(
     '-x',

catalyst/examples/dual_moving_average.py

@@ -4,8 +4,7 @@ import pandas as pd
 from logbook import Logger
 
 from catalyst import run_algorithm
-from catalyst.api import (record, symbol, order_target_percent,
+from catalyst.api import (record, symbol, order_target_percent,)
-                          get_open_orders)
 from catalyst.exchange.utils.stats_utils import extract_transactions
 
 NAMESPACE = 'dual_moving_average'
@@ -20,8 +19,8 @@ def initialize(context):
 
 def handle_data(context, data):
     # define the windows for the moving averages
-    short_window = 2
+    short_window = 50
-    long_window = 2
+    long_window = 200
 
     # Skip as many bars as long_window to properly compute the average
     context.i += 1
@@ -63,7 +62,7 @@ def handle_data(context, data):
 
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.asset)
+    orders = context.blotter.open_orders
     if len(orders) > 0:
         return
 
@@ -150,27 +149,16 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
+    
     run_algorithm(
-        capital_base=1000,
+            capital_base=1000,
-        data_frequency='minute',
+            data_frequency='minute',
-        initialize=initialize,
+            initialize=initialize,
-        handle_data=handle_data,
+            handle_data=handle_data,
-        analyze=analyze,
+            analyze=analyze,
-        exchange_name='bitfinex',
+            exchange_name='bitfinex',
-        algo_namespace=NAMESPACE,
+            algo_namespace=NAMESPACE,
-        base_currency='usd',
+            base_currency='usd',
-        simulate_orders=True,
+            start=pd.to_datetime('2017-9-22', utc=True),
-        live=True,
+            end=pd.to_datetime('2017-9-23', utc=True),
-    )
+        )
-    # run_algorithm(
-    #         capital_base=1000,
-    #         data_frequency='minute',
-    #         initialize=initialize,
-    #         handle_data=handle_data,
-    #         analyze=analyze,
-    #         exchange_name='bitfinex',
-    #         algo_namespace=NAMESPACE,
-    #         base_currency='usd',
-    #         start=pd.to_datetime('2017-9-22', utc=True),
-    #         end=pd.to_datetime('2017-9-23', utc=True),
-    #     )

catalyst/examples/marketplace/github-research.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, get_dataset
+
+START = '2017-01-01'
+END = '2017-12-31'
+
+
+def initialize(context):
+    pass
+
+
+def handle_data(context, data):
+    context.github = get_dataset('github')
+    context.github.sort_index(level=0, inplace=True)
+
+    context.zec = data.history(symbol('zec_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+    context.xmr = data.history(symbol('xmr_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+
+
+def analyze(context=None, results=None):
+    ax1 = plt.subplot(211)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'ZEC']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax1, color='blue')
+    ax1.legend(loc=2)
+    ax1.set_title('Zcash')
+    ax2 = ax1.twinx()
+    context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
+    ax2.legend(loc=1)
+
+    ax3 = plt.subplot(212)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'XMR']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax3, color='blue')
+    ax3.legend(loc=2)
+    ax3.set_title('Monero')
+    ax4 = ax3.twinx()
+    context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
+    ax4.legend(loc=1)
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='algo-github',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime(END, utc=True),
+            end=pd.to_datetime(END, utc=True),
+        )

catalyst/examples/mean_reversion_simple.py

@@ -37,8 +37,8 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 40
+    context.RSI_OVERSOLD = 60
-    context.RSI_OVERBOUGHT = 60
+    context.RSI_OVERBOUGHT = 70
     context.CANDLE_SIZE = '15T'
 
     context.start_time = time.time()

catalyst/examples/portfolio_optimization.py

@@ -146,4 +146,5 @@ if __name__ == '__main__':
                             start=start,
                             end=end,
                             exchange_name='poloniex',
-                            capital_base=100000, )
+                            capital_base=100000,
+                            base_currency='usdt', )

catalyst/examples/simple_loop.py

@@ -26,7 +26,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='30T'
+        frequency='2H'
     )
     last_traded = prices.index[-1]
     log.info('last candle date: {}'.format(last_traded))
@@ -114,7 +114,7 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
-    mode = 'backtest'
+    mode = 'live'
 
     if mode == 'backtest':
         run_algorithm(

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -43,7 +43,8 @@ SUPPORTED_EXCHANGES = dict(
 
 
 class CCXT(Exchange):
-    def __init__(self, exchange_name, key, secret, base_currency):
+    def __init__(self, exchange_name, key,
+                 secret, password, base_currency):
         log.debug(
             'finding {} in CCXT exchanges:\n{}'.format(
                 exchange_name, ccxt.exchanges
@@ -60,6 +61,7 @@ class CCXT(Exchange):
             self.api = exchange_attr({
                 'apiKey': key,
                 'secret': secret,
+                'password': password,
             })
             self.api.enableRateLimit = True
 
@@ -426,25 +428,18 @@ class CCXT(Exchange):
             )
 
         if start_dt is None:
-            # TODO: determine why binance is failing
+            if end_dt is None:
-            if end_dt is None and self.name not in ['binance']:
                 end_dt = pd.Timestamp.utcnow()
 
-            if end_dt is not None:
+            dt_range = get_periods_range(
-                dt_range = get_periods_range(
+                end_dt=end_dt,
-                    end_dt=end_dt,
+                periods=bar_count,
-                    periods=bar_count,
+                freq=freq,
-                    freq=freq,
+            )
-                )
+            start_dt = dt_range[0]
-                start_dt = dt_range[0]
 
-        if start_dt is not None:
+        delta = start_dt - get_epoch()
-            # Convert out start date to a UNIX timestamp, then translate to
+        since = int(delta.total_seconds()) * 1000
-            # milliseconds
-            delta = start_dt - get_epoch()
-            since = int(delta.total_seconds()) * 1000
-        else:
-            since = None
 
         candles = dict()
         for index, asset in enumerate(assets):
@@ -985,7 +980,8 @@ class CCXT(Exchange):
             )
             raise ExchangeRequestError(error=e)
 
-    def cancel_order(self, order_param, asset_or_symbol=None):
+    def cancel_order(self, order_param,
+                     asset_or_symbol=None, params={}):
         order_id = order_param.id \
             if isinstance(order_param, Order) else order_param
 
@@ -997,7 +993,8 @@ class CCXT(Exchange):
         try:
             symbol = self.get_symbol(asset_or_symbol) \
                 if asset_or_symbol is not None else None
-            self.api.cancel_order(id=order_id, symbol=symbol)
+            self.api.cancel_order(id=order_id,
+                                  symbol=symbol, params= params)
 
         except (ExchangeError, NetworkError) as e:
             log.warn(

catalyst/exchange/exchange.py

@@ -1,4 +1,5 @@
 import abc
+import pytz
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
@@ -502,7 +503,7 @@ class Exchange:
 
         """
         freq, candle_size, unit, data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
         # The get_history method supports multiple asset
         candles = self.get_candles(
@@ -514,31 +515,37 @@ class Exchange:
 
         series = dict()
         for asset in candles:
-            first_candle = candles[asset][0]
+            if candles[asset]:
-            asset_series = self.get_series_from_candles(
+                first_candle = candles[asset][0]
-                candles=candles[asset],
+                asset_series = self.get_series_from_candles(
-                start_dt=first_candle['last_traded'],
+                    candles=candles[asset],
-                end_dt=end_dt,
+                    start_dt=first_candle['last_traded'],
-                data_frequency=frequency,
+                    end_dt=end_dt,
-                field=field,
+                    data_frequency=frequency,
-            )
+                    field=field,
-
-            # Checking to make sure that the dates match
-            delta = get_delta(candle_size, data_frequency)
-            adj_end_dt = end_dt - delta
-            last_traded = asset_series.index[-1]
-
-            if last_traded < adj_end_dt:
-                raise LastCandleTooEarlyError(
-                    last_traded=last_traded,
-                    end_dt=adj_end_dt,
-                    exchange=self.name,
                 )
 
+                delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
+                # Checking to make sure that the dates match
+                delta = get_delta(delta_candle_size, data_frequency)
+                adj_end_dt = end_dt - delta
+                last_traded = asset_series.index[-1]
+
+                if last_traded < adj_end_dt:
+                    raise LastCandleTooEarlyError(
+                        last_traded=last_traded,
+                        end_dt=adj_end_dt,
+                        exchange=self.name,
+                    )
+            else: # empty candle received
+                # because other assets are tz-aware, we need its tz to be set as well
+                asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc))
+
+
             series[asset] = asset_series
 
         df = pd.DataFrame(series)
-        df.dropna(inplace=True)
+        #df.dropna(inplace=True) # commented out due to issue 236
 
         return df
 
@@ -664,7 +671,8 @@ class Exchange:
         else:
             return free, False
 
-    def sync_positions(self, positions, cash=None, check_balances=False):
+    def sync_positions(self, positions, cash=None,
+                       check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
@@ -920,7 +928,8 @@ class Exchange:
         """
 
     @abstractmethod
-    def cancel_order(self, order_param, symbol_or_asset=None):
+    def cancel_order(self, order_param,
+                     symbol_or_asset=None, params={}):
         """Cancel an open order.
 
         Parameters
@@ -929,6 +938,7 @@ class Exchange:
             The order_id or order object to cancel.
         symbol_or_asset: str|TradingPair
             The catalyst symbol, some exchanges need this
+        params:
         """
         pass
 

catalyst/exchange/exchange_algorithm.py

@@ -16,7 +16,7 @@ import signal
 import sys
 from datetime import timedelta
 from os import listdir
-from os.path import isfile, join
+from os.path import isfile, join, exists
 
 import catalyst.protocol as zp
 import logbook
@@ -36,9 +36,11 @@ from catalyst.exchange.utils.exchange_utils import (
     get_algo_folder,
     get_algo_df,
     save_algo_df,
+    clear_frame_stats_directory,
+    remove_old_files,
     group_assets_by_exchange, )
-from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
+from catalyst.exchange.utils.stats_utils import \
-    stats_to_algo_folder
+    get_pretty_stats, stats_to_s3, stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
 from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
@@ -67,8 +69,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         self.current_day = None
 
-        if self.simulate_orders is None \
+        if self.simulate_orders is None and \
-            and self.sim_params.arena == 'backtest':
+                self.sim_params.arena == 'backtest':
             self.simulate_orders = True
 
         # Operations with retry features
@@ -118,7 +120,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             # be in-line with CXXT and many exchanges. We'll consider
             # adding more order types in the future.
             if not isinstance(style, ExchangeLimitOrder) or \
-                not isinstance(style, MarketOrder):
+                    not isinstance(style, MarketOrder):
                 raise OrderTypeNotSupported(
                     order_type=style.__class__.__name__
                 )
@@ -368,19 +370,35 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self._clock = None
         self.frame_stats = list()
 
-        self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
+        # erase the frame_stats folder to avoid overloading the disk
+        error = clear_frame_stats_directory(self.algo_namespace)
+        if error:
+            log.warning(error)
 
-        self.custom_signals_stats = \
+        # in order to save paper & live files separately
-            get_algo_df(self.algo_namespace, 'custom_signals_stats')
+        self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live'
 
-        self.exposure_stats = \
+        self.pnl_stats = get_algo_df(
-            get_algo_df(self.algo_namespace, 'exposure_stats')
+                self.algo_namespace,
+                'pnl_stats_{}'.format(self.mode_name),
+            )
+
+        self.custom_signals_stats = get_algo_df(
+                self.algo_namespace,
+                'custom_signals_stats_{}'.format(self.mode_name)
+            )
+
+        self.exposure_stats = get_algo_df(
+                self.algo_namespace,
+                'exposure_stats_{}'.format(self.mode_name)
+            )
 
         self.is_running = True
 
         self.stats_minutes = 1
 
         self._last_orders = []
+        self._last_open_orders = []
         self.trading_client = None
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
@@ -392,6 +410,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      "Exit should be handled by the user.")
 
     def interrupt_algorithm(self):
+        """
+
+        when algorithm comes to an end this function is called.
+        extracts the stats and calls analyze.
+        after finishing, it exits the run.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
         self.is_running = False
 
         if self._analyze is None:
@@ -401,21 +432,31 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             log.info('Exiting the algorithm. Calling `analyze()` '
                      'before exiting the algorithm.')
 
+            # add the last day stats which is not saved in the directory
+            current_stats = pd.DataFrame(self.frame_stats)
+            current_stats.set_index('period_close', drop=False, inplace=True)
+
+            # get the location of the directory
             algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_performance')
+            folder = join(algo_folder, 'frame_stats')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
 
-            daily_perf_list = []
+            if exists(folder):
-            for item in files:
+                files = [f for f in listdir(folder) if isfile(join(folder, f))]
-                filename = join(folder, item)
 
-                with open(filename, 'rb') as handle:
+                period_stats_list = []
-                    perf_period = pickle.load(handle)
+                for item in files:
-                    perf_period_dict = perf_period.to_dict()
+                    filename = join(folder, item)
-                    daily_perf_list.append(perf_period_dict)
 
-            stats = pd.DataFrame(daily_perf_list)
+                    with open(filename, 'rb') as handle:
-            stats.set_index('period_close', drop=False, inplace=True)
+                        perf_period = pickle.load(handle)
+                        period_stats_list.extend(perf_period)
+
+                stats = pd.DataFrame(period_stats_list)
+                stats.set_index('period_close', drop=False, inplace=True)
+
+                stats = pd.concat([stats, current_stats])
+            else:
+                stats = current_stats
 
             self.analyze(stats)
 
@@ -484,7 +525,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         """
         self.state = get_algo_object(
             algo_name=self.algo_namespace,
-            key='context.state',
+            key='context.state_{}'.format(self.mode_name),
         )
         if self.state is None:
             self.state = {}
@@ -507,7 +548,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             # Unpacking the perf_tracker and positions if available
             cum_perf = get_algo_object(
                 algo_name=self.algo_namespace,
-                key='cumulative_performance',
+                key='cumulative_performance_{}'.format(self.mode_name),
             )
             if cum_perf is not None:
                 tracker.cumulative_performance = cum_perf
@@ -518,7 +559,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             todays_perf = get_algo_object(
                 algo_name=self.algo_namespace,
                 key=today.strftime('%Y-%m-%d'),
-                rel_path='daily_performance',
+                rel_path='daily_performance_{}'.format(self.mode_name),
             )
             if todays_perf is not None:
                 # Ensure single common position tracker
@@ -601,8 +642,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             if base_currency is None:
                 base_currency = exchange.base_currency
 
-            # Don't check the cash if there are open orders. This could
-            # results in false positives.
             orders = []
             for asset in self.blotter.open_orders:
                 asset_orders = self.blotter.open_orders[asset]
@@ -657,7 +696,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.pnl_stats = pd.concat([self.pnl_stats, df])
 
-        save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
+        save_algo_df(
+            self.algo_namespace,
+            'pnl_stats_{}'.format(self.mode_name),
+            self.pnl_stats,
+        )
 
     def add_custom_signals_stats(self, period_stats):
         """
@@ -678,8 +721,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.custom_signals_stats = pd.concat([self.custom_signals_stats, df])
 
-        save_algo_df(self.algo_namespace, 'custom_signals_stats',
+        save_algo_df(
-                     self.custom_signals_stats)
+            self.algo_namespace,
+            'custom_signals_stats_{}'.format(self.mode_name),
+            self.custom_signals_stats,
+        )
 
     def add_exposure_stats(self, period_stats):
         """
@@ -706,9 +752,43 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.exposure_stats = pd.concat([self.exposure_stats, df])
 
         save_algo_df(
-            self.algo_namespace, 'exposure_stats', self.exposure_stats
+            self.algo_namespace,
+            'exposure_stats_{}'.format(self.mode_name),
+            self.exposure_stats
         )
 
+    def nullify_frame_stats(self, now):
+        """
+
+        Save all period_stats to local directory
+        erase old files from the folder and nullify
+        self.frame_stats
+
+        Parameters
+        ----------
+        now: Timestamp
+
+        Returns
+        -------
+
+        """
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=now.floor('1D').strftime('%Y-%m-%d'),
+            obj=self.frame_stats,
+            rel_path='frame_stats'
+        )
+
+        error = remove_old_files(
+            algo_name=self.algo_namespace,
+            today=now,
+            rel_path='frame_stats'
+        )
+        if error:
+            log.warning(error)
+
+        self.frame_stats = list()
+
     def handle_data(self, data):
         """
         Wrapper around the handle_data method of each algo.
@@ -728,15 +808,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # Resetting the frame stats every day to minimize memory footprint
         today = data.current_dt.floor('1D')
         if self.current_day is not None and today > self.current_day:
-            self.frame_stats = list()
+            self.nullify_frame_stats(now=data.current_dt)
 
         self.performance_needs_update = False
-        orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
+        last_orders_list = list(self.blotter.orders.keys())
-        if orders != self._last_orders:
+        open_orders_list = list(self.blotter.open_orders.keys())
+
+        if last_orders_list != self._last_orders or \
+                open_orders_list != self._last_open_orders:
             self.performance_needs_update = True
 
-        # Saving current orders to detect changes in the next frame
+        # Saving current order positions
-        self._last_orders = copy.deepcopy(orders)
+        # to detect changes in the next frame
+        self._last_orders = copy.deepcopy(last_orders_list)
+        self._last_open_orders = copy.deepcopy(open_orders_list)
 
         if self.performance_needs_update:
             self.perf_tracker.update_performance()
@@ -778,7 +863,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         log.debug('saving cumulative performance object')
         save_algo_object(
             algo_name=self.algo_namespace,
-            key='cumulative_performance',
+            key='cumulative_performance_{}'.format(self.mode_name),
             obj=self.perf_tracker.cumulative_performance,
         )
         log.debug('saving todays performance object')
@@ -786,12 +871,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             algo_name=self.algo_namespace,
             key=today.strftime('%Y-%m-%d'),
             obj=self.perf_tracker.todays_performance,
-            rel_path='daily_performance'
+            rel_path='daily_performance_{}'.format(self.mode_name)
         )
         log.debug('saving context.state object')
         save_algo_object(
             algo_name=self.algo_namespace,
-            key='context.state',
+            key='context.state_{}'.format(self.mode_name),
             obj=self.state)
 
     def _process_stats(self, data):
@@ -808,6 +893,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
 
+        # creating and saving the pnl_stats into the local
+        # directory
         self.add_pnl_stats(frame_stats)
         if self.recorded_vars:
             self.add_custom_signals_stats(frame_stats)
@@ -845,6 +932,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             csv_bytes = stats_to_algo_folder(
                 stats=self.frame_stats,
                 algo_namespace=self.algo_namespace,
+                folder_name='stats_{}'.format(self.mode_name),
                 recorded_cols=recorded_cols,
             )
         except Exception as e:
@@ -949,13 +1037,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             args=(order_id,))
 
     @api_method
-    def cancel_order(self, order_param, exchange_name):
+    def cancel_order(self, order_param, exchange_name,
+                     symbol=None, params={}):
         """Cancel an open order.
 
         Parameters
         ----------
         order_param : str or Order
             The order_id or order object to cancel.
+
+        exchange_name: name of exchange from
+                        which you want to cancel the order
+        symbol:
+        params:
         """
         exchange = self.exchanges[exchange_name]
 
@@ -969,4 +1063,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             sleeptime=self.attempts['retry_sleeptime'],
             retry_exceptions=(ExchangeRequestError,),
             cleanup=lambda: log.warn('cancelling order again.'),
-            args=(order_id,))
+            args=(order_id, symbol, params))

catalyst/exchange/exchange_blotter.py

@@ -238,9 +238,12 @@ class ExchangeBlotter(Blotter):
                 else:
                     delta = pd.Timestamp.utcnow() - order.dt
                     log.info(
-                        'order {order_id} still open after {delta}'.format(
+                        '{exchange} order {order_id} for {symbol} still open '
+                        'after {delta}'.format(
+                            exchange=exchange.name,
                             order_id=order.id,
-                            delta=delta
+                            delta=delta,
+                            symbol=order.asset.symbol,
                         )
                     )
 

catalyst/exchange/exchange_data_portal.py

@@ -9,8 +9,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     PricingDataNotLoadedError)
-from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
+from catalyst.exchange.utils.exchange_utils import resample_history_df, \
-from catalyst.exchange.utils.datetime_utils import get_frequency
+    group_assets_by_exchange
+from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
 from logbook import Logger
 from redo import retry
 
@@ -311,7 +312,8 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             algo_end_dt=self._last_available_session,
         )
 
-        df = resample_history_df(pd.DataFrame(series), freq, field)
+        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
+        df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
         return df
 
     def get_exchange_spot_value(self,

catalyst/exchange/utils/datetime_utils.py

@@ -92,7 +92,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
         adj_periods = periods * unit_periods
 
         # TODO: standardize time aliases to avoid any mapping
-        unit = 'd' if unit == 'D' else 'm'
+        unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
         delta = pd.Timedelta(adj_periods, unit)
 
         if start_dt is not None:
@@ -248,7 +248,7 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_frequency(freq, data_frequency=None):
+def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
     """
     Get the frequency parameters.
 
@@ -302,17 +302,18 @@ def get_frequency(freq, data_frequency=None):
     elif unit.lower() == 'm' or unit == 'T':
         unit = 'T'
         alias = '{}T'.format(candle_size)
+        data_frequency = 'minute'
 
-        if data_frequency == 'daily':
+    elif unit.lower() == 'h':
+        if 'H' in supported_freqs:
+            unit = 'H'
+            alias = '{}H'.format(candle_size)
+
+        else:
+            candle_size = candle_size * 60
+            alias = '{}T'.format(candle_size)
             data_frequency = 'minute'
 
-    # elif unit.lower() == 'h':
-    #     candle_size = candle_size * 60
-    #
-    #     alias = '{}T'.format(candle_size)
-    #     if data_frequency == 'daily':
-    #         data_frequency = 'minute'
-
     else:
         raise InvalidHistoryFrequencyAlias(freq=freq)
 

catalyst/exchange/utils/exchange_utils.py

@@ -126,11 +126,11 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     filename = get_exchange_symbols_filename(exchange_name, is_local)
 
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-                pd.Timestamp('now', tz='UTC') - last_modified_time(
+            pd.Timestamp('now', tz='UTC') - last_modified_time(
-                filename)).days > 1):
+            filename)).days > 1):
         try:
             download_exchange_symbols(exchange_name, environ)
-        except Exception as e:
+        except Exception:
             pass
 
     if os.path.isfile(filename):
@@ -273,6 +273,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     key: str
     environ:
     rel_path: str
+    how: str
 
     Returns
     -------
@@ -316,6 +317,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
     obj: Object
     environ:
     rel_path: str
+    how: str
 
     """
     folder = get_algo_folder(algo_name, environ)
@@ -392,6 +394,71 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
         df.to_csv(handle, encoding='UTF_8')
 
 
+def clear_frame_stats_directory(algo_name):
+    """
+    remove the outdated directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+
+    Returns
+    -------
+    error: str
+
+    """
+    error = None
+    algo_folder = get_algo_folder(algo_name)
+    folder = os.path.join(algo_folder, 'frame_stats')
+    if os.path.exists(folder):
+        try:
+            shutil.rmtree(folder)
+        except OSError:
+            error = 'unable to remove {}, the analyze ' \
+                    'data will be inconsistent'.format(folder)
+    return error
+
+
+def remove_old_files(algo_name, today, rel_path, environ=None):
+    """
+    remove old files from a directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+    today: Timestamp
+    rel_path: str
+    environ:
+
+    Returns
+    -------
+    error: str
+
+    """
+
+    error = None
+    algo_folder = get_algo_folder(algo_name, environ)
+    folder = os.path.join(algo_folder, rel_path)
+    ensure_directory(folder)
+
+    # run on all files in the folder
+    for f in os.listdir(folder):
+        try:
+            file_path = os.path.join(folder, f)
+            creation_unix = os.path.getctime(file_path)
+            creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
+
+            # if the file is older than 30 days erase it
+            if today - pd.DateOffset(30) > creation_time:
+                os.unlink(file_path)
+        except OSError:
+            error = 'unable to erase files in {}'.format(folder)
+
+    return error
+
+
 def get_exchange_minute_writer_root(exchange_name, environ=None):
     """
     The minute writer folder for the exchange.
@@ -512,7 +579,7 @@ def get_common_assets(exchanges):
     return assets
 
 
-def resample_history_df(df, freq, field):
+def resample_history_df(df, freq, field, start_dt=None):
     """
     Resample the OHCLV DataFrame using the specified frequency.
 
@@ -540,7 +607,16 @@ def resample_history_df(df, freq, field):
     else:
         raise ValueError('Invalid field.')
 
-    resampled_df = df.resample(freq, closed='left', label='left').agg(agg)
+    resampled_df = df.resample(
+        freq, closed='left', label='left'
+    ).agg(agg)  # type: pd.DataFrame
+
+    # Because the samples are closed left, we get one more candle at
+    # the beginning then the requested number for bars. Removing this
+    # candle to avoid confusion.
+    if start_dt and not resampled_df.empty:
+        resampled_df = resampled_df[resampled_df.index >= start_dt]
+
     return resampled_df
 
 
@@ -566,8 +642,9 @@ def mixin_market_params(exchange_name, params, market):
         params['maker'] = 0.001
         params['taker'] = 0.002
 
-    elif 'maker' in market and 'taker' in market \
+    elif 'maker' in market and 'taker' in market and \
-            and market['maker'] is not None and market['taker'] is not None:
+            market['maker'] is not None and market['taker'] is not None:
+
         params['maker'] = market['maker']
         params['taker'] = market['taker']
 
@@ -649,12 +726,14 @@ def get_candles_df(candles, field, freq, bar_count, end_dt,
         values = [candle[field] for candle in candles[asset]]
         series = pd.Series(values, index=dates)
 
+        """
         series = series.reindex(
             periods,
             method='ffill',
             fill_value=previous_value,
         )
         series.sort_index(inplace=True)
+        """
         all_series[asset] = series
 
     df = pd.DataFrame(all_series)

catalyst/exchange/utils/factory.py

@@ -33,6 +33,8 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
         exchange_name=exchange_name,
         key=exchange_auth['key'],
         secret=exchange_auth['secret'],
+        password=exchange_auth['password'] if 'password'
+                                              in exchange_auth.keys() else '',
         base_currency=base_currency,
     )
     exchange_cache[key] = exchange

catalyst/exchange/utils/stats_utils.py

@@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None):
             )})
 
 
-def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
+def stats_to_algo_folder(stats, algo_namespace,
+                         folder_name, recorded_cols=None):
     """
     Saves the performance stats to the algo local folder.
 
@@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     ----------
     stats: list[Object]
     algo_namespace: str
+    folder_name: str
     recorded_cols: list[str]
 
     Returns
@@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     timestr = time.strftime('%Y%m%d')
     folder = get_algo_folder(algo_namespace)
 
-    stats_folder = os.path.join(folder, 'stats')
+    stats_folder = os.path.join(folder, folder_name)
     ensure_directory(stats_folder)
 
     filename = os.path.join(stats_folder, '{}.csv'.format(timestr))

catalyst/marketplace/utils/auth_utils.py

@@ -47,11 +47,11 @@ def get_key_secret(pubAddr, wallet='mew'):
     if wallet == 'mew':
         print('\nObtaining a key/secret pair to streamline all future '
               'requests with the authentication server.\n'
-              'Visit https://www.myetherwallet.com/signmsg.html and sign the'
+              'Visit https://www.myetherwallet.com/signmsg.html and sign the '
               'following message:\n{}'.format(nonce))
         signature = input('Copy and Paste the "sig" field from '
                           'the signature here (without the double quotes, '
-                          'only the HEX value:\n')
+                          'only the HEX value):\n')
     else:
         raise MarketplaceWalletNotSupported(wallet=wallet)
 

catalyst/marketplace/utils/bundle_utils.py

@@ -2,6 +2,7 @@ import os
 import shutil
 
 import bcolz
+import pandas as pd
 
 
 def merge_bundles(zsource, ztarget):
@@ -18,14 +19,13 @@ def merge_bundles(zsource, ztarget):
     """
     # TODO: find a way to do this iteratively instead of in-memory
     df_source = zsource.todataframe()
-    df_source.set_index('date', drop=False, inplace=True)
     df_target = ztarget.todataframe()
-    df_target.set_index('date', drop=False, inplace=True)
 
-    df = df_target.merge(
+    df = pd.concat(
-        right=df_source,
+        [df_source, df_target], ignore_index=True
-        how='right',
     )  # type: pd.DataFrame
+    df.drop_duplicates(inplace=True)
+    df.set_index(['date', 'symbol'], drop=False, inplace=True)
 
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))

docs/source/beginner-tutorial.rst

@@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in
 the ``examples`` directory:
 `dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
 
-.. code-block:: python
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
-
+   :language: python
-    import numpy as np
-    import pandas as pd
-    from logbook import Logger
-    import matplotlib.pyplot as plt 
-
-    from catalyst import run_algorithm
-    from catalyst.api import (order, record, symbol, order_target_percent, 
-            get_open_orders)
-    from catalyst.exchange.utils.stats_utils import extract_transactions
-
-    NAMESPACE = 'dual_moving_average'
-    log = Logger(NAMESPACE)
-
-    def initialize(context):
-        context.i = 0
-        context.asset = symbol('ltc_usd')
-        context.base_price = None
-
-
-    def handle_data(context, data):
-        # define the windows for the moving averages 
-        short_window = 50
-        long_window = 200
-
-        # Skip as many bars as long_window to properly compute the average
-        context.i += 1
-        if context.i < long_window:
-           return
-
-        # Compute moving averages calling data.history() for each 
-        # moving average with the appropriate parameters. We choose to use
-        # minute bars for this simulation -> freq="1m"
-        # Returns a pandas dataframe.
-        short_mavg = data.history(context.asset, 'price', 
-                            bar_count=short_window, frequency="1m").mean()
-        long_mavg = data.history(context.asset, 'price',
-                            bar_count=long_window, frequency="1m").mean()
-
-        # Let's keep the price of our asset in a more handy variable
-        price = data.current(context.asset, 'price')
-
-        # If base_price is not set, we use the current value. This is the
-        # price at the first bar which we reference to calculate price_change.
-        if context.base_price is None:
-            context.base_price = price
-        price_change = (price - context.base_price) / context.base_price
-
-        # Save values for later inspection
-        record(price=price,
-               cash=context.portfolio.cash,
-               price_change=price_change,
-               short_mavg=short_mavg,
-               long_mavg=long_mavg)
-
-        # Since we are using limit orders, some orders may not execute immediately
-        # we wait until all orders are executed before considering more trades.
-        orders = get_open_orders(context.asset)
-        if len(orders) > 0:
-            return
-
-        # Exit if we cannot trade
-        if not data.can_trade(context.asset):
-            return
-
-        # We check what's our position on our portfolio and trade accordingly
-        pos_amount = context.portfolio.positions[context.asset].amount
-
-        # Trading logic
-        if short_mavg > long_mavg and pos_amount == 0:
-           # we buy 100% of our portfolio for this asset
-           order_target_percent(context.asset, 1)
-        elif short_mavg < long_mavg and pos_amount > 0:
-           # we sell all our positions for this asset
-           order_target_percent(context.asset, 0)
-
-
-    def analyze(context, perf):
-
-        # Get the base_currency that was passed as a parameter to the simulation
-        exchange = list(context.exchanges.values())[0]
-        base_currency = exchange.base_currency.upper()
-
-        # First chart: Plot portfolio value using base_currency
-        ax1 = plt.subplot(411)
-        perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-        ax1.legend_.remove()
-        ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-        start, end = ax1.get_ylim()
-        ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Second chart: Plot asset price, moving averages and buys/sells
-        ax2 = plt.subplot(412, sharex=ax1)
-        perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-        ax2.legend_.remove()
-        ax2.set_ylabel('{asset}\n({base})'.format(
-            asset = context.asset.symbol,
-            base = base_currency
-            ))
-        start, end = ax2.get_ylim()
-        ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        transaction_df = extract_transactions(perf)
-        if not transaction_df.empty:
-            buy_df = transaction_df[transaction_df['amount'] > 0]
-            sell_df = transaction_df[transaction_df['amount'] < 0]
-            ax2.scatter(
-                buy_df.index.to_pydatetime(),
-                perf.loc[buy_df.index, 'price'],
-                marker='^',
-                s=100,
-                c='green',
-                label=''
-            )
-            ax2.scatter(
-                sell_df.index.to_pydatetime(),
-                perf.loc[sell_df.index, 'price'],
-                marker='v',
-                s=100,
-                c='red',
-                label=''
-            )
-
-        # Third chart: Compare percentage change between our portfolio
-        # and the price of the asset
-        ax3 = plt.subplot(413, sharex=ax1)
-        perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-        ax3.legend_.remove()
-        ax3.set_ylabel('Percent Change')
-        start, end = ax3.get_ylim()
-        ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Fourth chart: Plot our cash
-        ax4 = plt.subplot(414, sharex=ax1)
-        perf.cash.plot(ax=ax4)
-        ax4.set_ylabel('Cash\n({})'.format(base_currency))
-        start, end = ax4.get_ylim()
-        ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-        plt.show()
-
-
-    if __name__ == '__main__':
-        run_algorithm(
-                capital_base=1000,
-                data_frequency='minute',
-                initialize=initialize,
-                handle_data=handle_data,
-                analyze=analyze,
-                exchange_name='bitfinex',
-                algo_namespace=NAMESPACE,
-                base_currency='usd',
-                start=pd.to_datetime('2017-9-22', utc=True),
-                end=pd.to_datetime('2017-9-23', utc=True),
-            )
 
 In order to run the code above, you have to ingest the needed data first:
 
@@ -16609,7 +16455,49 @@ NaN
 
    </div>
 
+PyCharm IDE
+~~~~~~~~~~~
 
+PyCharm is an Integrated Development Environment (IDE) used in computer 
+programming, specifically for the Python language. It streamlines the continuos
+development of Python code, and among other things includes a debugger that 
+comes in handy to see the inner workings of Catalyst, and your trading 
+algorithms.
+
+Install
+^^^^^^^
+Install PyCharm from their `Website <https://www.jetbrains.com/pycharm/download/>`__.
+There is a free and open-source **Community** version.
+
+Setup
+^^^^^
+
+1. When creating a new project in PyCharm, right under you specify the Location,
+   click on **Project Interpreter** to display a drop down menu
+
+2. Select **Existing interpreter**, click the gear box right next to it and 
+   select 'add local'. Depending on your installation, select either 
+   "*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to
+   navigate to your catalyst env and select the Python binary file: 
+   ``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows 
+   installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). 
+   Select OK. You may want to click on *Make available to all projects* for your
+   future reference. Click OK again, and create your new environment using the
+   set up of your virtual environment.
+
+Alternatively, if you already have your project created, in Windows do:
+
+1. File -> Default Settings -> Project Interpreter. Click the gear box next to 
+   the project interpreter and select ‘add local’, and follow the steps from the
+   second step above.
+
+On MacOS:
+
+1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ -> 
+   Project Interpreter. Click the gear box next to the project interpreter 
+   and select ‘add local’, and follow the steps from the second step above.
+
+You should now be able to run your project/scripts in PyCharm.
 
 Next steps
 ~~~~~~~~~~

docs/source/example-algos.rst

@@ -52,35 +52,8 @@ Buy BTC Simple Algorithm
 
 Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
 
-.. code-block:: python
+.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
-
+   :language: python
-  '''
-    Run this example, by executing the following from your terminal:
-      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
-      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    If you want to run this code using another exchange, make sure that 
-    the asset is available on that exchange. For example, if you were to run 
-    it for exchange Poloniex, you would need to edit the following line:
-
-      context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
-
-    and specify exchange poloniex as follows:
-    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
-    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    To see which assets are available on each exchange, visit:
-    https://www.enigma.co/catalyst/status
-  '''
-
-  from catalyst.api import order, record, symbol
-
-  def initialize(context):
-      context.asset = symbol('btc_usd')
-
-  def handle_data(context, data):
-      order(context.asset, 1)
-      record(btc = data.current(context.asset, 'price'))
 
 This simple algorithm does not produce any output nor displays any chart.
 
@@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart.
 Buy and Hodl Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
-
 First ingest the historical pricing data needed to run this algorithm:
 
 .. code-block:: bash
@@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
 earlier date, you'll get an error), and the most recent date you can choose is 
 one day prior to the current date. 
 
+Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
 
-.. code-block:: python
+.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
-
+   :language: python
-  #!/usr/bin/env python
-  #
-  # Copyright 2017 Enigma MPC, Inc.
-  # Copyright 2015 Quantopian, Inc.
-  #
-  # Licensed under the Apache License, Version 2.0 (the "License");
-  # you may not use this file except in compliance with the License.
-  # You may obtain a copy of the License at
-  #
-  #     http://www.apache.org/licenses/LICENSE-2.0
-  #
-  # Unless required by applicable law or agreed to in writing, software
-  # distributed under the License is distributed on an "AS IS" BASIS,
-  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  # See the License for the specific language governing permissions and
-  # limitations under the License.
-  import pandas as pd
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order_target_value, symbol, record,
-      cancel_order, get_open_orders, )
-
-
-  def initialize(context):
-      context.ASSET_NAME = 'btc_usd'
-      context.TARGET_HODL_RATIO = 0.8
-      context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
-
-      context.is_buying = True
-      context.asset = symbol(context.ASSET_NAME)
-
-      context.i = 0
-
-
-  def handle_data(context, data):
-      context.i += 1
-
-      starting_cash = context.portfolio.starting_cash
-      target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
-      reserve_value = context.RESERVE_RATIO * starting_cash
-
-      # Cancel any outstanding orders
-      orders = get_open_orders(context.asset) or []
-      for order in orders:
-          cancel_order(order)
-
-      # Stop buying after passing the reserve threshold
-      cash = context.portfolio.cash
-      if cash <= reserve_value:
-          context.is_buying = False
-
-      # Retrieve current asset price from pricing data
-      price = data.current(context.asset, 'price')
-
-      # Check if still buying and could (approximately) afford another purchase
-      if context.is_buying and cash > price:
-          print('buying')
-          # Place order to make position in asset equal to target_hodl_value
-          order_target_value(
-              context.asset,
-              target_hodl_value,
-              limit_price=price * 1.1,
-          )
-
-      record(
-          price=price,
-          volume=data.current(context.asset, 'volume'),
-          cash=cash,
-          starting_cash=context.portfolio.starting_cash,
-          leverage=context.account.leverage,
-      )
-
-
-  def analyze(context=None, results=None):
-
-      # Plot the portfolio and asset data.
-      ax1 = plt.subplot(611)
-      results[['portfolio_value']].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio Value (USD)')
-
-      ax2 = plt.subplot(612, sharex=ax1)
-      ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-      results[['price']].plot(ax=ax2)
-
-      trans = results.ix[[t != [] for t in results.transactions]]
-      buys = trans.ix[
-          [t[0]['amount'] > 0 for t in trans.transactions]
-      ]
-      ax2.scatter(
-          buys.index.to_pydatetime(),
-          results.price[buys.index],
-          marker='^',
-          s=100,
-          c='g',
-          label=''
-      )
-
-      ax3 = plt.subplot(613, sharex=ax1)
-      results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-      ax3.set_ylabel('Leverage ')
-
-      ax4 = plt.subplot(614, sharex=ax1)
-      results[['starting_cash', 'cash']].plot(ax=ax4)
-      ax4.set_ylabel('Cash (USD)')
-
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]] = results[[
-          'treasury_period_return',
-          'algorithm_period_return',
-          'benchmark_period_return',
-      ]]
-
-      ax5 = plt.subplot(615, sharex=ax1)
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]].plot(ax=ax5)
-      ax5.set_ylabel('Percent Change')
-
-      ax6 = plt.subplot(616, sharex=ax1)
-      results[['volume']].plot(ax=ax6)
-      ax6.set_ylabel('Volume (mCoins/5min)')
-
-      plt.legend(loc=3)
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-          capital_base=10000,
-          data_frequency='daily',
-          initialize=initialize,
-          handle_data=handle_data,
-          analyze=analyze,
-          exchange_name='bitfinex',
-          algo_namespace='buy_and_hodl',
-          base_currency='usd',
-          start=pd.to_datetime('2015-03-01', utc=True),
-          end=pd.to_datetime('2017-10-31', utc=True),
-      )
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
 
@@ -278,166 +102,13 @@ one day prior to the current date.
 Dual Moving Average Crossover
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
-
 This strategy is covered in detail in the last part of 
 `this tutorial <beginner-tutorial.html#history>`_.
 
-.. code-block:: python
+Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
 
-  import numpy as np
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
-  import pandas as pd
+   :language: python
-  from logbook import Logger
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order, record, symbol, order_target_percent,
-          get_open_orders)
-  from catalyst.exchange.stats_utils import extract_transactions
-
-  NAMESPACE = 'dual_moving_average'
-  log = Logger(NAMESPACE)
-
-  def initialize(context):
-      context.i = 0
-      context.asset = symbol('ltc_usd')
-      context.base_price = None
-
-
-  def handle_data(context, data):
-      # define the windows for the moving averages
-      short_window = 50
-      long_window = 200
-
-      # Skip as many bars as long_window to properly compute the average
-      context.i += 1
-      if context.i < long_window:
-         return
-
-      # Compute moving averages calling data.history() for each
-      # moving average with the appropriate parameters. We choose to use
-      # minute bars for this simulation -> freq="1m"
-      # Returns a pandas dataframe.
-      short_mavg = data.history(context.asset, 'price',
-                          bar_count=short_window, frequency="1m").mean()
-      long_mavg = data.history(context.asset, 'price',
-                          bar_count=long_window, frequency="1m").mean()
-
-      # Let's keep the price of our asset in a more handy variable
-      price = data.current(context.asset, 'price')
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-      price_change = (price - context.base_price) / context.base_price
-
-      # Save values for later inspection
-      record(price=price,
-             cash=context.portfolio.cash,
-             price_change=price_change,
-             short_mavg=short_mavg,
-             long_mavg=long_mavg)
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.asset)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.asset):
-          return
-
-      # We check what's our position on our portfolio and trade accordingly
-      pos_amount = context.portfolio.positions[context.asset].amount
-
-      # Trading logic
-      if short_mavg > long_mavg and pos_amount == 0:
-         # we buy 100% of our portfolio for this asset
-         order_target_percent(context.asset, 1)
-      elif short_mavg < long_mavg and pos_amount > 0:
-         # we sell all our positions for this asset
-         order_target_percent(context.asset, 0)
-
-
-  def analyze(context, perf):
-
-      # Get the base_currency that was passed as a parameter to the simulation
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # First chart: Plot portfolio value using base_currency
-      ax1 = plt.subplot(411)
-      perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-      ax1.legend_.remove()
-      ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-      start, end = ax1.get_ylim()
-      ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Second chart: Plot asset price, moving averages and buys/sells
-      ax2 = plt.subplot(412, sharex=ax1)
-      perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-      ax2.legend_.remove()
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset = context.asset.symbol,
-          base = base_currency
-          ))
-      start, end = ax2.get_ylim()
-      ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index, 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index, 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      # Third chart: Compare percentage change between our portfolio
-      # and the price of the asset
-      ax3 = plt.subplot(413, sharex=ax1)
-      perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-      ax3.legend_.remove()
-      ax3.set_ylabel('Percent Change')
-      start, end = ax3.get_ylim()
-      ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Fourth chart: Plot our cash
-      ax4 = plt.subplot(414, sharex=ax1)
-      perf.cash.plot(ax=ax4)
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-      start, end = ax4.get_ylim()
-      ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-              capital_base=1000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-9-22', utc=True),
-              end=pd.to_datetime('2017-9-23', utc=True),
-          )
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
 
@@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of
 Mean Reversion Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
-
 This algorithm is based on a simple momentum strategy. When the cryptoasset goes
 up quickly, we're going to buy; when it goes down quickly, we're going to sell. 
 Hopefully, we'll ride the waves.
@@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type:
 
    python mean_reversion_simple.py
 
-.. code-block:: python
+Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
 
-  import os
+.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
-  import tempfile
+   :language: python
-  import time
-
-  import numpy as np
-  import pandas as pd
-  import talib
-  from logbook import Logger
-
-  from catalyst import run_algorithm
-  from catalyst.api import symbol, record, order_target_percent, get_open_orders
-  from catalyst.exchange.stats_utils import extract_transactions
-  # We give a name to the algorithm which Catalyst will use to persist its state.
-  # In this example, Catalyst will create the `.catalyst/data/live_algos`
-  # directory. If we stop and start the algorithm, Catalyst will resume its
-  # state using the files included in the folder.
-  from catalyst.utils.paths import ensure_directory
-
-  NAMESPACE = 'mean_reversion_simple'
-  log = Logger(NAMESPACE)
-
-
-  # To run an algorithm in Catalyst, you need two functions: initialize and
-  # handle_data.
-
-  def initialize(context):
-      # This initialize function sets any data or variables that you'll use in
-      # your algorithm.  For instance, you'll want to define the trading pair (or
-      # trading pairs) you want to backtest.  You'll also want to define any
-      # parameters or values you're going to use.
-
-      # In our example, we're looking at Neo in USD.
-      context.neo_eth = symbol('neo_usd')
-      context.base_price = None
-      context.current_day = None
-
-      context.RSI_OVERSOLD = 30
-      context.RSI_OVERBOUGHT = 80
-      context.CANDLE_SIZE = '15T'
-
-      context.start_time = time.time()
-
-
-  def handle_data(context, data):
-      # This handle_data function is where the real work is done.  Our data is
-      # minute-level tick data, and each minute is called a frame.  This function
-      # runs on each frame of the data.
-
-      # We flag the first period of each day.
-      # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-      # would only execute once. This method works with minute and daily
-      # frequencies.
-      today = data.current_dt.floor('1D')
-      if today != context.current_day:
-          context.traded_today = False
-          context.current_day = today
-
-      # We're computing the volume-weighted-average-price of the security
-      # defined above, in the context.neo_eth variable.  For this example, we're 
-      # using three bars on the 15 min bars.
-
-      # The frequency attribute determine the bar size. We use this convention
-      # for the frequency alias:
-      # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-      prices = data.history(
-          context.neo_eth,
-          fields='close',
-          bar_count=50,
-          frequency=context.CANDLE_SIZE
-      )
-
-      # Ta-lib calculates various technical indicator based on price and
-      # volume arrays.
-
-      # In this example, we are comp
-      rsi = talib.RSI(prices.values, timeperiod=14)
-
-      # We need a variable for the current price of the security to compare to
-      # the average. Since we are requesting two fields, data.current()
-      # returns a DataFrame with
-      current = data.current(context.neo_eth, fields=['close', 'volume'])
-      price = current['close']
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-
-      price_change = (price - context.base_price) / context.base_price
-      cash = context.portfolio.cash
-
-      # Now that we've collected all current data for this frame, we use
-      # the record() method to save it. This data will be available as
-      # a parameter of the analyze() function for further analysis.
-      record(
-          price=price,
-          volume=current['volume'],
-          price_change=price_change,
-          rsi=rsi[-1],
-          cash=cash
-      )
-
-      # We are trying to avoid over-trading by limiting our trades to
-      # one per day.
-      if context.traded_today:
-          return
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.neo_eth)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.neo_eth):
-          return
-
-      # Another powerful built-in feature of the Catalyst backtester is the
-      # portfolio object.  The portfolio object tracks your positions, cash,
-      # cost basis of specific holdings, and more.  In this line, we calculate
-      # how long or short our position is at this minute.   
-      pos_amount = context.portfolio.positions[context.neo_eth].amount
-
-      if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-          log.info(
-              '{}: buying - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          # Set a style for limit orders,
-          limit_price = price * 1.005
-          order_target_percent(
-              context.neo_eth, 1, limit_price=limit_price
-          )
-          context.traded_today = True
-
-      elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-          log.info(
-              '{}: selling - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          limit_price = price * 0.995
-          order_target_percent(
-              context.neo_eth, 0, limit_price=limit_price
-          )
-          context.traded_today = True
-
-
-  def analyze(context=None, perf=None):
-      end = time.time()
-      log.info('elapsed time: {}'.format(end - context.start_time))
-
-      import matplotlib.pyplot as plt
-      # The base currency of the algo exchange
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # Plot the portfolio value over time.
-      ax1 = plt.subplot(611)
-      perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
-
-      # Plot the price increase or decrease over time.
-      ax2 = plt.subplot(612, sharex=ax1)
-      perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset=context.neo_eth.symbol, base=base_currency
-      ))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      ax4 = plt.subplot(613, sharex=ax1)
-      perf.loc[:, 'cash'].plot(
-          ax=ax4, label='Base Currency ({})'.format(base_currency)
-      )
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-
-      perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-      ax5 = plt.subplot(614, sharex=ax1)
-      perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-      ax5.set_ylabel('Percent\nChange')
-
-      ax6 = plt.subplot(615, sharex=ax1)
-      perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-      ax6.set_ylabel('RSI')
-      ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
-      ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
-
-      if not transaction_df.empty:
-          ax6.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'rsi'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax6.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'rsi'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-      plt.legend(loc=3)
-      start, end = ax6.get_ylim()
-      ax6.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-      pass
-
-
-  if __name__ == '__main__':
-      # The execution mode: backtest or live
-      MODE = 'backtest'
-
-      if MODE == 'backtest':
-          folder = os.path.join(
-              tempfile.gettempdir(), 'catalyst', NAMESPACE
-          )
-          ensure_directory(folder)
-
-          timestr = time.strftime('%Y%m%d-%H%M%S')
-          out = os.path.join(folder, '{}.p'.format(timestr))
-          # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
-          run_algorithm(
-              capital_base=10000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-10-01', utc=True),
-              end=pd.to_datetime('2017-11-10', utc=True),
-              output=out
-          )
-          log.info('saved perf stats: {}'.format(out))
-
-      elif MODE == 'live':
-          run_algorithm(
-              capital_base=0.5,
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bittrex',
-              live=True,
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              live_graph=False
-          )
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
 
@@ -763,8 +158,6 @@ strategy.
 Simple Universe
 ~~~~~~~~~~~~~~~
 
-Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
-
 This example aims to provide an easy way for users to learn how to 
 collect data from any given exchange and select a subset of the available
 currency pairs for trading. You simply need to specify the exchange and 
@@ -791,142 +184,10 @@ of the file:
   
   catalyst ingest-exchange -x bitfinex -f minute
 
-.. code-block:: bash
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
-
-  python simple_universe.py
-
-Credits: This code was originally submitted by `Abner Ayala-Acevedo 
-<https://github.com/abnera>`_. Thank you!
-
-.. code-block:: python
-
-  from datetime import timedelta
-
-  import numpy as np
-  import pandas as pd
-
-  from catalyst import run_algorithm
-  from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
-  from catalyst.api import (symbols, )
-
-
-  def initialize(context):
-      context.i = -1  # minute counter
-      context.exchange = context.exchanges.values()[0].name.lower()  
-      context.base_currency = context.exchanges.values()[0].base_currency.lower()  
-
-
-  def handle_data(context, data):
-      context.i += 1
-      lookback_days = 7  # 7 days
-
-      # current date & time in each iteration formatted into a string
-      now = data.current_dt
-      date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-      lookback_date = now - timedelta(days=lookback_days) 
-      # keep only the date as a string, discard the time
-      lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  
-
-      one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
-      # update universe everyday at midnight
-      if not context.i % one_day_in_minutes:
-          context.universe = universe(context, lookback_date, date)
-
-      # get data every 30 minutes
-      minutes = 30
-      # get lookback_days of history data: that is 'lookback' number of bins
-      lookback = one_day_in_minutes / minutes * lookback_days  
-      if not context.i % minutes and context.universe:
-          # we iterate for every pair in the current universe
-          for coin in context.coins:
-              pair = str(coin.symbol)
-
-              # Get 30 minute interval OHLCV data. This is the standard data 
-              # required for candlestick or indicators/signals. Return Pandas
-              # DataFrames. 30T means 30-minute re-sampling of one minute data. 
-              # Adjust it to your desired time interval as needed.
-              opened = fill(data.history(coin, 'open', 
-                                    bar_count=lookback, frequency='30T')).values
-              high = fill(data.history(coin, 'high', 
-                                    bar_count=lookback, frequency='30T')).values
-              low = fill(data.history(coin, 'low', 
-                                    bar_count=lookback, frequency='30T')).values
-              close = fill(data.history(coin, 'price', 
-                                    bar_count=lookback, frequency='30T')).values
-              volume = fill(data.history(coin, 'volume', 
-                                    bar_count=lookback, frequency='30T')).values
-
-              # close[-1] is the last value in the set, which is the equivalent 
-              # to current price (as in the most recent value)
-              # displays the minute price for each pair every 30 minutes
-              print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
-                      now=now, 
-                      pair=pair, 
-                      o=opened[-1], 
-                      h=high[-1], 
-                      l=low[-1],
-                      c=close[-1],
-                      v=volume[-1],
-                   ))
-
-              # -------------------------------------------------------------
-              # --------------- Insert Your Strategy Here -------------------
-              # -------------------------------------------------------------
-
-
-  def analyze(context=None, results=None):
-      pass
-
-
-  # Get the universe for a given exchange and a given base_currency market
-  # Example: Poloniex BTC Market
-  def universe(context, lookback_date, current_date):
-      # get all the pairs for the given exchange
-      json_symbols = get_exchange_symbols(context.exchange)  
-      # convert into a DataFrame for easier processing
-      df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) 
-      df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
-      df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
-
-      # Filter all the pairs to get only the ones for a given base_currency
-      df = df[df['base_currency'] == context.base_currency]
-
-      # Filter all the pairs to ensure that pair existed in the current date range
-      df = df[df.start_date < lookback_date]
-      df = df[df.end_daily >= current_date]
-      context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
-
-      return df.symbol.tolist()
-
-
-  # Replace all NA, NAN or infinite values with its nearest value
-  def fill(series):
-      if isinstance(series, pd.Series):
-          return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-      elif isinstance(series, np.ndarray):
-          return pd.Series(series).replace(
-                       [np.inf, -np.inf], np.nan
-                      ).ffill().bfill().values
-      else:
-          return series
-
-
-  if __name__ == '__main__':
-      start_date = pd.to_datetime('2017-11-10', utc=True)
-      end_date = pd.to_datetime('2017-11-13', utc=True)
-
-      performance = run_algorithm(start=start_date, end=end_date,
-                                  capital_base=100.0,  # amount of base_currency
-                                  initialize=initialize,
-                                  handle_data=handle_data,
-                                  analyze=analyze,
-                                  exchange_name='bitfinex',
-                                  data_frequency='minute',
-                                  base_currency='btc',
-                                  live=False,
-                                  live_graph=False,
-                                  algo_namespace='simple_universe')
 
+.. literalinclude:: ../../catalyst/examples/simple_universe.py
+   :language: python
 
 
 .. _portfolio_optimization:
@@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used
 in writting the following article: 
 `Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
 
-.. code-block:: python
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
 
-  '''
+.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
-     You can run this code using the Python interpreter:
+   :language: python
-
-     $ python portfolio_optimization.py
-  '''
-
-  from __future__ import division
-  import os
-  import pytz
-  import numpy as np
-  import pandas as pd
-  from scipy.optimize import minimize
-  import matplotlib.pyplot as plt
-  from datetime import datetime
-
-  from catalyst.api import record, symbol, symbols, order_target_percent
-  from catalyst.utils.run_algo import run_algorithm
-
-  np.set_printoptions(threshold='nan', suppress=True)
-
-
-  def initialize(context):
-     # Portfolio assets list
-     context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
-                              'xmr_usdt')
-     context.nassets = len(context.assets)
-     # Set the time window that will be used to compute expected return 
-     # and asset correlations
-     context.window = 180
-     # Set the number of days between each portfolio rebalancing
-     context.rebalance_period = 30                   
-     context.i = 0
-
-     
-  def handle_data(context, data):
-     # Only rebalance at the beggining of the algorithm execution and 
-     # every multiple of the rebalance period
-     if context.i == 0 or context.i%context.rebalance_period == 0:
-         n = context.window
-         prices = data.history(context.assets, fields='price', 
-                               bar_count=n+1, frequency='1d') 
-         pr = np.asmatrix(prices)
-         t_prices = prices.iloc[1:n+1]
-         t_val = t_prices.values
-         tminus_prices = prices.iloc[0:n]
-         tminus_val = tminus_prices.values
-         # Compute daily returns (r)
-         r = np.asmatrix(t_val/tminus_val-1)
-         # Compute the expected returns of each asset with the average 
-         # daily return for the selected time window
-         m = np.asmatrix(np.mean(r, axis=0))
-         # ###
-         stds = np.std(r, axis=0)
-         # Compute excess returns matrix (xr)
-         xr = r - m
-         # Matrix algebra to get variance-covariance matrix
-         cov_m = np.dot(np.transpose(xr),xr)/n
-         # Compute asset correlation matrix (informative only)
-         corr_m = cov_m/np.dot(np.transpose(stds),stds)
-         
-         # Define portfolio optimization parameters
-         n_portfolios = 50000
-         results_array = np.zeros((3+context.nassets,n_portfolios))
-         for p in xrange(n_portfolios):
-             weights = np.random.random(context.nassets)
-             weights /= np.sum(weights)
-             w = np.asmatrix(weights)
-             p_r = np.sum(np.dot(w,np.transpose(m)))*365
-             p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
-             
-             #store results in results array
-             results_array[0,p] = p_r
-             results_array[1,p] = p_std
-             #store Sharpe Ratio (return / volatility) - risk free rate element 
-             #excluded for simplicity
-             results_array[2,p] = results_array[0,p] / results_array[1,p]
-             i = 0
-             for iw in weights:
-                 results_array[3+i,p] = weights[i]
-                 i += 1
-         
-         #convert results array to Pandas DataFrame
-         results_frame = pd.DataFrame(np.transpose(results_array),
-                            columns=['r','stdev','sharpe']+context.assets)
-         #locate position of portfolio with highest Sharpe Ratio
-         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
-         #locate positon of portfolio with minimum standard deviation
-         min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
-         
-         #order optimal weights for each asset
-         for asset in context.assets:
-             if data.can_trade(asset):
-                 order_target_percent(asset, max_sharpe_port[asset])
-         
-         #create scatter plot coloured by Sharpe Ratio
-         plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
-         plt.xlabel('Volatility')
-         plt.ylabel('Returns')
-         plt.colorbar()
-         #plot red star to highlight position of portfolio with highest Sharpe Ratio
-         plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
-         #plot green star to highlight position of minimum variance portfolio
-         plt.show()
-         print(max_sharpe_port)
-         record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
-     context.i += 1
-     
-         
-  def analyze(context=None, results=None):
-     # Form DataFrame with selected data
-     data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
-     
-     # Save results in CSV file
-     filename = os.path.splitext(os.path.basename(__file__))[0]
-     data.to_csv(filename + '.csv')
-
-
-  # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
-  start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
-  end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) 
-  results = run_algorithm(initialize=initialize,
-                          handle_data=handle_data,
-                          analyze=analyze,
-                          start=start,
-                          end=end,
-                          exchange_name='poloniex',
-                          capital_base=100000, )
 
 .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
    :align: center

docs/source/install.rst

@@ -47,8 +47,10 @@ you can install MiniConda, which is a smaller footprint (fewer packages and
 smaller size) than its big brother Anaconda, but it still contains all the 
 main packages needed. To install MiniConda, you can follow these steps:
 
-1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select either 
-   for your Operating System.
+   Python 3.6 (recommended) or Python 2.7 for your Operating System. The 
+   `Enigma Data Marketplace <https://enigmampc.github.io/marketplace/>`_ will 
+   require Python3, that's why we are recommending to opt for the newer version.
 2. Install MiniConda. See the `Installation Instructions 
    <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
 3. Ensure the correct installation by running ``conda list`` in a Terminal 
@@ -64,18 +66,27 @@ main packages needed. To install MiniConda, you can follow these steps:
 
 Once either Conda or MiniConda has been set up you can install Catalyst:
 
-1. Download the file `python2.7-environment.yml 
+1. Download the file `python3.6-environment.yml 
-   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python3.6-environment.yml>`_ 
+   (recommended) or `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_ 
+   matching your Conda installation from step #1 above.
 
      To download, simply click on the 'Raw' button and save the file locally 
      to a folder you can remember. Make sure that the file gets saved with the
      ``.yml`` extension, and nothing like a ``.txt`` file or anything else.
 
 2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
-   saved the above ``python2.7-environment.yml`` file.
+   saved the above ``.yml`` file.
 
 3. Install using this file. This step can take about 5-10 minutes to install.
 
+   .. code-block:: bash
+
+      conda env create -f python3.6-environment.yml
+
+  or
+
    .. code-block:: bash
 
       conda env create -f python2.7-environment.yml
@@ -122,10 +133,19 @@ with the following steps:
 
 2. Create the environment:
    
+   for python 2.7:    
+
    .. code-block:: bash
 
       conda create --name catalyst python=2.7 scipy zlib
     
+  or for python 3.6:
+    
+   .. code-block:: bash
+
+      conda create --name catalyst python=3.6 scipy zlib
+
+
 3. Activate the environment:
 
    **Linux or MacOS:**
@@ -443,12 +463,22 @@ about matplotlib backends, please refer to the
 Windows Requirements
 --------------------
 
-In Windows, you will first need to install the `Microsoft Visual C++ Compiler 
+In Windows, you will first need to install the Microsoft Visual C++ Compiler, 
-for Python 2.7 
+which is different depending on the version of Python that you plan to use:
-<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
+
-package contains the compiler and the set of system headers necessary for 
+* Python 3.5, 3.6: `Visual C++ 2015 Build Tools 
-producing binary wheels for Python 2.7 packages. If it's not already in your 
+  <http://landinghub.visualstudio.com/visual-cpp-build-tools>`_, 
-system, download it and install it before proceeding to the next step.
+  which installs Visual C++ version 14.0. **This is the recommended version**
+
+* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7 
+  <https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_, which 
+  installs version Visual C++ version 9.0
+
+This package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python packages. If it's not already in your 
+system, download it and install it before proceeding to the next step. If you 
+need additional help, or are looking for other versions of Visual C++ for 
+Windows (only advanced users), follow `this link <https://wiki.python.org/moin/WindowsCompilers>`_.
 
 Once you have the above compiler installed, the easiest and best supported way 
 to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't 

docs/source/live-trading.rst

@@ -30,22 +30,24 @@ Paper Trading vs Live Trading modes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Catalyst currently supports three different modes in which you can execute your
-trading algorithm. The first is backtesting, which is covered extensively in the 
+trading algorithm. The first is **backtesting**, which is covered extensively in
-tutorial, and uses historical data to run your algorithm. There is no 
+the tutorial, and uses historical data to run your algorithm. There is no 
 interaction with the exchange in backtesting mode, and this is the first mode 
 that you should test any new algorithm.
 
 Once you are confident with the simulations that you have obtained with your
 algorithm in backtesting, you may switch to live trading, where you have two
 different modes:
-* *Paper Trading*: The simulated algorithm runs in real time, and fetches 
+
-pricing data in real time from the exchange, but the orders never reach the 
+* **Paper Trading**: The simulated algorithm runs in real time, and fetches 
-exchange, and are instead kept within Catalyst and simulated. No real currency
+  pricing data in real time from the exchange, but the orders never reach the 
-is bought or sold. Think of it as a `backtesting happening in real time`.
+  exchange, and are instead kept within Catalyst and simulated. No real currency
-* *Live Trading*: This is the proper live trading mode in which an algorithm
+  is bought or sold. Think of it as a `backtesting happening in real time`.
-runs in real time, fetching pricing data from live exchanges and placing orders 
+
-against the exchange. Real currency is transacted on the exchange driven by the 
+* **Live Trading**: This is the proper live trading mode in which an algorithm
-algorithm.
+  runs in real time, fetching pricing data from live exchanges and placing 
+  orders against the exchange. Real currency is transacted on the exchange 
+  driven by the algorithm.
 
 These three modes are controlled by the following variables:
 
@@ -174,6 +176,22 @@ Here is the breakdown of the new arguments:
 - ``simulate_orders``: Enables the paper trading mode, in which orders are
   simulated in Catalyst instead of processed on the exchange. It defaults to 
   ``True``.
+- ``end_date``: When setting the end_date to a time in the **future**,
+  it will schedule the live algo to finish gracefully at the specified date.
+- ``start_date``: (**Will be implemented in the future**)
+  The live algo starts by default in the present, as mentioned above.
+  by setting the start_date to a time in the future, the algorithm would
+  essentially sleep and when the predefined time comes, it would start executing.
+
+
+
+The `catalyst live` command offers additional parameters.
+You can learn more by running the following from the command line:
+
+.. code-block:: bash
+
+    catalyst live --help
+
 
 Here is a complete algorithm for reference:
 `Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_

etc/python2.7-environment.yml

@@ -1,9 +1,11 @@
 name: catalyst
 channels:
 - defaults
+- conda-forge
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
+- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -20,7 +22,7 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
-  - ccxt==1.10.1049
+  - ccxt==1.10.1094
   - web3==4.0.0b7
   - requests-toolbelt==0.8.0
   - click==6.7
@@ -59,4 +61,4 @@ dependencies:
   - tables==3.4.2
   - toolz==0.8.2
   - urllib3==1.22
-  - enigma-catalyst>=0.3
+  - enigma-catalyst>=0.5

etc/python3.6-environment.yml

@@ -0,0 +1,90 @@
+name: catalyst
+channels:
+- defaults
+- conda-forge
+dependencies:
+- ca-certificates=2017.08.26
+- certifi=2018.1.18
+- intel-openmp=2018.0.0
+- mkl=2018.0.1
+- numpy=1.14.0
+- openssl=1.0.2n
+- matplotlib=2.1.2=py36_0
+- pip=9.0.1
+- python=3.6.4
+- scipy=1.0.0
+- setuptools=38.4.0=py36_0
+- sqlite=3.22.0
+- tk=8.6.7
+- wheel=0.30.0
+- xz=5.2.3
+- zlib=1.2.11
+- pip:
+  - aiodns==1.1.1
+  - aiohttp==3.0.1
+  - alembic==0.9.7
+  - async-timeout==2.0.0
+  - attrdict==2.0.0
+  - attrs==17.4.0
+  - bcolz==0.12.1
+  - boto3==1.5.27
+  - botocore==1.8.41
+  - bottleneck==1.2.1
+  - cchardet==2.1.1
+  - ccxt==1.10.1102
+  - chardet==3.0.4
+  - click==6.7
+  - contextlib2==0.5.5
+  - cyordereddict==1.0.0
+  - cython==0.27.3
+  - cytoolz==0.9.0
+  - decorator==4.2.1
+  - docutils==0.14
+  - empyrical==0.2.1
+  - enigma-catalyst>=0.5.3
+  - eth-abi==1.0.0b0
+  - eth-account==0.1.0a2
+  - eth-keyfile==0.5.1
+  - eth-keys==0.2.0b1
+  - eth-rlp==0.1.0a2
+  - eth-utils==1.0.0b1
+  - hexbytes==0.1.0b0
+  - idna==2.6
+  - idna-ssl==1.0.0
+  - intervaltree==2.1.0
+  - jmespath==0.9.3
+  - logbook==1.2.1
+  - lru-dict==1.1.6
+  - lxml==4.1.1
+  - mako==1.0.7
+  - markupsafe==1.0
+  - multidict==4.1.0
+  - multipledispatch==0.4.9
+  - networkx==2.1
+  - numexpr==2.6.4
+  - pandas==0.19.2
+  - pandas-datareader==0.6.0
+  - patsy==0.5.0
+  - pycares==2.3.0
+  - pycryptodome==3.4.11
+  - pysha3==1.0.2
+  - python-dateutil==2.6.1
+  - python-editor==1.0.3
+  - pytz==2018.3
+  - redo==1.6
+  - requests==2.18.4
+  - requests-file==1.4.3
+  - requests-ftp==0.3.1
+  - requests-toolbelt==0.8.0
+  - rlp==0.6.0
+  - s3transfer==0.1.12
+  - six==1.11.0
+  - sortedcontainers==1.5.9
+  - sqlalchemy==1.2.2
+  - statsmodels==0.8.0
+  - tables==3.4.2
+  - toolz==0.9.0
+  - urllib3==1.22
+  - web3==4.0.0b9
+  - wrapt==1.10.11
+  - yarl==1.1.0

etc/requirements.txt

@@ -81,7 +81,7 @@ empyrical==0.2.1
 tables==3.3.0
 
 #Catalyst dependencies
-ccxt==1.10.1049
+ccxt==1.10.1094
 boto3==1.4.8
 redo==1.6
 web3==4.0.0b7

tests/exchange/test_suites/test_suite_bundle.py

@@ -37,7 +37,7 @@ class TestSuiteBundle:
         return data_portal
 
     def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     freq, data_frequency, data_portal):
+                                     freq, data_frequency, data_portal, field):
         """
         Creates DataFrames from the bundle and exchange for the specified
         data set.
@@ -62,8 +62,8 @@ class TestSuiteBundle:
         with log_catcher:
             symbols = [asset.symbol for asset in assets]
             print(
-                'comparing data for {}/{} with {} timeframe until {}'.format(
+                'comparing {} for {}/{} with {} timeframe until {}'.format(
-                    exchange.name, symbols, freq, end_dt
+                    field, exchange.name, symbols, freq, end_dt
                 )
             )
             data['bundle'] = data_portal.get_history_window(
@@ -71,13 +71,13 @@ class TestSuiteBundle:
                 end_dt=end_dt,
                 bar_count=bar_count,
                 frequency=freq,
-                field='close',
+                field=field,
                 data_frequency=data_frequency,
             )
             set_print_settings()
             print(
-                'the bundle first / last row:\n{}'.format(
+                'the bundle data:\n{}'.format(
-                    data['bundle'].iloc[[-1, 0]]
+                    data['bundle']
                 )
             )
             candles = exchange.get_candles(
@@ -88,14 +88,14 @@ class TestSuiteBundle:
             )
             data['exchange'] = get_candles_df(
                 candles=candles,
-                field='close',
+                field=field,
                 freq=freq,
                 bar_count=bar_count,
                 end_dt=end_dt,
             )
             print(
-                'the exchange first / last row:\n{}'.format(
+                'the exchange data:\n{}'.format(
-                    data['exchange'].iloc[[-1, 0]]
+                    data['exchange']
                 )
             )
             for source in data:
@@ -118,8 +118,10 @@ class TestSuiteBundle:
                     check_less_precise=min([a.decimals for a in assets]),
                 )
             except Exception as e:
-                print('Some differences were found within a 1 decimal point '
+                print(
-                      'interval of confidence: {}'.format(e))
+                    'Some differences were found within a 1 decimal point '
+                    'interval of confidence: {}'.format(e)
+                )
                 with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
                     handle.write(e.args[0])
 
@@ -203,8 +205,11 @@ class TestSuiteBundle:
 
             frequencies = exchange.get_candle_frequencies(data_frequency)
             freq = random.sample(frequencies, 1)[0]
+            rnd = random.SystemRandom()
+            # field = rnd.choice(['open', 'high', 'low', 'close', 'volume'])
+            field = rnd.choice(['volume'])
 
-            bar_count = random.randint(1, 10)
+            bar_count = random.randint(3, 6)
 
             assets = select_random_assets(
                 exchange.assets, asset_population
@@ -229,6 +234,7 @@ class TestSuiteBundle:
                 freq=freq,
                 data_frequency=data_frequency,
                 data_portal=data_portal,
+                field=field,
             )
         pass
 

tests/marketplace/test_marketplace.py

@@ -21,7 +21,7 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
 
     def test_ingest(self):
         marketplace = Marketplace()
-        ds_def = marketplace.ingest('marketcap1234')
+        ds_def = marketplace.ingest('github')
         pass
 
     def test_publish(self):