Merge remote-tracking branch 'origin/develop' into develop

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png
     :target: https://enigmampc.github.io/catalyst
     :align: center
     :alt: Enigma | Catalyst
@@ -17,9 +17,7 @@ insights regarding a particular strategy's performance. Catalyst also supports
 live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
 and Poloniex) with more being added over time. Catalyst empowers users to share 
 and curate data and build profitable, data-driven investment strategies. Please 
-visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
-refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
-further technical details.
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
 
 Catalyst builds on top of the well-established 
 `Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 

catalyst/__main__.py

@@ -6,6 +6,7 @@ import click
 import sys
 import logbook
 import pandas as pd
+from catalyst.marketplace.marketplace import Marketplace
 from six import text_type
 
 from catalyst.data import bundles as bundles_module
@@ -579,7 +580,8 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout)
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
+               sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,
@@ -633,7 +635,8 @@ def clean_exchange(ctx, exchange_name, data_frequency):
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout)
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name),
+               sys.stdout)
     exchange_bundle.clean(
         data_frequency=data_frequency,
     )
@@ -761,5 +764,129 @@ def bundles():
             click.echo("%s %s" % (bundle, timestamp), sys.stdout)
 
 
+@main.group()
+@click.pass_context
+def marketplace(ctx):
+    pass
+
+
+@marketplace.command()
+@click.pass_context
+def ls(ctx):
+    click.echo('Listing of available data sources on the marketplace:',
+               sys.stdout)
+    marketplace = Marketplace()
+    marketplace.list()
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def subscribe(ctx, dataset):
+    if dataset is None:
+        ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
+    marketplace = Marketplace()
+    marketplace.subscribe(dataset)
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.option(
+    '-f',
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}),
+    default='daily',
+    show_default=True,
+    help='The data frequency of the desired OHLCV bars.',
+)
+@click.option(
+    '-s',
+    '--start',
+    default=None,
+    type=Date(tz='utc', as_timestamp=True),
+    help='The start date of the data range. (default: one year from end date)',
+)
+@click.option(
+    '-e',
+    '--end',
+    default=None,
+    type=Date(tz='utc', as_timestamp=True),
+    help='The end date of the data range. (default: today)',
+)
+@click.pass_context
+def ingest(ctx, dataset, data_frequency, start, end):
+    if dataset is None:
+        ctx.fail("must specify a dataset to clean with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
+    click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
+    marketplace = Marketplace()
+    marketplace.ingest(dataset, data_frequency, start, end)
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def clean(ctx, dataset):
+    if dataset is None:
+        ctx.fail("must specify a dataset to ingest with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
+    click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
+    marketplace = Marketplace()
+    marketplace.clean(dataset)
+    click.echo('Done', sys.stdout)
+
+
+@marketplace.command()
+@click.pass_context
+def register(ctx):
+    marketplace = Marketplace()
+    marketplace.register()
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the Marketplace dataset to publish data for.',
+)
+@click.option(
+    '--datadir',
+    default=None,
+    help='The folder that contains the CSV data files to publish.',
+)
+@click.option(
+    '--watch/--no-watch',
+    is_flag=True,
+    default=False,
+    help='Whether to watch the datadir for live data.',
+)
+@click.pass_context
+def publish(ctx, dataset, datadir, watch):
+    marketplace = Marketplace()
+    if dataset is None:
+        ctx.fail("must specify a dataset to publish data for "
+                 " with '--dataset'\n")
+    if datadir is None:
+        ctx.fail("must specify a datadir where to find the files to publish "
+                 " with '--datadir'\n")
+    marketplace.publish(dataset, datadir, watch)
+
+
 if __name__ == '__main__':
     main()

catalyst/algorithm.py

@@ -939,7 +939,7 @@ class TradingAlgorithm(object):
             The field to query. The options have the following meanings:
               arena : str
                   The arena from the simulation parameters. This will normally
-                  be ``'backtest'`` but some systems may use this distinguish
+                  be ``backtest`` but some systems may use this distinguish
                   live trading from backtesting.
               data_frequency : {'daily', 'minute'}
                   data_frequency tells the algorithm if it is running with
@@ -954,7 +954,7 @@ class TradingAlgorithm(object):
                   The platform that the code is running on. By default this
                   will be the string 'catalyst'. This can allow algorithms to
                   know if they are running on the Quantopian platform instead.
-              * : dict[str -> any]
+              \* : dict[str -> any]
                   Returns all of the fields in a dictionary.
 
         Returns
@@ -1032,7 +1032,7 @@ class TradingAlgorithm(object):
             argument is the name of the column in the preprocessed dataframe
             containing the symbols. This will be used along with the date
             information to map the sids in the asset finder.
-        **kwargs
+        \*\*kwargs
             Forwarded to :func:`pandas.read_csv`.
 
         Returns
@@ -1156,7 +1156,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        **kwargs
+        \*\*kwargs
             The names and values to record.
 
         Notes
@@ -1273,7 +1273,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        *args : iterable[str]
+        \*args : iterable[str]
             The ticker symbols to lookup.
 
         Returns

catalyst/api.pyi

@@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None):
     :func:`catalyst.api.pipeline_output`
     """
 
+
 def batch_market_order(share_counts):
     """Place a batch market order for multiple assets.
 
@@ -48,6 +49,7 @@ def batch_market_order(share_counts):
         Index of ids for newly-created orders.
     """
 
+
 def cancel_order(order_param):
     """Cancel an open order.
 
@@ -57,7 +59,9 @@ def cancel_order(order_param):
         The order_id or order object to cancel.
     """
 
-def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
+
+def continuous_future(root_symbol_str, offset=0, roll='volume',
+                      adjustment='mul'):
     """Create a specifier for a continuous contract.
 
     Parameters
@@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'
         The continuous future specifier.
     """
 
-def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
+
+def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
+              date_format=None, timezone='UTC', symbol=None, mask=True,
+              symbol_column=None, special_params_checker=None, **kwargs):
     """Fetch a csv from a remote url and register the data so that it is
     queryable from the ``data`` object.
 
@@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma
         A requests source that will pull data from the url specified.
     """
 
+
 def future_symbol(symbol):
     """Lookup a futures contract with a given symbol.
 
@@ -144,6 +152,7 @@ def future_symbol(symbol):
         Raised when no contract named 'symbol' is found.
     """
 
+
 def get_datetime(tz=None):
     """
 Returns the current simulation datetime.
@@ -159,6 +168,7 @@ dt : datetime
     The current simulation datetime converted to ``tz``.
     """
 
+
 def get_environment(field='platform'):
     """Query the execution environment.
 
@@ -198,6 +208,7 @@ def get_environment(field='platform'):
         Raised when ``field`` is not a valid option.
     """
 
+
 def get_order(order_id):
     """Lookup an order based on the order id returned from one of the
     order functions.
@@ -213,10 +224,12 @@ def get_order(order_id):
         The order object.
     """
 
+
 def history(bar_count, frequency, field, ffill=True):
     """DEPRECATED: use ``data.history`` instead.
     """
 
+
 def order(asset, amount, limit_price=None, stop_price=None, style=None):
     """Place an order.
 
@@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
-def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
+
+def order_percent(asset, percent, limit_price=None, stop_price=None,
+                  style=None):
     """Place an order in the specified asset corresponding to the given
     percent of the current portfolio value.
 
@@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None)
     :func:`catalyst.api.order_value`
     """
 
+
 def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     """Place an order to adjust a position to a target number of shares. If
     the position doesn't already exist, this is equivalent to placing a new
@@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_target_value`
     """
 
-def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
+
+def order_target_percent(asset, target, limit_price=None, stop_price=None,
+                         style=None):
     """Place an order to adjust a position to a target percent of the
     current portfolio value. If the position doesn't already exist, this is
     equivalent to placing a new order. If the position does exist, this is
@@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style
     :func:`catalyst.api.order_target_value`
     """
 
-def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
+
+def order_target_value(asset, target, limit_price=None, stop_price=None,
+                       style=None):
     """Place an order to adjust a position to a target value. If
     the position doesn't already exist, this is equivalent to placing a new
     order. If the position does exist, this is equivalent to placing an
@@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N
     :func:`catalyst.api.order_target_percent`
     """
 
+
 def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     """Place an order by desired value rather than desired number of
     shares.
@@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
+
 def pipeline_output(name):
     """Get the results of the pipeline that was attached with the name:
     ``name``.
@@ -514,6 +536,7 @@ def pipeline_output(name):
     :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
     """
 
+
 def record(*args, **kwargs):
     """Track and record values each day.
 
@@ -529,7 +552,9 @@ def record(*args, **kwargs):
     :func:`~catalyst.run_algorithm`.
     """
 
-def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
+
+def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
+                      calendar=None):
     """Schedules a function to be called according to some timed rules.
 
     Parameters
@@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale
     :class:`catalyst.api.time_rules`
     """
 
+
 def set_asset_restrictions(restrictions, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'):
     catalyst.finance.asset_restrictions.Restrictions
     """
 
+
 def set_benchmark(benchmark):
     """Set the benchmark asset.
 
@@ -576,6 +603,7 @@ def set_benchmark(benchmark):
     automatically reinvested.
     """
 
+
 def set_cancel_policy(cancel_policy):
     """Sets the order cancellation policy for the simulation.
 
@@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy):
     :class:`catalyst.api.NeverCancel`
     """
 
+
 def set_commission(commission):
     """Sets the commission model for the simulation.
 
@@ -605,6 +634,7 @@ def set_commission(commission):
     :class:`catalyst.finance.commission.PerDollar`
     """
 
+
 def set_do_not_order_list(restricted_list, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
         The assets that cannot be ordered.
     """
 
+
 def set_long_only(on_error='fail'):
     """Set a rule specifying that this algorithm cannot take short
     positions.
     """
 
+
 def set_max_leverage(max_leverage):
     """Set a limit on the maximum leverage of the algorithm.
 
@@ -629,6 +661,7 @@ def set_max_leverage(max_leverage):
         be no maximum.
     """
 
+
 def set_max_order_count(max_count, on_error='fail'):
     """Set a limit on the number of orders that can be placed in a single
     day.
@@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'):
         The maximum number of orders that can be placed on any single day.
     """
 
-def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
+
+def set_max_order_size(asset=None, max_shares=None, max_notional=None,
+                       on_error='fail'):
     """Set a limit on the number of shares and/or dollar value of any single
     order placed for sid.  Limits are treated as absolute values and are
     enforced at the time that the algo attempts to place an order for sid.
@@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error=
         The maximum value that can be ordered at one time.
     """
 
-def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
+
+def set_max_position_size(asset=None, max_shares=None, max_notional=None,
+                          on_error='fail'):
     """Set a limit on the number of shares and/or dollar value held for the
     given sid. Limits are treated as absolute values and are enforced at
     the time that the algo attempts to place an order for sid. This means
@@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err
         The maximum value to hold for an asset.
     """
 
+
 def set_slippage(slippage):
     """Set the slippage model for the simulation.
 
@@ -694,6 +732,7 @@ def set_slippage(slippage):
     :class:`catalyst.finance.slippage.SlippageModel`
     """
 
+
 def set_symbol_lookup_date(dt):
     """Set the date for which symbols will be resolved to their assets
     (symbols may map to different firms or underlying assets at
@@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt):
         The new symbol lookup date.
     """
 
+
 def sid(sid):
     """Lookup an Asset by its unique asset identifier.
 
@@ -724,6 +764,7 @@ def sid(sid):
         When a requested ``sid`` does not map to any asset.
     """
 
+
 def symbol(symbol_str):
     """Lookup an Equity by its ticker symbol.
 
@@ -748,6 +789,7 @@ def symbol(symbol_str):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
+
 def symbols(*args):
     """Lookup multuple Equities as a list.
 
@@ -773,3 +815,18 @@ def symbols(*args):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
+
+def get_dataset(ds_name, start=None, end=None):
+    """
+    Lookup a data source from the marketplace
+
+    Parameters
+    ----------
+    ds_name: str
+    start: pd.Timestamp
+    end: pd.Timestamp
+
+    Returns
+    -------
+
+    """

catalyst/constants.py

@@ -15,4 +15,32 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
 DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
 DATE_FORMAT = '%Y-%m-%d'
 
+try:
+    ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+except Exception as e:
+    print('unable to get catalyst path: {}'.format(e))
+
 AUTO_INGEST = False
+
+AUTH_SERVER = 'https://data.enigma.co'
+
+# TODO: switch to mainnet
+ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
+
+# TODO: move to MASTER branch on github
+MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
+                       'catalyst/develop/catalyst/marketplace/' \
+                       'contract_marketplace_address.txt'
+
+MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
+                           'catalyst/develop/catalyst/marketplace/' \
+                           'contract_marketplace_abi.json'
+
+# TODO: switch to mainnet
+ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
+                  'develop/catalyst/marketplace/' \
+                  'contract_enigma_address.txt'
+
+ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
+                      'catalyst/develop/catalyst/marketplace/' \
+                      'contract_enigma_abi.json'

catalyst/examples/buy_low_sell_high.py

@@ -60,7 +60,7 @@ def _handle_data(context, data):
         rsi=rsi,
     )
 
-    orders = get_open_orders(context.asset)
+    orders = context.blotter.open_orders
     if orders:
         log.info('skipping bar until all open orders execute')
         return
@@ -146,11 +146,11 @@ if __name__ == '__main__':
     live = True
     if live:
         run_algorithm(
-            capital_base=0.001,
+            capital_base=1000,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
-            exchange_name='binance',
+            exchange_name='bittrex',
             live=True,
             algo_namespace=algo_namespace,
             base_currency='btc',

catalyst/examples/dual_moving_average.py

@@ -20,8 +20,8 @@ def initialize(context):
 
 def handle_data(context, data):
     # define the windows for the moving averages
-    short_window = 50
-    long_window = 200
+    short_window = 2
+    long_window = 3
 
     # Skip as many bars as long_window to properly compute the average
     context.i += 1
@@ -32,16 +32,18 @@ def handle_data(context, data):
     # moving average with the appropriate parameters. We choose to use
     # minute bars for this simulation -> freq="1m"
     # Returns a pandas dataframe.
-    short_mavg = data.history(context.asset,
+    short_data = data.history(context.asset,
                               'price',
                               bar_count=short_window,
-                              frequency="1m",
-                              ).mean()
-    long_mavg = data.history(context.asset,
+                              frequency="1T",
+                              )
+    short_mavg = short_data.mean()
+    long_data = data.history(context.asset,
                              'price',
                              bar_count=long_window,
-                             frequency="1m",
-                             ).mean()
+                             frequency="1T",
+                             )
+    long_mavg = long_data.mean()
 
     # Let's keep the price of our asset in a more handy variable
     price = data.current(context.asset, 'price')
@@ -82,7 +84,6 @@ def handle_data(context, data):
 
 
 def analyze(context, perf):
-
     # Get the base_currency that was passed as a parameter to the simulation
     exchange = list(context.exchanges.values())[0]
     base_currency = exchange.base_currency.upper()
@@ -93,7 +94,7 @@ def analyze(context, perf):
     ax1.legend_.remove()
     ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
     start, end = ax1.get_ylim()
-    ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     # Second chart: Plot asset price, moving averages and buys/sells
     ax2 = plt.subplot(412, sharex=ax1)
@@ -104,9 +105,9 @@ def analyze(context, perf):
     ax2.set_ylabel('{asset}\n({base})'.format(
         asset=context.asset.symbol,
         base=base_currency
-        ))
+    ))
     start, end = ax2.get_ylim()
-    ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     transaction_df = extract_transactions(perf)
     if not transaction_df.empty:
@@ -136,28 +137,40 @@ def analyze(context, perf):
     ax3.legend_.remove()
     ax3.set_ylabel('Percent Change')
     start, end = ax3.get_ylim()
-    ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     # Fourth chart: Plot our cash
     ax4 = plt.subplot(414, sharex=ax1)
     perf.cash.plot(ax=ax4)
     ax4.set_ylabel('Cash\n({})'.format(base_currency))
     start, end = ax4.get_ylim()
-    ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+    ax4.yaxis.set_ticks(np.arange(0, end, end / 5))
 
     plt.show()
 
 
 if __name__ == '__main__':
     run_algorithm(
-            capital_base=1000,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='bitfinex',
-            algo_namespace=NAMESPACE,
-            base_currency='usd',
-            start=pd.to_datetime('2017-9-22', utc=True),
-            end=pd.to_datetime('2017-9-23', utc=True),
-        )
+        capital_base=1000,
+        data_frequency='minute',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bitfinex',
+        algo_namespace=NAMESPACE,
+        base_currency='usd',
+        simulate_orders=True,
+        live=True,
+    )
+    # run_algorithm(
+    #         capital_base=1000,
+    #         data_frequency='minute',
+    #         initialize=initialize,
+    #         handle_data=handle_data,
+    #         analyze=analyze,
+    #         exchange_name='bitfinex',
+    #         algo_namespace=NAMESPACE,
+    #         base_currency='usd',
+    #         start=pd.to_datetime('2017-9-22', utc=True),
+    #         end=pd.to_datetime('2017-9-23', utc=True),
+    #     )

catalyst/examples/marketplace/github-research.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, get_dataset
+
+START = '2017-01-01'
+END = '2017-12-31'
+
+
+def initialize(context):
+    pass
+
+
+def handle_data(context, data):
+    context.github = get_dataset('github')
+    context.github.sort_index(level=0, inplace=True)
+
+    context.zec = data.history(symbol('zec_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+    context.xmr = data.history(symbol('xmr_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+
+
+def analyze(context=None, results=None):
+    ax1 = plt.subplot(211)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'ZEC']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax1, color='blue')
+    ax1.legend(loc=2)
+    ax1.set_title('Zcash')
+    ax2 = ax1.twinx()
+    context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
+    ax2.legend(loc=1)
+
+    ax3 = plt.subplot(212)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'XMR']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax3, color='blue')
+    ax3.legend(loc=2)
+    ax3.set_title('Monero')
+    ax4 = ax3.twinx()
+    context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
+    ax4.legend(loc=1)
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='algo-github',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime(END, utc=True),
+            end=pd.to_datetime(END, utc=True),
+        )

catalyst/examples/marketplace/mean_reversion_by_marketcap.py

@@ -0,0 +1,237 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_dataset
+from catalyst.exchange.utils.stats_utils import set_print_settings, \
+    get_pretty_stats
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    df = get_dataset('testmarketcap2')  # type: pd.DataFrame
+
+    # Picking a specific date in our DataFrame
+    first_dt = df.index.get_level_values(0)[0]
+    # Since we use a MultiIndex with date / symbol, picking a date will
+    # result in a new DataFrame for the selected date with a single
+    # symbol index
+    df = df.xs(first_dt, level=0)
+    # Keep only the top coins by market cap
+    df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
+
+    set_print_settings()
+
+    df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
+    print('the marketplace data:\n{}'.format(df))
+
+    # Pick the 5 assets with the lowest market cap for trading
+    quote_currency = 'eth'
+    exchange = context.exchanges[next(iter(context.exchanges))]
+    symbols = [a.symbol for a in exchange.assets
+               if a.start_date < context.datetime]
+    context.assets = []
+    for currency, price in df['market_cap_usd'].iteritems():
+        if len(context.assets) >= 5:
+            break
+
+        s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
+        if s in symbols:
+            context.assets.append(symbol(s))
+
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = dict()
+        context.current_day = today
+
+    # Preparing dictionaries for asset-level data points
+    volumes = dict()
+    rsis = dict()
+    price_values = dict()
+    cash = context.portfolio.cash
+
+    for asset in context.assets:
+        # We're computing the volume-weighted-average-price of the security
+        # defined above, in the context.assets variable.  For this example,
+        #  we're using three bars on the 15 min bars.
+
+        # The frequency attribute determine the bar size. We use this
+        # convention for the frequency alias:
+        # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+        prices = data.history(
+            asset,
+            fields='close',
+            bar_count=50,
+            frequency=context.CANDLE_SIZE
+        )
+
+        # Ta-lib calculates various technical indicator based on price and
+        # volume arrays.
+
+        # In this example, we are comp
+        rsi = talib.RSI(prices.values, timeperiod=14)
+
+        # We need a variable for the current price of the security to compare
+        #  to the average. Since we are requesting two fields, data.current()
+        # returns a DataFrame with
+        current = data.current(asset, fields=['close', 'volume'])
+        price = current['close']
+
+        # If base_price is not set, we use the current value. This is the
+        # price at the first bar which we reference to calculate price_change.
+        # if asset not in context.base_price:
+        #     context.base_price[asset] = price
+        #
+        # base_price = context.base_price[asset]
+        # price_change = (price - base_price) / base_price
+
+        # Tracking the relevant data
+        volumes[asset] = current['volume']
+        rsis[asset] = rsi[-1]
+        price_values[asset] = price
+        # price_changes[asset] = price_change
+
+        # We are trying to avoid over-trading by limiting our trades to
+        # one per day.
+        if asset in context.traded_today:
+            continue
+
+        # Exit if we cannot trade
+        if not data.can_trade(asset):
+            continue
+
+        # Another powerful built-in feature of the Catalyst backtester is the
+        # portfolio object.  The portfolio object tracks your positions, cash,
+        # cost basis of specific holdings, and more.  In this line, we
+        # calculate how long or short our position is at this minute.
+        pos_amount = context.portfolio.positions[asset].amount
+
+        if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+            log.info(
+                '{}: buying - price: {}, rsi: {}'.format(
+                    data.current_dt, price, rsi[-1]
+                )
+            )
+            # Set a style for limit orders,
+            limit_price = price * 1.005
+            target = 1.0 / len(context.assets)
+            order_target_percent(
+                asset, target, limit_price=limit_price
+            )
+            context.traded_today[asset] = True
+
+        elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+            log.info(
+                '{}: selling - price: {}, rsi: {}'.format(
+                    data.current_dt, price, rsi[-1]
+                )
+            )
+            limit_price = price * 0.995
+            order_target_percent(
+                asset, 0, limit_price=limit_price
+            )
+            context.traded_today[asset] = True
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        current_price=price_values,
+        volume=volumes,
+        rsi=rsis,
+        cash=cash,
+    )
+
+
+def analyze(context=None, perf=None):
+    stats = get_pretty_stats(perf)
+    print('the algo stats:\n{}'.format(stats))
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=100,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-10-15', utc=True),
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/examples/mean_reversion_simple.py

@@ -37,8 +37,8 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 40
-    context.RSI_OVERBOUGHT = 50
+    context.RSI_OVERSOLD = 60
+    context.RSI_OVERBOUGHT = 70
     context.CANDLE_SIZE = '15T'
 
     context.start_time = time.time()

catalyst/examples/portfolio_optimization.py

@@ -146,4 +146,5 @@ if __name__ == '__main__':
                             start=start,
                             end=end,
                             exchange_name='poloniex',
-                            capital_base=100000, )
+                            capital_base=100000,
+                            base_currency='usdt', )

catalyst/examples/simple_loop.py

@@ -26,7 +26,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='30T'
+        frequency='2H'
     )
     last_traded = prices.index[-1]
     log.info('last candle date: {}'.format(last_traded))
@@ -114,7 +114,7 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
-    mode = 'backtest'
+    mode = 'live'
 
     if mode == 'backtest':
         run_algorithm(

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -43,7 +43,8 @@ SUPPORTED_EXCHANGES = dict(
 
 
 class CCXT(Exchange):
-    def __init__(self, exchange_name, key, secret, base_currency):
+    def __init__(self, exchange_name, key,
+                 secret, password, base_currency):
         log.debug(
             'finding {} in CCXT exchanges:\n{}'.format(
                 exchange_name, ccxt.exchanges
@@ -60,6 +61,7 @@ class CCXT(Exchange):
             self.api = exchange_attr({
                 'apiKey': key,
                 'secret': secret,
+                'password': password,
             })
             self.api.enableRateLimit = True
 
@@ -425,26 +427,19 @@ class CCXT(Exchange):
                 'Please provide either start_dt or end_dt, not both.'
             )
 
-        elif end_dt is not None:
-            # Make sure that end_dt really wants data in the past
-            # if it's close to now, we skip the 'since' parameters to
-            # lower the probability of error
-            bars_to_now = pd.date_range(
-                end_dt, pd.Timestamp.utcnow(), freq=freq
-            )
-            # See: https://github.com/ccxt/ccxt/issues/1360
-            if len(bars_to_now) > 1 or self.name in ['poloniex']:
-                dt_range = get_periods_range(
-                    end_dt=end_dt,
-                    periods=bar_count,
-                    freq=freq,
-                )
-                start_dt = dt_range[0]
+        if start_dt is None:
+            if end_dt is None:
+                end_dt = pd.Timestamp.utcnow()
 
-        since = None
-        if start_dt is not None:
-            delta = start_dt - get_epoch()
-            since = int(delta.total_seconds()) * 1000
+            dt_range = get_periods_range(
+                end_dt=end_dt,
+                periods=bar_count,
+                freq=freq,
+            )
+            start_dt = dt_range[0]
+
+        delta = start_dt - get_epoch()
+        since = int(delta.total_seconds()) * 1000
 
         candles = dict()
         for index, asset in enumerate(assets):
@@ -877,6 +872,7 @@ class CCXT(Exchange):
         order.commission = exc_order.commission
         order.filled = exc_order.amount
 
+        transactions = []
         if exc_order.status == ORDER_STATUS.FILLED:
             if order.amount > exc_order.amount:
                 log.warn(
@@ -898,7 +894,9 @@ class CCXT(Exchange):
                 order_id=order.id,
                 commission=order.commission,
             )
-        return [transaction]
+            transactions.append(transaction)
+
+        return transactions
 
     def process_order(self, order):
         # TODO: move to parent class after tracking features in the parent

catalyst/exchange/exchange.py

@@ -502,7 +502,7 @@ class Exchange:
 
         """
         freq, candle_size, unit, data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
         # The get_history method supports multiple asset
         candles = self.get_candles(
@@ -523,8 +523,9 @@ class Exchange:
                 field=field,
             )
 
+            delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
             # Checking to make sure that the dates match
-            delta = get_delta(candle_size, data_frequency)
+            delta = get_delta(delta_candle_size, data_frequency)
             adj_end_dt = end_dt - delta
             last_traded = asset_series.index[-1]
 
@@ -655,16 +656,21 @@ class Exchange:
 
         return df
 
-    def _check_low_balance(self, currency, balances, amount):
+    def _check_low_balance(self, currency, balances, amount, open_orders=None):
         free = balances[currency]['free'] if currency in balances else 0.0
 
+        if open_orders:
+            # TODO: make sure that this works
+            free += sum([order.amount for order in open_orders])
+
         if free < amount:
             return free, True
 
         else:
             return free, False
 
-    def sync_positions(self, positions, cash=None, check_balances=False):
+    def sync_positions(self, positions, open_orders=None, cash=None,
+                       check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
@@ -693,7 +699,7 @@ class Exchange:
                     balances=balances,
                     amount=cash,
                 )
-                if is_lower:
+                if is_lower and not open_orders:
                     raise NotEnoughCashError(
                         currency=self.base_currency,
                         exchange=self.name,

catalyst/exchange/exchange_algorithm.py

@@ -16,7 +16,7 @@ import signal
 import sys
 from datetime import timedelta
 from os import listdir
-from os.path import isfile, join
+from os.path import isfile, join, exists
 
 import catalyst.protocol as zp
 import logbook
@@ -36,13 +36,16 @@ from catalyst.exchange.utils.exchange_utils import (
     get_algo_folder,
     get_algo_df,
     save_algo_df,
+    clear_frame_stats_directory,
+    remove_old_files,
     group_assets_by_exchange, )
-from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
-    stats_to_algo_folder
+from catalyst.exchange.utils.stats_utils import \
+    get_pretty_stats, stats_to_s3, stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
 from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
+from catalyst.marketplace.marketplace import Marketplace
 from catalyst.utils.api_support import api_method
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
@@ -66,8 +69,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         self.current_day = None
 
-        if self.simulate_orders is None \
-            and self.sim_params.arena == 'backtest':
+        if self.simulate_orders is None and \
+                self.sim_params.arena == 'backtest':
             self.simulate_orders = True
 
         # Operations with retry features
@@ -92,6 +95,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             attempts=self.attempts,
         )
 
+        self._marketplace = None
+
     @staticmethod
     def __convert_order_params_for_blotter(limit_price, stop_price, style):
         """
@@ -115,7 +120,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             # be in-line with CXXT and many exchanges. We'll consider
             # adding more order types in the future.
             if not isinstance(style, ExchangeLimitOrder) or \
-                not isinstance(style, MarketOrder):
+                    not isinstance(style, MarketOrder):
                 raise OrderTypeNotSupported(
                     order_type=style.__class__.__name__
                 )
@@ -167,6 +172,15 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         """
         return round_nearest(amount, asset.min_trade_size)
 
+    @api_method
+    def get_dataset(self, data_source_name, start=None, end=None):
+        if self._marketplace is None:
+            self._marketplace = Marketplace()
+
+        return self._marketplace.get_dataset(
+            data_source_name, start, end,
+        )
+
     @api_method
     @preprocess(symbol_str=ensure_upper_case)
     def symbol(self, symbol_str, exchange_name=None):
@@ -356,6 +370,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self._clock = None
         self.frame_stats = list()
 
+        # erase the frame_stats folder to avoid overloading the disk
+        error = clear_frame_stats_directory(self.algo_namespace)
+        if error:
+            log.warning(error)
+
         self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
 
         self.custom_signals_stats = \
@@ -379,9 +398,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             log.warn("Can't initialize signal handler inside another thread."
                      "Exit should be handled by the user.")
 
-        log.info('initialized trading algorithm in live mode')
-
     def interrupt_algorithm(self):
+        """
+
+        when algorithm comes to an end this function is called.
+        extracts the stats and calls analyze.
+        after finishing, it exits the run.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
         self.is_running = False
 
         if self._analyze is None:
@@ -391,21 +421,31 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             log.info('Exiting the algorithm. Calling `analyze()` '
                      'before exiting the algorithm.')
 
+            # add the last day stats which is not saved in the directory
+            current_stats = pd.DataFrame(self.frame_stats)
+            current_stats.set_index('period_close', drop=False, inplace=True)
+
+            # get the location of the directory
             algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_performance')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
+            folder = join(algo_folder, 'frame_stats')
 
-            daily_perf_list = []
-            for item in files:
-                filename = join(folder, item)
+            if exists(folder):
+                files = [f for f in listdir(folder) if isfile(join(folder, f))]
 
-                with open(filename, 'rb') as handle:
-                    perf_period = pickle.load(handle)
-                    perf_period_dict = perf_period.to_dict()
-                    daily_perf_list.append(perf_period_dict)
+                period_stats_list = []
+                for item in files:
+                    filename = join(folder, item)
 
-            stats = pd.DataFrame(daily_perf_list)
-            stats.set_index('period_close', drop=False, inplace=True)
+                    with open(filename, 'rb') as handle:
+                        perf_period = pickle.load(handle)
+                        period_stats_list.extend(perf_period)
+
+                stats = pd.DataFrame(period_stats_list)
+                stats.set_index('period_close', drop=False, inplace=True)
+
+                stats = pd.concat([stats, current_stats])
+            else:
+                stats = current_stats
 
             self.analyze(stats)
 
@@ -591,23 +631,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             if base_currency is None:
                 base_currency = exchange.base_currency
 
-            # Don't check the cash if there are open orders. This could
-            # results in false positives.
             orders = []
             for asset in self.blotter.open_orders:
                 asset_orders = self.blotter.open_orders[asset]
                 if asset_orders:
                     orders += asset_orders
 
-            required_cash = self.portfolio.cash if not orders else None
-            cash, positions_value = exchange.sync_positions(
-                positions=exchange_positions,
-                check_balances=check_balances,
-                cash=required_cash,
-            )
-            total_cash += cash
-            total_positions_value += positions_value
-
             # Applying modifications to the original positions
             for position in exchange_positions:
                 tracker.update_position(
@@ -617,6 +646,16 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                     last_sale_price=position.last_sale_price,
                 )
 
+            required_cash = self.portfolio.cash if not orders else None
+            cash, positions_value = exchange.sync_positions(
+                positions=exchange_positions,
+                open_orders=orders,
+                check_balances=check_balances,
+                cash=required_cash,
+            )
+            total_cash += cash
+            total_positions_value += positions_value
+
         if not check_balances:
             total_cash = self.portfolio.cash
 
@@ -699,6 +738,37 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.algo_namespace, 'exposure_stats', self.exposure_stats
         )
 
+    def nullify_frame_stats(self, now):
+        """
+
+        Save all period_stats to local directory
+        erase old files from the folder and nullify
+        self.frame_stats
+
+        Parameters
+        ----------
+        now: Timestamp
+
+        Returns
+        -------
+
+        """
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=now.floor('1D').strftime('%Y-%m-%d'),
+            obj=self.frame_stats,
+            rel_path='frame_stats'
+        )
+        error = remove_old_files(
+            algo_name=self.algo_namespace,
+            today=now,
+            rel_path='frame_stats'
+        )
+        if error:
+            log.warning(error)
+
+        self.frame_stats = list()
+
     def handle_data(self, data):
         """
         Wrapper around the handle_data method of each algo.
@@ -718,7 +788,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # Resetting the frame stats every day to minimize memory footprint
         today = data.current_dt.floor('1D')
         if self.current_day is not None and today > self.current_day:
-            self.frame_stats = list()
+            self.nullify_frame_stats(now=data.current_dt)
 
         self.performance_needs_update = False
         orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
@@ -798,6 +868,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
 
+        # creating and saving the pnl_stats into the local
+        # directory
         self.add_pnl_stats(frame_stats)
         if self.recorded_vars:
             self.add_custom_signals_stats(frame_stats)
@@ -862,6 +934,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         raise NotImplementedError()
 
     def _get_open_orders(self, asset=None):
+        if self.simulate_orders:
+            raise ValueError(
+                'The get_open_orders() method only works in live mode. '
+                'The purpose is to list open orders on the exchange '
+                'regardless who placed them. To list the open orders of '
+                'this algo, use `context.blotter.open_orders`.'
+            )
         if asset:
             exchange = self.exchanges[asset.exchange]
             return exchange.get_open_orders(asset)
@@ -895,6 +974,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             If an asset is passed then this will return a list of the open
             orders for this asset.
         """
+        # TODO: should this be a shortcut to the open orders in the blotter?
         return retry(
             action=self._get_open_orders,
             attempts=self.attempts['get_open_orders_attempts'],

catalyst/exchange/exchange_blotter.py

@@ -238,9 +238,12 @@ class ExchangeBlotter(Blotter):
                 else:
                     delta = pd.Timestamp.utcnow() - order.dt
                     log.info(
-                        'order {order_id} still open after {delta}'.format(
+                        '{exchange} order {order_id} for {symbol} still open '
+                        'after {delta}'.format(
+                            exchange=exchange.name,
                             order_id=order.id,
-                            delta=delta
+                            delta=delta,
+                            symbol=order.asset.symbol,
                         )
                     )
 

catalyst/exchange/exchange_bundle.py

@@ -458,7 +458,7 @@ class ExchangeBundle:
                 last_entry = None
 
         if start is None or \
-                (earliest_trade is not None and earliest_trade > start):
+            (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
         if last_entry is not None and (end is None or end > last_entry):
@@ -600,14 +600,14 @@ class ExchangeBundle:
         if show_breakdown:
             for asset in chunks:
                 with maybe_show_progress(
-                        chunks[asset],
-                        show_progress,
-                        label='Ingesting {frequency} price data for '
-                              '{symbol} on {exchange}'.format(
-                            exchange=self.exchange_name,
-                            frequency=data_frequency,
-                            symbol=asset.symbol
-                        )) as it:
+                    chunks[asset],
+                    show_progress,
+                    label='Ingesting {frequency} price data for '
+                          '{symbol} on {exchange}'.format(
+                        exchange=self.exchange_name,
+                        frequency=data_frequency,
+                        symbol=asset.symbol
+                    )) as it:
                     for chunk in it:
                         problems += self.ingest_ctable(
                             asset=chunk['asset'],
@@ -625,13 +625,13 @@ class ExchangeBundle:
                 key=lambda chunk: pd.to_datetime(chunk['period'])
             )
             with maybe_show_progress(
-                    all_chunks,
-                    show_progress,
-                    label='Ingesting {frequency} price data on '
-                          '{exchange}'.format(
-                        exchange=self.exchange_name,
-                        frequency=data_frequency,
-                    )) as it:
+                all_chunks,
+                show_progress,
+                label='Ingesting {frequency} price data on '
+                      '{exchange}'.format(
+                    exchange=self.exchange_name,
+                    frequency=data_frequency,
+                )) as it:
                 for chunk in it:
                     problems += self.ingest_ctable(
                         asset=chunk['asset'],
@@ -830,7 +830,6 @@ class ExchangeBundle:
                                            field,
                                            data_frequency,
                                            algo_end_dt=None,
-                                           trailing_bar_count=None,
                                            force_auto_ingest=False
                                            ):
         """
@@ -858,7 +857,6 @@ class ExchangeBundle:
                     bar_count=bar_count,
                     field=field,
                     data_frequency=data_frequency,
-                    trailing_bar_count=trailing_bar_count,
                 )
                 return pd.DataFrame(series)
 
@@ -887,7 +885,6 @@ class ExchangeBundle:
                     field=field,
                     data_frequency=data_frequency,
                     reset_reader=True,
-                    trailing_bar_count=trailing_bar_count,
                 )
                 return series
 
@@ -898,7 +895,6 @@ class ExchangeBundle:
                 bar_count=bar_count,
                 field=field,
                 data_frequency=data_frequency,
-                trailing_bar_count=trailing_bar_count,
             )
             return pd.DataFrame(series)
 
@@ -962,12 +958,7 @@ class ExchangeBundle:
                                   bar_count,
                                   field,
                                   data_frequency,
-                                  trailing_bar_count=None,
                                   reset_reader=False):
-        if trailing_bar_count:
-            delta = get_delta(trailing_bar_count, data_frequency)
-            end_dt += delta
-
         start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
         start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency

catalyst/exchange/exchange_data_portal.py

@@ -9,8 +9,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     PricingDataNotLoadedError)
-from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
-from catalyst.exchange.utils.datetime_utils import get_frequency
+from catalyst.exchange.utils.exchange_utils import resample_history_df, \
+    group_assets_by_exchange
+from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
 from logbook import Logger
 from redo import retry
 
@@ -298,7 +299,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
-        trailing_bar_count = candle_size - 1
 
         if data_frequency == 'minute' and adj_data_frequency == 'daily':
             end_dt = end_dt.floor('1D')
@@ -310,10 +310,10 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
-            trailing_bar_count=trailing_bar_count,
         )
 
-        df = resample_history_df(pd.DataFrame(series), freq, field)
+        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
+        df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
         return df
 
     def get_exchange_spot_value(self,

catalyst/exchange/utils/datetime_utils.py

@@ -92,7 +92,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
         adj_periods = periods * unit_periods
 
         # TODO: standardize time aliases to avoid any mapping
-        unit = 'd' if unit == 'D' else 'm'
+        unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
         delta = pd.Timedelta(adj_periods, unit)
 
         if start_dt is not None:
@@ -248,7 +248,7 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_frequency(freq, data_frequency=None):
+def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
     """
     Get the frequency parameters.
 
@@ -302,17 +302,18 @@ def get_frequency(freq, data_frequency=None):
     elif unit.lower() == 'm' or unit == 'T':
         unit = 'T'
         alias = '{}T'.format(candle_size)
+        data_frequency = 'minute'
 
-        if data_frequency == 'daily':
+    elif unit.lower() == 'h':
+        if 'H' in supported_freqs:
+            unit = 'H'
+            alias = '{}H'.format(candle_size)
+
+        else:
+            candle_size = candle_size * 60
+            alias = '{}T'.format(candle_size)
             data_frequency = 'minute'
 
-    # elif unit.lower() == 'h':
-    #     candle_size = candle_size * 60
-    #
-    #     alias = '{}T'.format(candle_size)
-    #     if data_frequency == 'daily':
-    #         data_frequency = 'minute'
-
     else:
         raise InvalidHistoryFrequencyAlias(freq=freq)
 

catalyst/exchange/utils/exchange_utils.py

@@ -126,11 +126,11 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     filename = get_exchange_symbols_filename(exchange_name, is_local)
 
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-                pd.Timestamp('now', tz='UTC') - last_modified_time(
-                filename)).days > 1):
+            pd.Timestamp('now', tz='UTC') - last_modified_time(
+            filename)).days > 1):
         try:
             download_exchange_symbols(exchange_name, environ)
-        except Exception as e:
+        except Exception:
             pass
 
     if os.path.isfile(filename):
@@ -273,6 +273,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     key: str
     environ:
     rel_path: str
+    how: str
 
     Returns
     -------
@@ -316,6 +317,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
     obj: Object
     environ:
     rel_path: str
+    how: str
 
     """
     folder = get_algo_folder(algo_name, environ)
@@ -392,6 +394,67 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
         df.to_csv(handle, encoding='UTF_8')
 
 
+def clear_frame_stats_directory(algo_name):
+    """
+    remove the outdated directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+
+    Returns
+    -------
+    error: str
+
+    """
+    error = None
+    algo_folder = get_algo_folder(algo_name)
+    folder = os.path.join(algo_folder, 'frame_stats')
+    if os.path.exists(folder):
+        try:
+            shutil.rmtree(folder)
+        except OSError:
+            error = 'unable to remove {}, the analyze ' \
+                    'data will be inconsistent'.format(folder)
+    return error
+
+
+def remove_old_files(algo_name, today, rel_path):
+    """
+    remove old files from a directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+    today: Timestamp
+    rel_path: str
+
+    Returns
+    -------
+    error: str
+
+    """
+    error = None
+    algo_folder = get_algo_folder(algo_name)
+    folder = os.path.join(algo_folder, rel_path)
+
+    # run on all files in the folder
+    for f in os.listdir(folder):
+        creation_unix = os.path.getctime(f)
+        creation_time = pd.to_datetime(creation_unix, unit='s', )
+
+        # if the file is older than 30 days erase it
+        if today - pd.DateOffset(30) > creation_time:
+            try:
+                os.unlink(f)
+            except OSError:
+                error = 'unable to erase files in {}'.format(folder)
+
+    return error
+
+
 def get_exchange_minute_writer_root(exchange_name, environ=None):
     """
     The minute writer folder for the exchange.
@@ -512,7 +575,7 @@ def get_common_assets(exchanges):
     return assets
 
 
-def resample_history_df(df, freq, field):
+def resample_history_df(df, freq, field, start_dt=None):
     """
     Resample the OHCLV DataFrame using the specified frequency.
 
@@ -540,7 +603,16 @@ def resample_history_df(df, freq, field):
     else:
         raise ValueError('Invalid field.')
 
-    resampled_df = df.resample(freq).agg(agg)
+    resampled_df = df.resample(
+        freq, closed='left', label='left'
+    ).agg(agg)  # type: pd.DataFrame
+
+    # Because the samples are closed left, we get one more candle at
+    # the beginning then the requested number for bars. Removing this
+    # candle to avoid confusion.
+    if start_dt and not resampled_df.empty:
+        resampled_df = resampled_df[resampled_df.index >= start_dt]
+
     return resampled_df
 
 
@@ -566,8 +638,9 @@ def mixin_market_params(exchange_name, params, market):
         params['maker'] = 0.001
         params['taker'] = 0.002
 
-    elif 'maker' in market and 'taker' in market \
-            and market['maker'] is not None and market['taker'] is not None:
+    elif 'maker' in market and 'taker' in market and \
+            market['maker'] is not None and market['taker'] is not None:
+
         params['maker'] = market['maker']
         params['taker'] = market['taker']
 

catalyst/exchange/utils/factory.py

@@ -33,6 +33,8 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
         exchange_name=exchange_name,
         key=exchange_auth['key'],
         secret=exchange_auth['secret'],
+        password=exchange_auth['password'] if 'password'
+                                              in exchange_auth.keys() else '',
         base_currency=base_currency,
     )
     exchange_cache[key] = exchange

catalyst/marketplace/contract_enigma_abi.json

@@ -0,0 +1,302 @@
+[
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "name",
+      "outputs": [
+        {
+          "name": "",
+          "type": "string"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "approve",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "totalSupply",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_from",
+          "type": "address"
+        },
+        {
+          "name": "_to",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "transferFrom",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "INITIAL_SUPPLY",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "decimals",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint8"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_subtractedValue",
+          "type": "uint256"
+        }
+      ],
+      "name": "decreaseApproval",
+      "outputs": [
+        {
+          "name": "success",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [],
+      "name": "getAfterApproveTest",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [
+        {
+          "name": "_owner",
+          "type": "address"
+        }
+      ],
+      "name": "balanceOf",
+      "outputs": [
+        {
+          "name": "balance",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "symbol",
+      "outputs": [
+        {
+          "name": "",
+          "type": "string"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_to",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "transfer",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_addedValue",
+          "type": "uint256"
+        }
+      ],
+      "name": "increaseApproval",
+      "outputs": [
+        {
+          "name": "success",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [
+        {
+          "name": "_owner",
+          "type": "address"
+        },
+        {
+          "name": "_spender",
+          "type": "address"
+        }
+      ],
+      "name": "allowance",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "inputs": [
+        {
+          "name": "testValue",
+          "type": "address"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "constructor"
+    },
+    {
+      "anonymous": false,
+      "inputs": [
+        {
+          "indexed": true,
+          "name": "owner",
+          "type": "address"
+        },
+        {
+          "indexed": true,
+          "name": "spender",
+          "type": "address"
+        },
+        {
+          "indexed": false,
+          "name": "value",
+          "type": "uint256"
+        }
+      ],
+      "name": "Approval",
+      "type": "event"
+    },
+    {
+      "anonymous": false,
+      "inputs": [
+        {
+          "indexed": true,
+          "name": "from",
+          "type": "address"
+        },
+        {
+          "indexed": true,
+          "name": "to",
+          "type": "address"
+        },
+        {
+          "indexed": false,
+          "name": "value",
+          "type": "uint256"
+        }
+      ],
+      "name": "Transfer",
+      "type": "event"
+    }
+  ]

catalyst/marketplace/contract_enigma_address.txt

@@ -0,0 +1 @@
+0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10

catalyst/marketplace/contract_marketplace_address.txt

@@ -0,0 +1 @@
+0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4

catalyst/marketplace/marketplace.py

@@ -0,0 +1,705 @@
+from __future__ import print_function
+
+import glob
+import json
+import os
+import re
+import shutil
+import sys
+import time
+
+import bcolz
+import logbook
+import pandas as pd
+import requests
+from requests_toolbelt import MultipartDecoder
+from requests_toolbelt.multipart.decoder import \
+    NonMultipartContentTypeException
+
+from catalyst.constants import (
+    LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
+    MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
+from catalyst.exchange.utils.stats_utils import set_print_settings
+from catalyst.marketplace.marketplace_errors import (
+    MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
+    MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
+    MarketplaceNoCSVFiles)
+from catalyst.marketplace.utils.auth_utils import get_key_secret, \
+    get_signed_headers
+from catalyst.marketplace.utils.bundle_utils import merge_bundles
+from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
+    to_grains
+from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
+    get_data_source_folder, get_marketplace_folder, \
+    get_user_pubaddr, get_temp_bundles_folder, extract_bundle
+
+if sys.version_info.major < 3:
+    import urllib
+else:
+    import urllib.request as urllib
+
+log = logbook.Logger('Marketplace', level=LOG_LEVEL)
+
+
+class Marketplace:
+    def __init__(self):
+        global Web3
+        from web3 import Web3, HTTPProvider
+
+        self.addresses = get_user_pubaddr()
+
+        if self.addresses[0]['pubAddr'] == '':
+            raise MarketplacePubAddressEmpty(
+                filename=os.path.join(
+                    get_marketplace_folder(), 'addresses.json')
+            )
+        self.default_account = self.addresses[0]['pubAddr']
+
+        self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
+
+        contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
+
+        self.mkt_contract_address = Web3.toChecksumAddress(
+            contract_url.readline().strip())
+
+        abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
+        abi = json.load(abi_url)
+
+        self.mkt_contract = self.web3.eth.contract(
+            self.mkt_contract_address,
+            abi=abi,
+        )
+
+        contract_url = urllib.urlopen(ENIGMA_CONTRACT)
+
+        self.eng_contract_address = Web3.toChecksumAddress(
+            contract_url.readline().strip())
+
+        abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
+        abi = json.load(abi_url)
+
+        self.eng_contract = self.web3.eth.contract(
+            self.eng_contract_address,
+            abi=abi,
+        )
+
+    # def get_data_sources_map(self):
+    #     return [
+    #         dict(
+    #             name='Marketcap',
+    #             desc='The marketcap value in USD.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily'],
+    #         ),
+    #         dict(
+    #             name='GitHub',
+    #             desc='The rate of development activity on GitHub.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily', 'hour'],
+    #         ),
+    #         dict(
+    #             name='Influencers',
+    #             desc='Tweets & related sentiments by selected influencers.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily', 'hour', 'minute'],
+    #         ),
+    #     ]
+
+    def to_text(self, hex):
+        return Web3.toText(hex).rstrip('\0')
+
+    def choose_pubaddr(self):
+        if len(self.addresses) == 1:
+            address = self.addresses[0]['pubAddr']
+            address_i = 0
+            print('Using {} for this transaction.'.format(address))
+        else:
+            while True:
+                for i in range(0, len(self.addresses)):
+                    print('{}\t{}\t{}'.format(
+                        i,
+                        self.addresses[i]['pubAddr'],
+                        self.addresses[i]['desc'])
+                    )
+                address_i = int(input('Choose your address associated with '
+                                      'this transaction: [default: 0] ') or 0)
+                if not (0 <= address_i < len(self.addresses)):
+                    print('Please choose a number between 0 and {}\n'.format(
+                        len(self.addresses) - 1))
+                else:
+                    address = Web3.toChecksumAddress(
+                        self.addresses[address_i]['pubAddr'])
+                    break
+
+        return address, address_i
+
+    def sign_transaction(self, from_address, tx):
+
+        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
+              'enter the following parameters:\n\n'
+              'From Address:\t\t{_from}\n'
+              '\n\tClick the "Generate Information" button\n\n'
+              'To Address:\t\t{to}\n'
+              'Value / Amount to Send:\t{value}\n'
+              'Gas Limit:\t\t{gas}\n'
+              'Gas Price:\t\t[Accept the default value]\n'
+              'Nonce:\t\t\t{nonce}\n'
+              'Data:\t\t\t{data}\n'.format(
+            _from=from_address,
+            to=tx['to'],
+            value=tx['value'],
+            gas=tx['gas'],
+            nonce=tx['nonce'],
+            data=tx['data'], )
+        )
+
+        signed_tx = input('Copy and Paste the "Signed Transaction" '
+                          'field here:\n')
+
+        if signed_tx.startswith('0x'):
+            signed_tx = signed_tx[2:]
+
+        return signed_tx
+
+    def check_transaction(self, tx_hash):
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
+                tx_hash)
+        else:
+            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
+
+        print('\nYou can check the outcome of your transaction here:\n'
+              '{}\n\n'.format(etherscan))
+
+    def list(self):
+
+        data_sources = self.mkt_contract.functions.getAllProviders().call()
+
+        data = []
+        for index, data_source in enumerate(data_sources):
+            if index > 0:
+                if 'test' not in Web3.toText(data_source).lower():
+                    data.append(
+                        dict(
+                            dataset=self.to_text(data_source)
+                        )
+                    )
+
+        df = pd.DataFrame(data)
+        set_print_settings()
+        if df.empty:
+            print('There are no datasets available yet.')
+        else:
+            print(df)
+
+    def subscribe(self, dataset):
+
+        dataset = dataset.lower()
+
+        address = self.choose_pubaddr()[0]
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset)
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(dataset))
+            return
+
+        grains = provider_info[1]
+        price = from_grains(grains)
+
+        subscribed = self.mkt_contract.functions.checkAddressSubscription(
+            address, Web3.toHex(dataset)
+        ).call()
+
+        if subscribed[5]:
+            print(
+                '\nYou are already subscribed to the "{}" dataset.\n'
+                'Your subscription started on {} UTC, and is valid until '
+                '{} UTC.'.format(
+                    dataset,
+                    pd.to_datetime(subscribed[3], unit='s', utc=True),
+                    pd.to_datetime(subscribed[4], unit='s', utc=True)
+                )
+            )
+            return
+
+        print('\nThe price for a monthly subscription to this dataset is'
+              ' {} ENG'.format(price))
+
+        print(
+            'Checking that the ENG balance in {} is greater than {} '
+            'ENG... '.format(address, price), end=''
+        )
+
+        wallet_address = address[2:]
+        balance = self.web3.eth.call({
+            'from': address,
+            'to': self.eng_contract_address,
+            'data': '0x70a08231000000000000000000000000{}'.format(
+                wallet_address
+            )
+        })
+
+        try:
+            balance = Web3.toInt(balance)  # web3 >= 4.0.0b7
+        except TypeError:
+            balance = Web3.toInt(hexstr=balance)  # web3 <= 4.0.0b6
+
+        if balance > grains:
+            print('OK.')
+        else:
+            print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
+                  'than the price of the dataset that you are trying to\n'
+                  'buy: {} ENG. Get enough ENG to cover the costs of the '
+                  'monthly\nsubscription for what you are trying to buy, '
+                  'and try again.'.format(
+                address, from_grains(balance), price))
+            return
+
+        while True:
+            agree_pay = input('Please confirm that you agree to pay {} ENG '
+                              'for a monthly subscription to the dataset "{}" '
+                              'starting today. [default: Y] '.format(
+                price, dataset)) or 'y'
+            if agree_pay.lower() not in ('y', 'n'):
+                print("Please answer Y or N.")
+            else:
+                if agree_pay.lower() == 'y':
+                    break
+                else:
+                    return
+
+        print('Ready to subscribe to dataset {}.\n'.format(dataset))
+        print('In order to execute the subscription, you will need to sign '
+              'two different transactions:\n'
+              '1. First transaction is to authorize the Marketplace contract '
+              'to spend {} ENG on your behalf.\n'
+              '2. Second transaction is the actual subscription for the '
+              'desired dataset'.format(price))
+
+        tx = self.eng_contract.functions.approve(
+            self.mkt_contract_address,
+            grains,
+        ).buildTransaction(
+            {'nonce': self.web3.eth.getTransactionCount(address)}
+        )
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
+        try:
+            tx_hash = '0x{}'.format(
+                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
+            )
+            print(
+                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
+            )
+
+        except Exception as e:
+            print('Unable to subscribe to data source: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the first transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nFirst transaction successful!\n'
+              'Now processing second transaction.')
+
+        tx = self.mkt_contract.functions.subscribe(
+            Web3.toHex(dataset),
+        ).buildTransaction(
+            {'nonce': self.web3.eth.getTransactionCount(address)})
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
+
+        try:
+            tx_hash = '0x{}'.format(bin_hex(
+                self.web3.eth.sendRawTransaction(signed_tx)))
+            print('\nThis is the TxHash for this transaction: '
+                  '{}'.format(tx_hash))
+
+        except Exception as e:
+            print('Unable to subscribe to data source: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the second transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nSecond transaction successful!\n'
+              'You have successfully subscribed to dataset {} with'
+              'address {}.\n'
+              'You can now ingest this dataset anytime during the '
+              'next month by running the following command:\n'
+              'catalyst marketplace ingest --dataset={}'.format(
+            dataset, address, dataset))
+
+    def process_temp_bundle(self, ds_name, path):
+        """
+        Merge the temp bundle into the main bundle for the specified
+        data source.
+
+        Parameters
+        ----------
+        ds_name
+        path
+
+        Returns
+        -------
+
+        """
+        tmp_bundle = extract_bundle(path)
+        bundle_folder = get_data_source_folder(ds_name)
+        if os.listdir(bundle_folder):
+            zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
+            ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
+            merge_bundles(zsource, ztarget)
+
+        else:
+            os.rename(tmp_bundle, bundle_folder)
+
+        pass
+
+    def ingest(self, ds_name, start=None, end=None, force_download=False):
+
+        # ds_name = ds_name.lower()
+
+        # TODO: catch error conditions
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(ds_name)
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(ds_name))
+            return
+
+        address, address_i = self.choose_pubaddr()
+        fns = self.mkt_contract.functions
+        check_sub = fns.checkAddressSubscription(
+            address, Web3.toHex(ds_name)
+        ).call()
+
+        if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
+            print('You are not subscribed to dataset "{}" with address {}. '
+                  'Plese subscribe first.'.format(ds_name, address))
+            return
+
+        if not check_sub[5]:
+            print('Your subscription to dataset "{}" expired on {} UTC.'
+                  'Please renew your subscription by running:\n'
+                  'catalyst marketplace subscribe --dataset={}'.format(
+                ds_name,
+                pd.to_datetime(check_sub[4], unit='s', utc=True),
+                ds_name)
+            )
+
+        if 'key' in self.addresses[address_i]:
+            key = self.addresses[address_i]['key']
+            secret = self.addresses[address_i]['secret']
+        else:
+            key, secret = get_key_secret(address)
+
+        headers = get_signed_headers(ds_name, key, secret)
+        log.debug('Starting download of dataset for ingestion...')
+        r = requests.post(
+            '{}/marketplace/ingest'.format(AUTH_SERVER),
+            headers=headers,
+            stream=True,
+        )
+        if r.status_code == 200:
+            target_path = get_temp_bundles_folder()
+            try:
+                decoder = MultipartDecoder.from_response(r)
+                for part in decoder.parts:
+                    h = part.headers[b'Content-Disposition'].decode('utf-8')
+                    # Extracting the filename from the header
+                    name = re.search(r'filename="(.*)"', h).group(1)
+
+                    filename = os.path.join(target_path, name)
+                    with open(filename, 'wb') as f:
+                        # for chunk in part.content.iter_content(
+                        #         chunk_size=1024):
+                        #     if chunk: # filter out keep-alive new chunks
+                        #         f.write(chunk)
+                        f.write(part.content)
+
+                    self.process_temp_bundle(ds_name, filename)
+
+            except NonMultipartContentTypeException:
+                response = r.json()
+                raise MarketplaceHTTPRequest(
+                    request='ingest dataset',
+                    error=response,
+                )
+        else:
+            raise MarketplaceHTTPRequest(
+                request='ingest dataset',
+                error=r.status_code,
+            )
+
+        log.info('{} ingested successfully'.format(ds_name))
+
+    def get_dataset(self, ds_name, start=None, end=None):
+        ds_name = ds_name.lower()
+
+        # TODO: filter ctable by start and end date
+        bundle_folder = get_data_source_folder(ds_name)
+        z = bcolz.ctable(rootdir=bundle_folder, mode='r')
+
+        df = z.todataframe()  # type: pd.DataFrame
+        df.set_index(['date', 'symbol'], drop=True, inplace=True)
+
+        # TODO: implement the filter more carefully
+        # if start and end is None:
+        #     df = df.xs(start, level=0)
+
+        return df
+
+    def clean(self, data_source_name, data_frequency=None):
+        data_source_name = data_source_name.lower()
+
+        if data_frequency is None:
+            folder = get_data_source_folder(data_source_name)
+
+        else:
+            folder = get_bundle_folder(data_source_name, data_frequency)
+
+        shutil.rmtree(folder)
+        pass
+
+    def create_metadata(self, key, secret, ds_name, data_frequency, desc,
+                        has_history=True, has_live=True):
+        """
+
+        Returns
+        -------
+
+        """
+        headers = get_signed_headers(ds_name, key, secret)
+        r = requests.post(
+            '{}/marketplace/register'.format(AUTH_SERVER),
+            json=dict(
+                ds_name=ds_name,
+                desc=desc,
+                data_frequency=data_frequency,
+                has_history=has_history,
+                has_live=has_live,
+            ),
+            headers=headers,
+        )
+
+        if r.status_code != 200:
+            raise MarketplaceHTTPRequest(
+                request='register', error=r.status_code
+            )
+
+        if 'error' in r.json():
+            raise MarketplaceHTTPRequest(
+                request='upload file', error=r.json()['error']
+            )
+
+    def register(self):
+        while True:
+            desc = input('Enter the name of the dataset to register: ')
+            dataset = desc.lower()
+            provider_info = self.mkt_contract.functions.getDataProviderInfo(
+                Web3.toHex(dataset)
+            ).call()
+
+            if provider_info[4]:
+                print('There is already a dataset registered under '
+                      'the name "{}". Please choose a different '
+                      'name.'.format(dataset))
+            else:
+                break
+
+        price = int(
+            input(
+                'Enter the price for a monthly subscription to '
+                'this dataset in ENG: '
+            )
+        )
+        while True:
+            freq = input('Enter the data frequency [daily, hourly, minute]: ')
+            if freq.lower() not in ('daily', 'hourly', 'minute'):
+                print('Not a valid frequency.')
+            else:
+                break
+
+        while True:
+            reg_pub = input(
+                'Does it include historical data? [default: Y]: '
+            ) or 'y'
+            if reg_pub.lower() not in ('y', 'n'):
+                print('Please answer Y or N.')
+            else:
+                if reg_pub.lower() == 'y':
+                    has_history = True
+                else:
+                    has_history = False
+                break
+
+        while True:
+            reg_pub = input(
+                'Doest it include live data? [default: Y]: '
+            ) or 'y'
+            if reg_pub.lower() not in ('y', 'n'):
+                print('Please answer Y or N.')
+            else:
+                if reg_pub.lower() == 'y':
+                    has_live = True
+                else:
+                    has_live = False
+                break
+
+        address, address_i = self.choose_pubaddr()
+        if 'key' in self.addresses[address_i]:
+            key = self.addresses[address_i]['key']
+            secret = self.addresses[address_i]['secret']
+        else:
+            key, secret = get_key_secret(address)
+
+        grains = to_grains(price)
+
+        tx = self.mkt_contract.functions.register(
+            Web3.toHex(dataset),
+            grains,
+            address,
+        ).buildTransaction(
+            {'nonce': self.web3.eth.getTransactionCount(address)}
+        )
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
+
+        try:
+            tx_hash = '0x{}'.format(
+                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
+            )
+            print(
+                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
+            )
+
+        except Exception as e:
+            print('Unable to subscribe to data source: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nWarming up the {} dataset'.format(dataset))
+        self.create_metadata(
+            key=key,
+            secret=secret,
+            ds_name=dataset,
+            data_frequency=freq,
+            desc=desc,
+            has_history=has_history,
+            has_live=has_live,
+        )
+        print('\n{} registered successfully'.format(dataset))
+
+    def publish(self, dataset, datadir, watch):
+        dataset = dataset.lower()
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset)
+        ).call()
+
+        if not provider_info[4]:
+            raise MarketplaceDatasetNotFound(dataset=dataset)
+
+        match = next(
+            (l for l in self.addresses if l['pubAddr'] == provider_info[0]),
+            None
+        )
+        if not match:
+            raise MarketplaceNoAddressMatch(
+                dataset=dataset,
+                address=provider_info[0])
+
+        print('Using address: {} to publish this dataset.'.format(
+            provider_info[0]))
+
+        if 'key' in match:
+            key = match['key']
+            secret = match['secret']
+        else:
+            key, secret = get_key_secret(provider_info[0])
+
+        headers = get_signed_headers(dataset, key, secret)
+        filenames = glob.glob(os.path.join(datadir, '*.csv'))
+
+        if not filenames:
+            raise MarketplaceNoCSVFiles(datadir=datadir)
+
+        files = []
+        for file in filenames:
+            files.append(('file', open(file, 'rb')))
+
+        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                          files=files,
+                          headers=headers)
+
+        if r.status_code != 200:
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.status_code)
+
+        if 'error' in r.json():
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.json()['error'])
+
+        print('Dataset {} uploaded successfully.'.format(dataset))

catalyst/marketplace/marketplace_errors.py

@@ -0,0 +1,88 @@
+import sys
+import traceback
+
+from catalyst.errors import ZiplineError
+
+
+def silent_except_hook(exctype, excvalue, exctraceback):
+    if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
+                   MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
+                   MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
+                   MarketplaceSubscriptionExpired, MarketplaceJSONError,
+                   MarketplaceWalletNotSupported, MarketplaceEmptySignature]:
+        fn = traceback.extract_tb(exctraceback)[-1][0]
+        ln = traceback.extract_tb(exctraceback)[-1][1]
+        print("Error traceback: {1} (line {2})\n"
+              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
+    else:
+        sys.__excepthook__(exctype, excvalue, exctraceback)
+
+
+sys.excepthook = silent_except_hook
+
+
+class MarketplacePubAddressEmpty(ZiplineError):
+    msg = (
+        'Please enter your public address to use in the Data Marketplace '
+        'in the following file: {filename}'
+    ).strip()
+
+
+class MarketplaceDatasetNotFound(ZiplineError):
+    msg = (
+        'The dataset "{dataset}" is not registered in the Data Marketplace.'
+    ).strip()
+
+
+class MarketplaceNoAddressMatch(ZiplineError):
+    msg = (
+        'The address registered with the dataset {dataset}: {address} '
+        'does not match any of your addresses.'
+    ).strip()
+
+
+class MarketplaceHTTPRequest(ZiplineError):
+    msg = (
+        'Request to remote server to {request} failed: {error}'
+    ).strip()
+
+
+class MarketplaceNoCSVFiles(ZiplineError):
+    msg = (
+        'No CSV files found on {datadir} to upload.'
+    )
+
+
+class MarketplaceContractDataNoMatch(ZiplineError):
+    msg = (
+        'The information found on the contract does not match the '
+        'requested data:\n{params}.'
+    )
+
+
+class MarketplaceSubscriptionExpired(ZiplineError):
+    msg = (
+        'Your subscription to dataset "{dataset}" expired on {date} '
+        'and is no longer active. You have to subscribe again running the '
+        'following command:\n'
+        'catalyst marketplace subscribe --dataset={dataset}'
+    )
+
+
+class MarketplaceWalletNotSupported(ZiplineError):
+    msg = (
+        'Wallet {wallet} is not supported.'
+    )
+
+
+class MarketplaceEmptySignature(ZiplineError):
+    msg = (
+        'Signature cannot be empty.'
+    )
+
+
+class MarketplaceJSONError(ZiplineError):
+    msg = (
+        'The configuration file {file} is malformed. Please correct '
+        'the following error:\n{error}'
+    )

catalyst/marketplace/utils/auth_utils.py

@@ -0,0 +1,131 @@
+import hashlib
+import hmac
+
+import requests
+import time
+
+from catalyst.marketplace.marketplace_errors import (
+    MarketplaceHTTPRequest, MarketplaceWalletNotSupported,
+    MarketplaceEmptySignature)
+from catalyst.marketplace.utils.path_utils import (
+    get_user_pubaddr, save_user_pubaddr)
+from catalyst.constants import AUTH_SERVER
+
+
+def get_key_secret(pubAddr, wallet='mew'):
+    """
+    Obtain a new key/secret pair from authentication server
+
+    Parameters
+    ----------
+    pubAddr: str
+    dataset: str
+
+    Returns
+    -------
+    key: str
+    secret: str
+
+    """
+    session = requests.Session()
+    response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
+                           headers={
+                            'Authorization': 'Digest username="{0}"'.format(
+                                pubAddr)})
+
+    if response.status_code != 401:
+        raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                     error='Unexpected response code: '
+                                           '{}'.format(response.status_code))
+
+    header = response.headers.get('WWW-Authenticate')
+    auth_type, auth_info = header.split(None, 1)
+    d = requests.utils.parse_dict_header(auth_info)
+
+    nonce = '0x{}'.format(d['nonce'])
+
+    if wallet == 'mew':
+        print('\nObtaining a key/secret pair to streamline all future '
+              'requests with the authentication server.\n'
+              'Visit https://www.myetherwallet.com/signmsg.html and sign the '
+              'following message:\n{}'.format(nonce))
+        signature = input('Copy and Paste the "sig" field from '
+                          'the signature here (without the double quotes, '
+                          'only the HEX value):\n')
+    else:
+        raise MarketplaceWalletNotSupported(wallet=wallet)
+
+    if signature is None:
+        raise MarketplaceEmptySignature()
+
+    signature = signature[2:]
+    r = int(signature[0:64], base=16)
+    s = int(signature[64:128], base=16)
+    v = int(signature[128:130], base=16)
+    vrs = [v, r, s]
+
+    response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
+                           headers={
+                'Authorization': 'Digest username="{0}",realm="{1}",'
+                'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",'
+                'opaque="{4}"'.format(pubAddr,
+                                      d['realm'],
+                                      d['nonce'],
+                                      ','.join(str(e) for e in vrs+[wallet]),
+                                      d['opaque'])})
+
+    if response.status_code == 200:
+
+        if 'error' in response.json():
+            raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                         error=str(response.json()['error']))
+        else:
+            addresses = get_user_pubaddr()
+
+            match = next((l for l in addresses if
+                          l['pubAddr'] == pubAddr), None)
+            match['key'] = response.json()['key']
+            match['secret'] = response.json()['secret']
+
+            addresses[addresses.index(match)] = match
+
+            save_user_pubaddr(addresses)
+            print('Key/secret pair retrieved successfully from server.')
+
+            return match['key'], match['secret']
+
+    else:
+        raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                     error=response.status_code)
+
+
+def get_signed_headers(ds_name, key, secret):
+    """
+    Return a new request header including the key / secret signature
+
+    Parameters
+    ----------
+    ds_name
+    key
+    secret
+
+    Returns
+    -------
+
+    """
+    nonce = str(int(time.time()))
+
+    signature = hmac.new(
+        secret.encode('utf-8'),
+        '{}{}'.format(ds_name, nonce).encode('utf-8'),
+        hashlib.sha512
+    ).hexdigest()
+
+    headers = {
+        'Sign': signature,
+        'Key': key,
+        'Nonce': nonce,
+        'Dataset': ds_name,
+    }
+
+    return headers

catalyst/marketplace/utils/bundle_utils.py

@@ -0,0 +1,36 @@
+import os
+import shutil
+
+import bcolz
+import pandas as pd
+
+
+def merge_bundles(zsource, ztarget):
+    """
+    Merge
+    Parameters
+    ----------
+    zsource
+    ztarget
+
+    Returns
+    -------
+
+    """
+    # TODO: find a way to do this iteratively instead of in-memory
+    df_source = zsource.todataframe()
+    df_target = ztarget.todataframe()
+
+    df = pd.concat(
+        [df_source, df_target], ignore_index=True
+    )  # type: pd.DataFrame
+    df.drop_duplicates(inplace=True)
+    df.set_index(['date', 'symbol'], drop=False, inplace=True)
+
+    dirname = os.path.basename(ztarget.rootdir)
+    bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
+    os.rename(ztarget.rootdir, bak_dir)
+
+    z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
+    shutil.rmtree(bak_dir)
+    return z

catalyst/marketplace/utils/eth_utils.py

@@ -0,0 +1,82 @@
+import binascii
+
+
+# def bytes32(string):
+#     """
+#     Convert string to bytes32 data type for smart contract
+
+#     Parameters
+#     ----------
+#     string: str
+
+#     Returns
+#     -------
+#     list
+
+#     """
+#     return binascii.hexlify(string.encode('utf-8'))
+
+
+# def b32_str(bytes32):
+#     """
+#     Convert bytes32 to string
+
+#     Parameters
+#     ----------
+#     input: bytes object
+
+#     Returns
+#     -------
+#     str
+
+#     """
+#     return binascii.unhexlify(
+#             bytes32.decode('utf-8').rstrip('\0')).decode('ascii')
+
+
+def bin_hex(binary):
+    """
+    Convert bytes32 to string
+
+    Parameters
+    ----------
+    input: bytes object
+
+    Returns
+    -------
+    str
+
+    """
+    return binascii.hexlify(binary).decode('utf-8')
+
+
+def from_grains(amount):
+    """
+    Convert from grains to cryptocurrency
+
+    Parameters
+    ----------
+    input: amount
+
+    Returns
+    -------
+    int
+
+    """
+    return amount // 10 ** 8
+
+
+def to_grains(amount):
+    """
+    Convert from cryptocurrency to grains
+
+    Parameters
+    ----------
+    input: amount
+
+    Returns
+    -------
+    int
+
+    """
+    return amount * 10 ** 8

catalyst/marketplace/utils/path_utils.py

@@ -0,0 +1,166 @@
+import os
+import json
+import tarfile
+
+from catalyst.utils.deprecate import deprecated
+from catalyst.utils.paths import data_root, ensure_directory
+from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
+
+
+def get_marketplace_folder(environ=None):
+    """
+    The root path of the marketplace folder.
+
+    Parameters
+    ----------
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    marketplace_folder = os.path.join(root, 'marketplace')
+    ensure_directory(marketplace_folder)
+
+    return marketplace_folder
+
+
+def get_data_source_folder(data_source_name, environ=None):
+    """
+    The root path of an data_source folder.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
+    ensure_directory(data_source_folder)
+
+    return data_source_folder
+
+
+@deprecated
+def get_bundle_folder(data_source_name, data_frequency, environ=None):
+    data_source_folder = get_data_source_folder(data_source_name, environ)
+
+    bundle_folder = os.path.join(data_source_folder, data_frequency)
+
+    ensure_directory(bundle_folder)
+
+    return bundle_folder
+
+
+def get_temp_bundles_folder(environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    ds_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    root = data_root(environ)
+    folder = os.path.join(root, 'marketplace', 'temp_bundles')
+    ensure_directory(folder)
+
+    return folder
+
+
+def extract_bundle(tar_filename):
+    """
+    Extract a bcolz bundle.
+
+    Parameters
+    ----------
+    ds_name
+
+    Returns
+    -------
+    str
+
+    """
+    target_path = tar_filename.replace('.tar.gz', '')
+    with tarfile.open(tar_filename, 'r') as tar:
+        tar.extractall(target_path)
+
+    return target_path
+
+
+def get_user_pubaddr(environ=None):
+    """
+    The de-serialized contend of the user's addresses.json file.
+
+    Parameters
+    ----------
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
+    marketplace_folder = get_marketplace_folder(environ)
+    filename = os.path.join(marketplace_folder, 'addresses.json')
+
+    if os.path.isfile(filename):
+        with open(filename) as data_file:
+            try:
+                data = json.load(data_file)
+            except json.decoder.JSONDecodeError as e:
+                raise MarketplaceJSONError(file=filename, error=e)
+            try:
+                d = data[0]['pubAddr']
+            except Exception as e:
+                return [data, ]
+            return data
+    else:
+        data = []
+        data.append(dict(pubAddr='', desc=''))
+        with open(filename, 'w') as f:
+            json.dump(data, f, sort_keys=False, indent=2,
+                      separators=(',', ':'))
+            return data
+
+
+def save_user_pubaddr(data, environ=None):
+    """
+    Saves the user's public addresses and their related metadata in
+    the corresponding addresses.json file.
+
+    Parameters
+    ----------
+    data: dict
+
+    Returns
+    -------
+    True
+
+    """
+    marketplace_folder = get_marketplace_folder(environ)
+    filename = os.path.join(marketplace_folder, 'addresses.json')
+
+    with open(filename, 'w') as f:
+        json.dump(data, f, sort_keys=False, indent=2,
+                  separators=(',', ':'))
+
+    return True

catalyst/support/issue_216.py

@@ -0,0 +1,376 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python2
+
+import sys
+import os
+import pandas as pd
+import signal
+# import talib
+
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import (
+    symbol,
+    record,
+    order,
+    order_target,
+    order_target_percent,
+    get_open_orders
+)
+from catalyst.finance import commission
+
+
+# from base.telegrambot import TelegramBot
+
+
+class GracefulKiller:
+    # Source: https://stackoverflow.com/a/31464349
+    def __init__(self, context):
+        self.kill_now = False
+        self.signal = 0
+        self.context = context
+        signal.signal(signal.SIGINT, self.exit_gracefully)
+
+    def exit_gracefully(self, signum, frame):
+        self.kill_now = True
+        self.signal = signum
+        if hasattr(self.context,
+                   'telegram_bot') and self.context.telegram_bot is not None:
+            self.context.telegram_bot.updater.stop()
+        sys.exit(0)
+
+    def exit(self):
+        return self.kill_now
+
+
+class SimulationParameters:
+    MODE = 'paper'
+    CAPITAL_BASE = 1000
+    """
+    Capital base used on this simulation
+    """
+
+    DATA_FREQUECY = 'minute'
+
+    EXCHANGE_NAME = 'bitfinex'
+    # EXCHANGE_NAME = 'binance'
+    """
+    Exchange used on this simulation
+    """
+
+    DATA_DIR = '/home/av/Dropbox/simulations/data'
+    ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0]
+    ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images',
+                                                 ALGO_NAMESPACE)
+    ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables',
+                                                         ALGO_NAMESPACE + '_results')
+    ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR,
+                                                              'tables',
+                                                              ALGO_NAMESPACE + '_transactions')
+    BASE_CURRENCY = 'usd'
+    # BASE_CURRENCY = 'usdt'
+
+    # SHORT PERIOD
+    START_DATE = '2017-09-07'
+    """
+    Start date used on this simulation
+    """
+    END_DATE = '2017-12-12'
+    """
+    End date used on this simulation
+    """
+
+    SKIP_FIRST_CANDLES = 0
+
+    # CANDLES_SAMPLE_RATE = 60
+    # CANDLES_SAMPLE_RATE = 30
+    CANDLES_SAMPLE_RATE = 1
+    """
+    Candle interval used on this simulation (in minutes)
+    """
+
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    # 30 minute interval ohlcv data (the standard data required for candlestick or
+    # indicators/signals)
+    # 30T means 30 minutes re-sampling of one minute data.
+    # CANDLES_FREQUENCY = '60T'
+    # CANDLES_FREQUENCY = '30T'
+    CANDLES_FREQUENCY = '1T'
+    CANDLES_BUFFER_SIZE = 48
+    COIN_PAIR = 'btc_usd'
+    # COIN_PAIR = 'btc_usdt'
+    """
+    Coin pair used on this simulation
+    """
+
+    # TRANSACTIONS
+    COMMISSION_FEE = 0.0030
+    BUY_MIN_AMOUNT = 5  # i.e: USD
+    SELL_MIN_AMOUNT = 0.001  # i.e: USD
+    BUY_SELL_PERCENTAGE = 1  # 0.50
+    BUY_PERCENTAGE = BUY_SELL_PERCENTAGE
+    SELL_PERCENTAGE = BUY_SELL_PERCENTAGE
+
+    BASE_PRICE = 'close'
+    """
+    Base price used (close / Heiken Ashi)
+    """
+
+
+log = None
+parameters = None
+
+
+def print_facts(context):
+    context.log.info("""
+Index: {}
+Date: {}
+Candle:
+O: {}
+H: {}
+L: {}
+C: {}
+V: {}
+Metrics:
+...
+Portfolio:
+Base price: {}
+Base coin (coin2/usd): {}
+Amount (coin1/btc): {}
+""".format(
+        # Facts
+        context.i,
+        context.curr_minute,
+        context.candles_open[-1],
+        context.candles_high[-1],
+        context.candles_low[-1],
+        context.candles_close[-1],
+        context.candles_volume[-1],
+        # Metrics
+        # ...
+        # Portfolio
+        context.curr_base_price,
+        context.portfolio.cash,
+        context.portfolio.positions[context.coin_pair].amount,
+    ))
+
+
+def print_facts_telegram(context):
+    price = context.curr_base_price
+    amount = context.portfolio.positions[context.coin_pair].amount
+    pnl = context.portfolio.pnl
+    capital_used = context.portfolio.capital_used
+    portfolio_value = context.portfolio.portfolio_value
+    portfolio_returns = context.portfolio.returns
+    starting_cash = context.portfolio.starting_cash
+    cash = context.portfolio.cash
+
+    msg = """
+Status...
+Price: {}
+Starting cash: {}
+Cash: {}
+Capital used: {}
+Amount: {}
+Portfolio value: {}
+Returns: {}
+PnL: {}
+    """.format(
+        price,
+        starting_cash,
+        cash,
+        capital_used,
+        amount,
+        portfolio_value,
+        portfolio_returns,
+        pnl,
+    )
+    if hasattr(context, 'telegram_bot') and context.telegram_bot is not None:
+        context.telegram_bot.msg(msg)
+
+
+def default_initialize(context):
+    # FIXME: set_benchmark
+    # set_benchmark(symbol(context.parameters.COIN_PAIR))
+
+    context.coin_pair = symbol(context.parameters.COIN_PAIR)
+    context.base_price = None
+    context.current_day = None
+    context.counter = -1
+    context.i = 0
+
+    context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE
+    context.candles_frequency = context.parameters.CANDLES_FREQUENCY
+    context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE
+    context.set_commission(
+        commission.PerShare(cost=context.parameters.COMMISSION_FEE))
+
+
+def default_handle_data(context, data):
+    context.curr_minute = data.current_dt
+    context.counter += 1
+
+    if context.candles_sample_rate == 1:
+        context.i += 1
+    elif context.counter % context.candles_sample_rate != 0:
+        context.i += 1
+        return
+
+    if context.i < context.parameters.SKIP_FIRST_CANDLES:
+        return
+
+    context.candles_open = data.history(
+        context.coin_pair,
+        'open',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_high = data.history(
+        context.coin_pair,
+        'high',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_low = data.history(
+        context.coin_pair,
+        'low',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_close = data.history(
+        context.coin_pair,
+        'price',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_volume = data.history(
+        context.coin_pair,
+        'volume',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+
+    # FIXME: Here is the error!
+    # The candles_close frame shows more or less always a value of 94, while
+    # bitcoin price is very different from that
+    print(context.candles_close)
+
+    context.base_prices = context.candles_close
+    cash = context.portfolio.cash
+    amount = context.portfolio.positions[context.coin_pair].amount
+    price = data.current(context.coin_pair, 'price')
+    order_id = None
+    context.last_base_price = context.base_prices[-2]
+    context.curr_base_price = context.base_prices[-1]
+
+    # TA calculations
+    # ...
+
+    # Sanity checks
+    # assert cash >= 0
+    if cash < 0:
+        import ipdb;
+        ipdb.set_trace()  # BREAKPOINT
+
+    print_facts(context)
+    print_facts_telegram(context)
+
+    # Order management
+    net_shares = 0
+    if context.counter == 2:
+        brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE
+        share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
+        net_shares = brute_shares - share_commission_fee
+        buy_order_id = order(context.coin_pair, net_shares)
+
+    if context.counter == 3:
+        brute_shares = amount * context.parameters.SELL_PERCENTAGE
+        share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
+        net_shares = -(brute_shares - share_commission_fee)
+        sell_order_id = order(context.coin_pair, net_shares)
+
+    # Record
+    record(
+        price=price,
+        foo='bar',
+        # volume=current['volume'],
+        # price_change=price_change,
+        # Metrics
+        cash=cash,
+        # buy=context.buy,
+        # sell=context.sell
+    )
+
+
+def default_analyze(context=None, perf=None):
+    pass
+
+
+def initialize(context):
+    global log
+    context.parameters = parameters
+    context.log = Logger(context.parameters.ALGO_NAMESPACE)
+    log = context.log
+    default_initialize(context)
+    context.killer = GracefulKiller(context)
+    context.telegram_bot = None
+
+    # TELEGRAM_TOKEN='token'
+    # context.telegram_bot = TelegramBot()
+    # context.telegram_bot.initialize(TELEGRAM_TOKEN, context)
+
+
+if __name__ == '__main__':
+    # Parameters:
+    parameters = SimulationParameters()
+    start_date = pd.to_datetime(parameters.START_DATE, utc=True)
+    end_date = pd.to_datetime(parameters.END_DATE, utc=True)
+
+    if parameters.MODE == 'backtest':
+        results = run_algorithm(
+            capital_base=parameters.CAPITAL_BASE,
+            data_frequency=parameters.DATA_FREQUECY,
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            start=start_date,
+            end=end_date,
+            live=False,
+            live_graph=False
+        )
+
+        returns_daily = results
+        results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE))
+
+        # returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1)
+
+        # FIXME: pyfolio integration
+        # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results)
+        # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01'])
+        # pyfolio.create_full_tear_sheet(*pf_data)
+
+    elif parameters.MODE == 'paper':
+        results = run_algorithm(
+            capital_base=parameters.CAPITAL_BASE,
+            data_frequency=parameters.DATA_FREQUECY,
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            live=True,
+            simulate_orders=True,
+            live_graph=False
+        )
+
+    elif parameters.MODE == 'live':
+        results = run_algorithm(
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            live=True,
+            live_graph=True
+        )

catalyst/utils/run_algo.py

@@ -55,6 +55,7 @@ class _RunAlgoError(click.ClickException, ValueError):
     ----------
     pyfunc_msg : str
         The message that will be shown when called as a python function.
+
     cmdline_msg : str
         The message that will be shown on the command line.
     """
@@ -416,7 +417,8 @@ def run_algorithm(initialize,
                   auth_aliases=None,
                   stats_output=None,
                   output=os.devnull):
-    """Run a trading algorithm.
+    """
+    Run a trading algorithm.
 
     Parameters
     ----------
@@ -458,7 +460,7 @@ def run_algorithm(initialize,
         This argument is mutually exclusive with ``data``.
     default_extension : bool, optional
         Should the default catalyst extension be loaded. This is found at
-        ``$ZIPLINE_ROOT/extension.py``
+        ``$CATALYST_ROOT/extension.py``
     extensions : iterable[str], optional
         The names of any other extensions to load. Each element may either be
         a dotted module path like ``a.b.c`` or a path to a python file ending
@@ -469,12 +471,8 @@ def run_algorithm(initialize,
     environ : mapping[str -> str], optional
         The os environment to use. Many extensions use this to get parameters.
         This defaults to ``os.environ``.
-    live: execute live trading
-    exchange_conn: The exchange connection parameters
-
-    Supported Exchanges
-    -------------------
-    bitfinex
+    live : bool, optional
+        Execute algorithm in live trading mode.
 
     Returns
     -------

docs/source/appendix.rst

@@ -4,7 +4,7 @@ API Reference
 Running a Backtest
 ~~~~~~~~~~~~~~~~~~
 
-.. autofunction:: zipline.run_algorithm(...)
+.. autofunction:: catalyst.run_algorithm(...)
 
 Algorithm API
 ~~~~~~~~~~~~~
@@ -18,341 +18,335 @@ currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance.
 Data Object
 ```````````
 
-.. autoclass:: zipline.protocol.BarData
+.. autoclass:: catalyst.protocol.BarData
    :members:
 
 Scheduling Functions
 ````````````````````
 
-.. autofunction:: zipline.api.schedule_function
+.. autofunction:: catalyst.api.schedule_function
 
-.. autoclass:: zipline.api.date_rules
+.. autoclass:: catalyst.api.date_rules
    :members:
    :undoc-members:
 
-.. autoclass:: zipline.api.time_rules
+.. autoclass:: catalyst.api.time_rules
    :members:
 
 Orders
 ``````
 
-.. autofunction:: zipline.api.order
+.. autofunction:: catalyst.api.order
 
-.. autofunction:: zipline.api.order_value
+.. autofunction:: catalyst.api.order_value
 
-.. autofunction:: zipline.api.order_percent
+.. autofunction:: catalyst.api.order_percent
 
-.. autofunction:: zipline.api.order_target
+.. autofunction:: catalyst.api.order_target
 
-.. autofunction:: zipline.api.order_target_value
+.. autofunction:: catalyst.api.order_target_value
 
-.. autofunction:: zipline.api.order_target_percent
+.. autofunction:: catalyst.api.order_target_percent
 
-.. autoclass:: zipline.finance.execution.ExecutionStyle
+.. autoclass:: catalyst.finance.execution.ExecutionStyle
    :members:
 
-.. autoclass:: zipline.finance.execution.MarketOrder
+.. autoclass:: catalyst.finance.execution.MarketOrder
 
-.. autoclass:: zipline.finance.execution.LimitOrder
+.. autoclass:: catalyst.finance.execution.LimitOrder
 
-.. autoclass:: zipline.finance.execution.StopOrder
+.. autoclass:: catalyst.finance.execution.StopOrder
 
-.. autoclass:: zipline.finance.execution.StopLimitOrder
+.. autoclass:: catalyst.finance.execution.StopLimitOrder
 
-.. autofunction:: zipline.api.get_order
+.. autofunction:: catalyst.api.get_order
 
-.. autofunction:: zipline.api.get_open_orders
+.. autofunction:: catalyst.api.get_open_orders
 
-.. autofunction:: zipline.api.cancel_order
+.. autofunction:: catalyst.api.cancel_order
 
 Order Cancellation Policies
 '''''''''''''''''''''''''''
 
-.. autofunction:: zipline.api.set_cancel_policy
+.. autofunction:: catalyst.api.set_cancel_policy
 
-.. autoclass:: zipline.finance.cancel_policy.CancelPolicy
+.. autoclass:: catalyst.finance.cancel_policy.CancelPolicy
    :members:
 
-.. autofunction:: zipline.api.EODCancel
+.. autofunction:: catalyst.api.EODCancel
 
-.. autofunction:: zipline.api.NeverCancel
+.. autofunction:: catalyst.api.NeverCancel
 
 
 Assets
 ``````
 
-.. autofunction:: zipline.api.symbol
+.. autofunction:: catalyst.api.symbol
 
-.. autofunction:: zipline.api.symbols
+.. autofunction:: catalyst.api.symbols
 
-.. autofunction:: zipline.api.future_symbol
+.. autofunction:: catalyst.api.set_symbol_lookup_date
 
-.. autofunction:: zipline.api.set_symbol_lookup_date
-
-.. autofunction:: zipline.api.sid
+.. autofunction:: catalyst.api.sid
 
 
 Trading Controls
 ````````````````
 
-Zipline provides trading controls to help ensure that the algorithm is
+zipline provides trading controls to help ensure that the algorithm is
 performing as expected. The functions help protect the algorithm from certian
 bugs that could cause undesirable behavior when trading with real money.
 
-.. autofunction:: zipline.api.set_do_not_order_list
+.. autofunction:: catalyst.api.set_do_not_order_list
 
-.. autofunction:: zipline.api.set_long_only
+.. autofunction:: catalyst.api.set_long_only
 
-.. autofunction:: zipline.api.set_max_leverage
+.. autofunction:: catalyst.api.set_max_leverage
 
-.. autofunction:: zipline.api.set_max_order_count
+.. autofunction:: catalyst.api.set_max_order_count
 
-.. autofunction:: zipline.api.set_max_order_size
+.. autofunction:: catalyst.api.set_max_order_size
 
-.. autofunction:: zipline.api.set_max_position_size
+.. autofunction:: catalyst.api.set_max_position_size
 
 
 Simulation Parameters
 `````````````````````
 
-.. autofunction:: zipline.api.set_benchmark
+.. autofunction:: catalyst.api.set_benchmark
 
 Commission Models
 '''''''''''''''''
 
-.. autofunction:: zipline.api.set_commission
+.. autofunction:: catalyst.api.set_commission
 
-.. autoclass:: zipline.finance.commission.CommissionModel
+.. autoclass:: catalyst.finance.commission.CommissionModel
    :members:
 
-.. autoclass:: zipline.finance.commission.PerShare
+.. autoclass:: catalyst.finance.commission.PerShare
 
-.. autoclass:: zipline.finance.commission.PerTrade
+.. autoclass:: catalyst.finance.commission.PerTrade
 
-.. autoclass:: zipline.finance.commission.PerDollar
+.. autoclass:: catalyst.finance.commission.PerDollar
 
 Slippage Models
 '''''''''''''''
 
-.. autofunction:: zipline.api.set_slippage
+.. autofunction:: catalyst.api.set_slippage
 
-.. autoclass:: zipline.finance.slippage.SlippageModel
+.. autoclass:: catalyst.finance.slippage.SlippageModel
    :members:
 
-.. autoclass:: zipline.finance.slippage.FixedSlippage
+.. autoclass:: catalyst.finance.slippage.FixedSlippage
 
-.. autoclass:: zipline.finance.slippage.VolumeShareSlippage
+.. autoclass:: catalyst.finance.slippage.VolumeShareSlippage
 
 Pipeline
 ````````
 
-For more information, see :ref:`pipeline-api`
+Not supported yet.
 
-.. autofunction:: zipline.api.attach_pipeline
+.. For more information, see :ref:`pipeline-api`
 
-.. autofunction:: zipline.api.pipeline_output
+.. .. autofunction:: catalyst.api.attach_pipeline
+
+.. .. autofunction:: catalyst.api.pipeline_output
 
 
 Miscellaneous
 `````````````
 
-.. autofunction:: zipline.api.record
+.. autofunction:: catalyst.api.record
 
-.. autofunction:: zipline.api.get_environment
+.. autofunction:: catalyst.api.get_environment
 
-.. autofunction:: zipline.api.fetch_csv
+.. autofunction:: catalyst.api.fetch_csv
 
 
 .. _pipeline-api:
 
-Pipeline API
-~~~~~~~~~~~~
+.. Pipeline API
+.. ~~~~~~~~~~~~
 
-.. autoclass:: zipline.pipeline.Pipeline
-   :members:
-   :member-order: groupwise
+.. .. autoclass:: zipline.pipeline.Pipeline
+..    :members:
+..    :member-order: groupwise
 
-.. autoclass:: zipline.pipeline.CustomFactor
-   :members:
-   :member-order: groupwise
+.. .. autoclass:: zipline.pipeline.CustomFactor
+..    :members:
+..    :member-order: groupwise
 
-.. autoclass:: zipline.pipeline.filters.Filter
-   :members: __and__, __or__
-   :exclude-members: dtype
+.. .. autoclass:: zipline.pipeline.filters.Filter
+..    :members: __and__, __or__
+..    :exclude-members: dtype
 
-.. autoclass:: zipline.pipeline.factors.Factor
-   :members: bottom, deciles, demean, linear_regression, pearsonr,
-             percentile_between, quantiles, quartiles, quintiles, rank,
-             spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
-             __add__, __sub__, __mul__, __div__, __mod__, __pow__, __lt__,
-             __le__, __ne__, __ge__, __gt__
-   :exclude-members: dtype
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.factors.Factor
+..    :members: bottom, deciles, demean, linear_regression, pearsonr,
+..              percentile_between, quantiles, quartiles, quintiles, rank,
+..              spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
+..              \__add__, \__sub__, \__mul__, \__div__, \__mod__, \__pow__, 
+..              \__lt__, \__le__, \__ne__, \__ge__, \__gt__
+..    :exclude-members: dtype
+..    :member-order: bysource
 
-.. autoclass:: zipline.pipeline.term.Term
-   :members:
-   :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
+.. .. autoclass:: zipline.pipeline.term.Term
+..    :members:
+..    :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
 
-.. autoclass:: zipline.pipeline.data.USEquityPricing
-   :members: open, high, low, close, volume
-   :undoc-members:
+.. .. autoclass:: zipline.pipeline.data.USEquityPricing
+..    :members: open, high, low, close, volume
+..    :undoc-members:
 
-Built-in Factors
-````````````````
+.. Built-in Factors
+.. ````````````````
 
-.. autoclass:: zipline.pipeline.factors.AverageDollarVolume
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.AverageDollarVolume
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BollingerBands
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BollingerBands
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.Latest
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.Latest
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.MaxDrawdown
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.MaxDrawdown
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.Returns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.Returns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RSI
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RSI
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.VWAP
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.VWAP
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.WeightedAverageValue
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.WeightedAverageValue
+..    :members:
 
-Pipeline Engine
-```````````````
+.. Pipeline Engine
+.. ```````````````
 
-.. autoclass:: zipline.pipeline.engine.PipelineEngine
-   :members: run_pipeline, run_chunked_pipeline
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.engine.PipelineEngine
+..    :members: run_pipeline, run_chunked_pipeline
+..    :member-order: bysource
 
-.. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
-   :members: __init__, run_pipeline, run_chunked_pipeline
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
+..    :members: __init__, run_pipeline, run_chunked_pipeline
+..    :member-order: bysource
 
-.. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
+.. .. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
 
-Data Loaders
-````````````
+.. Data Loaders
+.. ````````````
 
-.. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
-   :members: __init__, from_files, load_adjusted_array
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
+..    :members: __init__, from_files, load_adjusted_array
+..    :member-order: bysource
 
 Asset Metadata
 ~~~~~~~~~~~~~~
 
-.. autoclass:: zipline.assets.Asset
+.. autoclass:: catalyst.assets.Asset
    :members:
 
-.. autoclass:: zipline.assets.Equity
-   :members:
-
-.. autoclass:: zipline.assets.Future
-   :members:
-
-.. autoclass:: zipline.assets.AssetConvertible
+.. autoclass:: catalyst.assets.AssetConvertible
    :members:
 
 
 Trading Calendar API
 ~~~~~~~~~~~~~~~~~~~~
 
-.. autofunction:: zipline.utils.calendars.get_calendar
+.. autofunction:: catalyst.utils.calendars.get_calendar
 
-.. autoclass:: zipline.utils.calendars.TradingCalendar
+.. autoclass:: catalyst.utils.calendars.TradingCalendar
    :members:
 
-.. autofunction:: zipline.utils.calendars.register_calendar
+.. autofunction:: catalyst.utils.calendars.register_calendar
 
-.. autofunction:: zipline.utils.calendars.register_calendar_type
+.. autofunction:: catalyst.utils.calendars.register_calendar_type
 
-.. autofunction:: zipline.utils.calendars.deregister_calendar
+.. autofunction:: catalyst.utils.calendars.deregister_calendar
 
-.. autofunction:: zipline.utils.calendars.clear_calendars
+.. autofunction:: catalyst.utils.calendars.clear_calendars
 
 
 Data API
 ~~~~~~~~
 
-Writers
-```````
-.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
-   :members:
+.. Writers
+.. ```````
+.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
+..    :members:
 
-.. autoclass:: zipline.assets.AssetDBWriter
-   :members:
+.. .. autoclass:: zipline.assets.AssetDBWriter
+..    :members:
 
-Readers
-```````
-.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
-   :members:
+.. Readers
+.. ```````
+.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
+..    :members:
 
-.. autoclass:: zipline.assets.AssetFinder
-   :members:
+.. .. autoclass:: zipline.assets.AssetFinder
+..    :members:
 
-.. autoclass:: zipline.data.data_portal.DataPortal
-   :members:
+.. .. autoclass:: zipline.data.data_portal.DataPortal
+..    :members:
 
-Bundles
-```````
-.. autofunction:: zipline.data.bundles.register
+.. Bundles
+.. ```````
+.. .. autofunction:: zipline.data.bundles.register
 
-.. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
+.. .. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
 
-.. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
+.. .. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
 
-.. autofunction:: zipline.data.bundles.unregister
+.. .. autofunction:: zipline.data.bundles.unregister
 
-.. data:: zipline.data.bundles.bundles
+.. .. data:: zipline.data.bundles.bundles
 
-   The bundles that have been registered as a mapping from bundle name to bundle
-   data. This mapping is immutable and should only be updated through
-   :func:`~zipline.data.bundles.register` or
-   :func:`~zipline.data.bundles.unregister`.
+..    The bundles that have been registered as a mapping from bundle name to bundle
+..    data. This mapping is immutable and should only be updated through
+..    :func:`~zipline.data.bundles.register` or
+..    :func:`~zipline.data.bundles.unregister`.
 
-.. autofunction:: zipline.data.bundles.yahoo_equities
+.. .. autofunction:: zipline.data.bundles.yahoo_equities
 
 
 
@@ -362,16 +356,16 @@ Utilities
 Caching
 ```````
 
-.. autoclass:: zipline.utils.cache.CachedObject
+.. autoclass:: catalyst.utils.cache.CachedObject
 
-.. autoclass:: zipline.utils.cache.ExpiringCache
+.. autoclass:: catalyst.utils.cache.ExpiringCache
 
-.. autoclass:: zipline.utils.cache.dataframe_cache
+.. autoclass:: catalyst.utils.cache.dataframe_cache
 
-.. autoclass:: zipline.utils.cache.working_file
+.. autoclass:: catalyst.utils.cache.working_file
 
-.. autoclass:: zipline.utils.cache.working_dir
+.. autoclass:: catalyst.utils.cache.working_dir
 
 Command Line
 ````````````
-.. autofunction:: zipline.utils.cli.maybe_show_progress
+.. autofunction:: catalyst.utils.cli.maybe_show_progress

docs/source/beginner-tutorial.rst

@@ -168,7 +168,7 @@ We'll start with the CLI, and introduce the ``run_algorithm()`` in the last
 example of this tutorial. Some of the :doc:`example algorithms <example-algos>` 
 provide instructions on how to run them both from the CLI, and using the 
 :func:`~catalyst.run_algorithm` function. For the third method, refer to the 
-corresponding section on :doc:`Catalyst & Jupyter Notebook <jupyter>` after you 
+corresponding section on :ref:`Catalyst & Jupyter Notebook <jupyter>` after you 
 have assimilated the contents of this tutorial.
 
 Command line interface
@@ -473,6 +473,7 @@ Which we execute by running:
     </div>
 
 |
+
 There is a row for each trading day, starting on the first day of our 
 simulation Jan 1st, 2016. In the columns you can find various
 information about the state of your algorithm. The column
@@ -518,7 +519,7 @@ alongside enigma-catalyst (with the exception of the ``Conda`` install, where it
 was included by default inside the conda environment we created). If for any 
 reason you don't have it installed, you can add it by running:
 
-.. code-block:: python
+.. code-block:: bash
 
   (catalyst)$ pip install matplotlib
 
@@ -579,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in
 the ``examples`` directory:
 `dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
 
-.. code-block:: python
-
-    import numpy as np
-    import pandas as pd
-    from logbook import Logger
-    import matplotlib.pyplot as plt 
-
-    from catalyst import run_algorithm
-    from catalyst.api import (order, record, symbol, order_target_percent, 
-            get_open_orders)
-    from catalyst.exchange.utils.stats_utils import extract_transactions
-
-    NAMESPACE = 'dual_moving_average'
-    log = Logger(NAMESPACE)
-
-    def initialize(context):
-        context.i = 0
-        context.asset = symbol('ltc_usd')
-        context.base_price = None
-
-
-    def handle_data(context, data):
-        # define the windows for the moving averages 
-        short_window = 50
-        long_window = 200
-
-        # Skip as many bars as long_window to properly compute the average
-        context.i += 1
-        if context.i < long_window:
-           return
-
-        # Compute moving averages calling data.history() for each 
-        # moving average with the appropriate parameters. We choose to use
-        # minute bars for this simulation -> freq="1m"
-        # Returns a pandas dataframe.
-        short_mavg = data.history(context.asset, 'price', 
-                            bar_count=short_window, frequency="1m").mean()
-        long_mavg = data.history(context.asset, 'price',
-                            bar_count=long_window, frequency="1m").mean()
-
-        # Let's keep the price of our asset in a more handy variable
-        price = data.current(context.asset, 'price')
-
-        # If base_price is not set, we use the current value. This is the
-        # price at the first bar which we reference to calculate price_change.
-        if context.base_price is None:
-            context.base_price = price
-        price_change = (price - context.base_price) / context.base_price
-
-        # Save values for later inspection
-        record(price=price,
-               cash=context.portfolio.cash,
-               price_change=price_change,
-               short_mavg=short_mavg,
-               long_mavg=long_mavg)
-
-        # Since we are using limit orders, some orders may not execute immediately
-        # we wait until all orders are executed before considering more trades.
-        orders = get_open_orders(context.asset)
-        if len(orders) > 0:
-            return
-
-        # Exit if we cannot trade
-        if not data.can_trade(context.asset):
-            return
-
-        # We check what's our position on our portfolio and trade accordingly
-        pos_amount = context.portfolio.positions[context.asset].amount
-
-        # Trading logic
-        if short_mavg > long_mavg and pos_amount == 0:
-           # we buy 100% of our portfolio for this asset
-           order_target_percent(context.asset, 1)
-        elif short_mavg < long_mavg and pos_amount > 0:
-           # we sell all our positions for this asset
-           order_target_percent(context.asset, 0)
-
-
-    def analyze(context, perf):
-
-        # Get the base_currency that was passed as a parameter to the simulation
-        exchange = list(context.exchanges.values())[0]
-        base_currency = exchange.base_currency.upper()
-
-        # First chart: Plot portfolio value using base_currency
-        ax1 = plt.subplot(411)
-        perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-        ax1.legend_.remove()
-        ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-        start, end = ax1.get_ylim()
-        ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Second chart: Plot asset price, moving averages and buys/sells
-        ax2 = plt.subplot(412, sharex=ax1)
-        perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-        ax2.legend_.remove()
-        ax2.set_ylabel('{asset}\n({base})'.format(
-            asset = context.asset.symbol,
-            base = base_currency
-            ))
-        start, end = ax2.get_ylim()
-        ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        transaction_df = extract_transactions(perf)
-        if not transaction_df.empty:
-            buy_df = transaction_df[transaction_df['amount'] > 0]
-            sell_df = transaction_df[transaction_df['amount'] < 0]
-            ax2.scatter(
-                buy_df.index.to_pydatetime(),
-                perf.loc[buy_df.index, 'price'],
-                marker='^',
-                s=100,
-                c='green',
-                label=''
-            )
-            ax2.scatter(
-                sell_df.index.to_pydatetime(),
-                perf.loc[sell_df.index, 'price'],
-                marker='v',
-                s=100,
-                c='red',
-                label=''
-            )
-
-        # Third chart: Compare percentage change between our portfolio
-        # and the price of the asset
-        ax3 = plt.subplot(413, sharex=ax1)
-        perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-        ax3.legend_.remove()
-        ax3.set_ylabel('Percent Change')
-        start, end = ax3.get_ylim()
-        ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Fourth chart: Plot our cash
-        ax4 = plt.subplot(414, sharex=ax1)
-        perf.cash.plot(ax=ax4)
-        ax4.set_ylabel('Cash\n({})'.format(base_currency))
-        start, end = ax4.get_ylim()
-        ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-        plt.show()
-
-
-    if __name__ == '__main__':
-        run_algorithm(
-                capital_base=1000,
-                data_frequency='minute',
-                initialize=initialize,
-                handle_data=handle_data,
-                analyze=analyze,
-                exchange_name='bitfinex',
-                algo_namespace=NAMESPACE,
-                base_currency='usd',
-                start=pd.to_datetime('2017-9-22', utc=True),
-                end=pd.to_datetime('2017-9-23', utc=True),
-            )
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
+   :language: python
 
 In order to run the code above, you have to ingest the needed data first:
 
@@ -806,6 +653,7 @@ the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
 ``pandas.DataFrame``\ s, so you can simply pass the underlying
 ``ndarray`` of a ``DataFrame`` via ``.values``).
 
+.. _jupyter:
 
 Jupyter Notebook
 ~~~~~~~~~~~~~~~~
@@ -826,13 +674,13 @@ In order to use Jupyter Notebook, you first have to install it inside your
 environment. It's available as ``pip`` package, so regardless of how you 
 installed Catalyst, go inside your catalyst environemnt and run:
 
-.. code:: bash
+.. code-block:: bash
 
     (catalyst)$ pip install jupyter
 
 Once you have Jupyter Notebook installed, every time you want to use it run:
 
-.. code:: bash
+.. code-block:: bash
 
     (catalyst)$ jupyter notebook
 
@@ -846,7 +694,7 @@ Before running your algorithms inside the Jupyter Notebook, remember to ingest
 the data from the command line interface (CLI). In the example below, you would
 need to run first:
 
-.. code:: bash
+.. code-block:: bash
 
   catalyst ingest-exchange -x bitfinex -i btc_usd
 
@@ -16607,7 +16455,49 @@ NaN
 
    </div>
 
+PyCharm IDE
+~~~~~~~~~~~
 
+PyCharm is an Integrated Development Environment (IDE) used in computer 
+programming, specifically for the Python language. It streamlines the continuos
+development of Python code, and among other things includes a debugger that 
+comes in handy to see the inner workings of Catalyst, and your trading 
+algorithms.
+
+Install
+^^^^^^^
+Install PyCharm from their `Website <https://www.jetbrains.com/pycharm/download/>`__.
+There is a free and open-source **Community** version.
+
+Setup
+^^^^^
+
+1. When creating a new project in PyCharm, right under you specify the Location,
+   click on **Project Interpreter** to display a drop down menu
+
+2. Select **Existing interpreter**, click the gear box right next to it and 
+   select 'add local'. Depending on your installation, select either 
+   "*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to
+   navigate to your catalyst env and select the Python binary file: 
+   ``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows 
+   installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). 
+   Select OK. You may want to click on *Make available to all projects* for your
+   future reference. Click OK again, and create your new environment using the
+   set up of your virtual environment.
+
+Alternatively, if you already have your project created, in Windows do:
+
+1. File -> Default Settings -> Project Interpreter. Click the gear box next to 
+   the project interpreter and select ‘add local’, and follow the steps from the
+   second step above.
+
+On MacOS:
+
+1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ -> 
+   Project Interpreter. Click the gear box next to the project interpreter 
+   and select ‘add local’, and follow the steps from the second step above.
+
+You should now be able to run your project/scripts in PyCharm.
 
 Next steps
 ~~~~~~~~~~

docs/source/conf.py

@@ -27,8 +27,8 @@ extlinks = {
 
 # -- Docstrings ---------------------------------------------------------------
 
-#extensions += ['numpydoc']
-#numpydoc_show_class_members = False
+extensions += ['numpydoc']
+numpydoc_show_class_members = False
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['.templates']
@@ -97,3 +97,6 @@ intersphinx_mapping = {
 doctest_global_setup = "import catalyst"
 
 todo_include_todos = True
+
+suppress_warnings = ['image.nonlocal_uri']
+

docs/source/development-guidelines.rst

@@ -36,25 +36,15 @@ Finally, you can build the C extensions by running:
 
    $ python setup.py build_ext --inplace
 
-.. To finish, make sure `tests`__ pass.
+Development with Docker
+-----------------------
 
-.. __ #style-guide-running-tests
+If you want to work with zipline using a `Docker`__ container, you'll need to 
+build the ``Dockerfile`` in the Zipline root directory, and then build 
+``Dockerfile-dev``. Instructions for building both containers can be found in 
+``Dockerfile`` and ``Dockerfile-dev``, respectively.
 
-.. If you get an error running nosetests after setting up a fresh virtualenv, please try running
-
-.. code-block
-
-..   # where zipline is the name of your virtualenv
-..   $ deactivate zipline
-..   $ workon zipline
-
-
-.. Development with Docker
-.. -----------------------
-
-..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
-
-.. __ https://docs.docker.com/get-started/
+__ https://docs.docker.com/get-started/
    
 Git Branching Structure
 -----------------------

docs/source/example-algos.rst

@@ -1,4 +1,5 @@
 |
+
 Example Algorithms
 ==================
 
@@ -51,35 +52,8 @@ Buy BTC Simple Algorithm
 
 Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
 
-.. code-block:: python
-
-  '''
-    Run this example, by executing the following from your terminal:
-      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
-      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    If you want to run this code using another exchange, make sure that 
-    the asset is available on that exchange. For example, if you were to run 
-    it for exchange Poloniex, you would need to edit the following line:
-
-      context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
-
-    and specify exchange poloniex as follows:
-    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
-    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    To see which assets are available on each exchange, visit:
-    https://www.enigma.co/catalyst/status
-  '''
-
-  from catalyst.api import order, record, symbol
-
-  def initialize(context):
-      context.asset = symbol('btc_usd')
-
-  def handle_data(context, data):
-      order(context.asset, 1)
-      record(btc = data.current(context.asset, 'price'))
+.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
+   :language: python
 
 This simple algorithm does not produce any output nor displays any chart.
 
@@ -89,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart.
 Buy and Hodl Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
-
 First ingest the historical pricing data needed to run this algorithm:
 
 .. code-block:: bash
@@ -118,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
 earlier date, you'll get an error), and the most recent date you can choose is 
 one day prior to the current date. 
 
+Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
 
-.. code-block:: python
-
-  #!/usr/bin/env python
-  #
-  # Copyright 2017 Enigma MPC, Inc.
-  # Copyright 2015 Quantopian, Inc.
-  #
-  # Licensed under the Apache License, Version 2.0 (the "License");
-  # you may not use this file except in compliance with the License.
-  # You may obtain a copy of the License at
-  #
-  #     http://www.apache.org/licenses/LICENSE-2.0
-  #
-  # Unless required by applicable law or agreed to in writing, software
-  # distributed under the License is distributed on an "AS IS" BASIS,
-  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  # See the License for the specific language governing permissions and
-  # limitations under the License.
-  import pandas as pd
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order_target_value, symbol, record,
-      cancel_order, get_open_orders, )
-
-
-  def initialize(context):
-      context.ASSET_NAME = 'btc_usd'
-      context.TARGET_HODL_RATIO = 0.8
-      context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
-
-      context.is_buying = True
-      context.asset = symbol(context.ASSET_NAME)
-
-      context.i = 0
-
-
-  def handle_data(context, data):
-      context.i += 1
-
-      starting_cash = context.portfolio.starting_cash
-      target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
-      reserve_value = context.RESERVE_RATIO * starting_cash
-
-      # Cancel any outstanding orders
-      orders = get_open_orders(context.asset) or []
-      for order in orders:
-          cancel_order(order)
-
-      # Stop buying after passing the reserve threshold
-      cash = context.portfolio.cash
-      if cash <= reserve_value:
-          context.is_buying = False
-
-      # Retrieve current asset price from pricing data
-      price = data.current(context.asset, 'price')
-
-      # Check if still buying and could (approximately) afford another purchase
-      if context.is_buying and cash > price:
-          print('buying')
-          # Place order to make position in asset equal to target_hodl_value
-          order_target_value(
-              context.asset,
-              target_hodl_value,
-              limit_price=price * 1.1,
-          )
-
-      record(
-          price=price,
-          volume=data.current(context.asset, 'volume'),
-          cash=cash,
-          starting_cash=context.portfolio.starting_cash,
-          leverage=context.account.leverage,
-      )
-
-
-  def analyze(context=None, results=None):
-
-      # Plot the portfolio and asset data.
-      ax1 = plt.subplot(611)
-      results[['portfolio_value']].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio Value (USD)')
-
-      ax2 = plt.subplot(612, sharex=ax1)
-      ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-      results[['price']].plot(ax=ax2)
-
-      trans = results.ix[[t != [] for t in results.transactions]]
-      buys = trans.ix[
-          [t[0]['amount'] > 0 for t in trans.transactions]
-      ]
-      ax2.scatter(
-          buys.index.to_pydatetime(),
-          results.price[buys.index],
-          marker='^',
-          s=100,
-          c='g',
-          label=''
-      )
-
-      ax3 = plt.subplot(613, sharex=ax1)
-      results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-      ax3.set_ylabel('Leverage ')
-
-      ax4 = plt.subplot(614, sharex=ax1)
-      results[['starting_cash', 'cash']].plot(ax=ax4)
-      ax4.set_ylabel('Cash (USD)')
-
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]] = results[[
-          'treasury_period_return',
-          'algorithm_period_return',
-          'benchmark_period_return',
-      ]]
-
-      ax5 = plt.subplot(615, sharex=ax1)
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]].plot(ax=ax5)
-      ax5.set_ylabel('Percent Change')
-
-      ax6 = plt.subplot(616, sharex=ax1)
-      results[['volume']].plot(ax=ax6)
-      ax6.set_ylabel('Volume (mCoins/5min)')
-
-      plt.legend(loc=3)
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-          capital_base=10000,
-          data_frequency='daily',
-          initialize=initialize,
-          handle_data=handle_data,
-          analyze=analyze,
-          exchange_name='bitfinex',
-          algo_namespace='buy_and_hodl',
-          base_currency='usd',
-          start=pd.to_datetime('2015-03-01', utc=True),
-          end=pd.to_datetime('2017-10-31', utc=True),
-      )
+.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
 
@@ -277,166 +102,13 @@ one day prior to the current date.
 Dual Moving Average Crossover
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
-
 This strategy is covered in detail in the last part of 
 `this tutorial <beginner-tutorial.html#history>`_.
 
-.. code-block:: python
+Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
 
-  import numpy as np
-  import pandas as pd
-  from logbook import Logger
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order, record, symbol, order_target_percent,
-          get_open_orders)
-  from catalyst.exchange.stats_utils import extract_transactions
-
-  NAMESPACE = 'dual_moving_average'
-  log = Logger(NAMESPACE)
-
-  def initialize(context):
-      context.i = 0
-      context.asset = symbol('ltc_usd')
-      context.base_price = None
-
-
-  def handle_data(context, data):
-      # define the windows for the moving averages
-      short_window = 50
-      long_window = 200
-
-      # Skip as many bars as long_window to properly compute the average
-      context.i += 1
-      if context.i < long_window:
-         return
-
-      # Compute moving averages calling data.history() for each
-      # moving average with the appropriate parameters. We choose to use
-      # minute bars for this simulation -> freq="1m"
-      # Returns a pandas dataframe.
-      short_mavg = data.history(context.asset, 'price',
-                          bar_count=short_window, frequency="1m").mean()
-      long_mavg = data.history(context.asset, 'price',
-                          bar_count=long_window, frequency="1m").mean()
-
-      # Let's keep the price of our asset in a more handy variable
-      price = data.current(context.asset, 'price')
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-      price_change = (price - context.base_price) / context.base_price
-
-      # Save values for later inspection
-      record(price=price,
-             cash=context.portfolio.cash,
-             price_change=price_change,
-             short_mavg=short_mavg,
-             long_mavg=long_mavg)
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.asset)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.asset):
-          return
-
-      # We check what's our position on our portfolio and trade accordingly
-      pos_amount = context.portfolio.positions[context.asset].amount
-
-      # Trading logic
-      if short_mavg > long_mavg and pos_amount == 0:
-         # we buy 100% of our portfolio for this asset
-         order_target_percent(context.asset, 1)
-      elif short_mavg < long_mavg and pos_amount > 0:
-         # we sell all our positions for this asset
-         order_target_percent(context.asset, 0)
-
-
-  def analyze(context, perf):
-
-      # Get the base_currency that was passed as a parameter to the simulation
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # First chart: Plot portfolio value using base_currency
-      ax1 = plt.subplot(411)
-      perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-      ax1.legend_.remove()
-      ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-      start, end = ax1.get_ylim()
-      ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Second chart: Plot asset price, moving averages and buys/sells
-      ax2 = plt.subplot(412, sharex=ax1)
-      perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-      ax2.legend_.remove()
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset = context.asset.symbol,
-          base = base_currency
-          ))
-      start, end = ax2.get_ylim()
-      ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index, 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index, 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      # Third chart: Compare percentage change between our portfolio
-      # and the price of the asset
-      ax3 = plt.subplot(413, sharex=ax1)
-      perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-      ax3.legend_.remove()
-      ax3.set_ylabel('Percent Change')
-      start, end = ax3.get_ylim()
-      ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Fourth chart: Plot our cash
-      ax4 = plt.subplot(414, sharex=ax1)
-      perf.cash.plot(ax=ax4)
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-      start, end = ax4.get_ylim()
-      ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-              capital_base=1000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-9-22', utc=True),
-              end=pd.to_datetime('2017-9-23', utc=True),
-          )
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
 
@@ -446,8 +118,6 @@ This strategy is covered in detail in the last part of
 Mean Reversion Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
-
 This algorithm is based on a simple momentum strategy. When the cryptoasset goes
 up quickly, we're going to buy; when it goes down quickly, we're going to sell. 
 Hopefully, we'll ride the waves.
@@ -468,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type:
 
    python mean_reversion_simple.py
 
-.. code-block:: python
+Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
 
-  import os
-  import tempfile
-  import time
-
-  import numpy as np
-  import pandas as pd
-  import talib
-  from logbook import Logger
-
-  from catalyst import run_algorithm
-  from catalyst.api import symbol, record, order_target_percent, get_open_orders
-  from catalyst.exchange.stats_utils import extract_transactions
-  # We give a name to the algorithm which Catalyst will use to persist its state.
-  # In this example, Catalyst will create the `.catalyst/data/live_algos`
-  # directory. If we stop and start the algorithm, Catalyst will resume its
-  # state using the files included in the folder.
-  from catalyst.utils.paths import ensure_directory
-
-  NAMESPACE = 'mean_reversion_simple'
-  log = Logger(NAMESPACE)
-
-
-  # To run an algorithm in Catalyst, you need two functions: initialize and
-  # handle_data.
-
-  def initialize(context):
-      # This initialize function sets any data or variables that you'll use in
-      # your algorithm.  For instance, you'll want to define the trading pair (or
-      # trading pairs) you want to backtest.  You'll also want to define any
-      # parameters or values you're going to use.
-
-      # In our example, we're looking at Neo in USD.
-      context.neo_eth = symbol('neo_usd')
-      context.base_price = None
-      context.current_day = None
-
-      context.RSI_OVERSOLD = 30
-      context.RSI_OVERBOUGHT = 80
-      context.CANDLE_SIZE = '15T'
-
-      context.start_time = time.time()
-
-
-  def handle_data(context, data):
-      # This handle_data function is where the real work is done.  Our data is
-      # minute-level tick data, and each minute is called a frame.  This function
-      # runs on each frame of the data.
-
-      # We flag the first period of each day.
-      # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-      # would only execute once. This method works with minute and daily
-      # frequencies.
-      today = data.current_dt.floor('1D')
-      if today != context.current_day:
-          context.traded_today = False
-          context.current_day = today
-
-      # We're computing the volume-weighted-average-price of the security
-      # defined above, in the context.neo_eth variable.  For this example, we're 
-      # using three bars on the 15 min bars.
-
-      # The frequency attribute determine the bar size. We use this convention
-      # for the frequency alias:
-      # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-      prices = data.history(
-          context.neo_eth,
-          fields='close',
-          bar_count=50,
-          frequency=context.CANDLE_SIZE
-      )
-
-      # Ta-lib calculates various technical indicator based on price and
-      # volume arrays.
-
-      # In this example, we are comp
-      rsi = talib.RSI(prices.values, timeperiod=14)
-
-      # We need a variable for the current price of the security to compare to
-      # the average. Since we are requesting two fields, data.current()
-      # returns a DataFrame with
-      current = data.current(context.neo_eth, fields=['close', 'volume'])
-      price = current['close']
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-
-      price_change = (price - context.base_price) / context.base_price
-      cash = context.portfolio.cash
-
-      # Now that we've collected all current data for this frame, we use
-      # the record() method to save it. This data will be available as
-      # a parameter of the analyze() function for further analysis.
-      record(
-          price=price,
-          volume=current['volume'],
-          price_change=price_change,
-          rsi=rsi[-1],
-          cash=cash
-      )
-
-      # We are trying to avoid over-trading by limiting our trades to
-      # one per day.
-      if context.traded_today:
-          return
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.neo_eth)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.neo_eth):
-          return
-
-      # Another powerful built-in feature of the Catalyst backtester is the
-      # portfolio object.  The portfolio object tracks your positions, cash,
-      # cost basis of specific holdings, and more.  In this line, we calculate
-      # how long or short our position is at this minute.   
-      pos_amount = context.portfolio.positions[context.neo_eth].amount
-
-      if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-          log.info(
-              '{}: buying - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          # Set a style for limit orders,
-          limit_price = price * 1.005
-          order_target_percent(
-              context.neo_eth, 1, limit_price=limit_price
-          )
-          context.traded_today = True
-
-      elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-          log.info(
-              '{}: selling - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          limit_price = price * 0.995
-          order_target_percent(
-              context.neo_eth, 0, limit_price=limit_price
-          )
-          context.traded_today = True
-
-
-  def analyze(context=None, perf=None):
-      end = time.time()
-      log.info('elapsed time: {}'.format(end - context.start_time))
-
-      import matplotlib.pyplot as plt
-      # The base currency of the algo exchange
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # Plot the portfolio value over time.
-      ax1 = plt.subplot(611)
-      perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
-
-      # Plot the price increase or decrease over time.
-      ax2 = plt.subplot(612, sharex=ax1)
-      perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset=context.neo_eth.symbol, base=base_currency
-      ))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      ax4 = plt.subplot(613, sharex=ax1)
-      perf.loc[:, 'cash'].plot(
-          ax=ax4, label='Base Currency ({})'.format(base_currency)
-      )
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-
-      perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-      ax5 = plt.subplot(614, sharex=ax1)
-      perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-      ax5.set_ylabel('Percent\nChange')
-
-      ax6 = plt.subplot(615, sharex=ax1)
-      perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-      ax6.set_ylabel('RSI')
-      ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
-      ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
-
-      if not transaction_df.empty:
-          ax6.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'rsi'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax6.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'rsi'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-      plt.legend(loc=3)
-      start, end = ax6.get_ylim()
-      ax6.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-      pass
-
-
-  if __name__ == '__main__':
-      # The execution mode: backtest or live
-      MODE = 'backtest'
-
-      if MODE == 'backtest':
-          folder = os.path.join(
-              tempfile.gettempdir(), 'catalyst', NAMESPACE
-          )
-          ensure_directory(folder)
-
-          timestr = time.strftime('%Y%m%d-%H%M%S')
-          out = os.path.join(folder, '{}.p'.format(timestr))
-          # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
-          run_algorithm(
-              capital_base=10000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-10-01', utc=True),
-              end=pd.to_datetime('2017-11-10', utc=True),
-              output=out
-          )
-          log.info('saved perf stats: {}'.format(out))
-
-      elif MODE == 'live':
-          run_algorithm(
-              capital_base=0.5,
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bittrex',
-              live=True,
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              live_graph=False
-          )
+.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
 
@@ -762,8 +158,6 @@ strategy.
 Simple Universe
 ~~~~~~~~~~~~~~~
 
-Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
-
 This example aims to provide an easy way for users to learn how to 
 collect data from any given exchange and select a subset of the available
 currency pairs for trading. You simply need to specify the exchange and 
@@ -790,142 +184,10 @@ of the file:
   
   catalyst ingest-exchange -x bitfinex -f minute
 
-.. code-block:: bash
-
-  python simple_universe.py
-
-Credits: This code was originally submitted by `Abner Ayala-Acevedo 
-<https://github.com/abnera>`_. Thank you!
-
-.. code-block:: python
-
-  from datetime import timedelta
-
-  import numpy as np
-  import pandas as pd
-
-  from catalyst import run_algorithm
-  from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
-  from catalyst.api import (symbols, )
-
-
-  def initialize(context):
-      context.i = -1  # minute counter
-      context.exchange = context.exchanges.values()[0].name.lower()  
-      context.base_currency = context.exchanges.values()[0].base_currency.lower()  
-
-
-  def handle_data(context, data):
-      context.i += 1
-      lookback_days = 7  # 7 days
-
-      # current date & time in each iteration formatted into a string
-      now = data.current_dt
-      date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-      lookback_date = now - timedelta(days=lookback_days) 
-      # keep only the date as a string, discard the time
-      lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  
-
-      one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
-      # update universe everyday at midnight
-      if not context.i % one_day_in_minutes:
-          context.universe = universe(context, lookback_date, date)
-
-      # get data every 30 minutes
-      minutes = 30
-      # get lookback_days of history data: that is 'lookback' number of bins
-      lookback = one_day_in_minutes / minutes * lookback_days  
-      if not context.i % minutes and context.universe:
-          # we iterate for every pair in the current universe
-          for coin in context.coins:
-              pair = str(coin.symbol)
-
-              # Get 30 minute interval OHLCV data. This is the standard data 
-              # required for candlestick or indicators/signals. Return Pandas
-              # DataFrames. 30T means 30-minute re-sampling of one minute data. 
-              # Adjust it to your desired time interval as needed.
-              opened = fill(data.history(coin, 'open', 
-                                    bar_count=lookback, frequency='30T')).values
-              high = fill(data.history(coin, 'high', 
-                                    bar_count=lookback, frequency='30T')).values
-              low = fill(data.history(coin, 'low', 
-                                    bar_count=lookback, frequency='30T')).values
-              close = fill(data.history(coin, 'price', 
-                                    bar_count=lookback, frequency='30T')).values
-              volume = fill(data.history(coin, 'volume', 
-                                    bar_count=lookback, frequency='30T')).values
-
-              # close[-1] is the last value in the set, which is the equivalent 
-              # to current price (as in the most recent value)
-              # displays the minute price for each pair every 30 minutes
-              print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
-                      now=now, 
-                      pair=pair, 
-                      o=opened[-1], 
-                      h=high[-1], 
-                      l=low[-1],
-                      c=close[-1],
-                      v=volume[-1],
-                   ))
-
-              # -------------------------------------------------------------
-              # --------------- Insert Your Strategy Here -------------------
-              # -------------------------------------------------------------
-
-
-  def analyze(context=None, results=None):
-      pass
-
-
-  # Get the universe for a given exchange and a given base_currency market
-  # Example: Poloniex BTC Market
-  def universe(context, lookback_date, current_date):
-      # get all the pairs for the given exchange
-      json_symbols = get_exchange_symbols(context.exchange)  
-      # convert into a DataFrame for easier processing
-      df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) 
-      df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
-      df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
-
-      # Filter all the pairs to get only the ones for a given base_currency
-      df = df[df['base_currency'] == context.base_currency]
-
-      # Filter all the pairs to ensure that pair existed in the current date range
-      df = df[df.start_date < lookback_date]
-      df = df[df.end_daily >= current_date]
-      context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
-
-      return df.symbol.tolist()
-
-
-  # Replace all NA, NAN or infinite values with its nearest value
-  def fill(series):
-      if isinstance(series, pd.Series):
-          return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-      elif isinstance(series, np.ndarray):
-          return pd.Series(series).replace(
-                       [np.inf, -np.inf], np.nan
-                      ).ffill().bfill().values
-      else:
-          return series
-
-
-  if __name__ == '__main__':
-      start_date = pd.to_datetime('2017-11-10', utc=True)
-      end_date = pd.to_datetime('2017-11-13', utc=True)
-
-      performance = run_algorithm(start=start_date, end=end_date,
-                                  capital_base=100.0,  # amount of base_currency
-                                  initialize=initialize,
-                                  handle_data=handle_data,
-                                  analyze=analyze,
-                                  exchange_name='bitfinex',
-                                  data_frequency='minute',
-                                  base_currency='btc',
-                                  live=False,
-                                  live_graph=False,
-                                  algo_namespace='simple_universe')
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
 
+.. literalinclude:: ../../catalyst/examples/simple_universe.py
+   :language: python
 
 
 .. _portfolio_optimization:
@@ -939,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used
 in writting the following article: 
 `Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
 
-.. code-block:: python
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
 
-  '''
-     You can run this code using the Python interpreter:
-
-     $ python portfolio_optimization.py
-  '''
-
-  from __future__ import division
-  import os
-  import pytz
-  import numpy as np
-  import pandas as pd
-  from scipy.optimize import minimize
-  import matplotlib.pyplot as plt
-  from datetime import datetime
-
-  from catalyst.api import record, symbol, symbols, order_target_percent
-  from catalyst.utils.run_algo import run_algorithm
-
-  np.set_printoptions(threshold='nan', suppress=True)
-
-
-  def initialize(context):
-     # Portfolio assets list
-     context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
-                              'xmr_usdt')
-     context.nassets = len(context.assets)
-     # Set the time window that will be used to compute expected return 
-     # and asset correlations
-     context.window = 180
-     # Set the number of days between each portfolio rebalancing
-     context.rebalance_period = 30                   
-     context.i = 0
-
-     
-  def handle_data(context, data):
-     # Only rebalance at the beggining of the algorithm execution and 
-     # every multiple of the rebalance period
-     if context.i == 0 or context.i%context.rebalance_period == 0:
-         n = context.window
-         prices = data.history(context.assets, fields='price', 
-                               bar_count=n+1, frequency='1d') 
-         pr = np.asmatrix(prices)
-         t_prices = prices.iloc[1:n+1]
-         t_val = t_prices.values
-         tminus_prices = prices.iloc[0:n]
-         tminus_val = tminus_prices.values
-         # Compute daily returns (r)
-         r = np.asmatrix(t_val/tminus_val-1)
-         # Compute the expected returns of each asset with the average 
-         # daily return for the selected time window
-         m = np.asmatrix(np.mean(r, axis=0))
-         # ###
-         stds = np.std(r, axis=0)
-         # Compute excess returns matrix (xr)
-         xr = r - m
-         # Matrix algebra to get variance-covariance matrix
-         cov_m = np.dot(np.transpose(xr),xr)/n
-         # Compute asset correlation matrix (informative only)
-         corr_m = cov_m/np.dot(np.transpose(stds),stds)
-         
-         # Define portfolio optimization parameters
-         n_portfolios = 50000
-         results_array = np.zeros((3+context.nassets,n_portfolios))
-         for p in xrange(n_portfolios):
-             weights = np.random.random(context.nassets)
-             weights /= np.sum(weights)
-             w = np.asmatrix(weights)
-             p_r = np.sum(np.dot(w,np.transpose(m)))*365
-             p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
-             
-             #store results in results array
-             results_array[0,p] = p_r
-             results_array[1,p] = p_std
-             #store Sharpe Ratio (return / volatility) - risk free rate element 
-             #excluded for simplicity
-             results_array[2,p] = results_array[0,p] / results_array[1,p]
-             i = 0
-             for iw in weights:
-                 results_array[3+i,p] = weights[i]
-                 i += 1
-         
-         #convert results array to Pandas DataFrame
-         results_frame = pd.DataFrame(np.transpose(results_array),
-                            columns=['r','stdev','sharpe']+context.assets)
-         #locate position of portfolio with highest Sharpe Ratio
-         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
-         #locate positon of portfolio with minimum standard deviation
-         min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
-         
-         #order optimal weights for each asset
-         for asset in context.assets:
-             if data.can_trade(asset):
-                 order_target_percent(asset, max_sharpe_port[asset])
-         
-         #create scatter plot coloured by Sharpe Ratio
-         plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
-         plt.xlabel('Volatility')
-         plt.ylabel('Returns')
-         plt.colorbar()
-         #plot red star to highlight position of portfolio with highest Sharpe Ratio
-         plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
-         #plot green star to highlight position of minimum variance portfolio
-         plt.show()
-         print(max_sharpe_port)
-         record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
-     context.i += 1
-     
-         
-  def analyze(context=None, results=None):
-     # Form DataFrame with selected data
-     data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
-     
-     # Save results in CSV file
-     filename = os.path.splitext(os.path.basename(__file__))[0]
-     data.to_csv(filename + '.csv')
-
-
-  # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
-  start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
-  end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) 
-  results = run_algorithm(initialize=initialize,
-                          handle_data=handle_data,
-                          analyze=analyze,
-                          start=start,
-                          end=end,
-                          exchange_name='poloniex',
-                          capital_base=100000, )
+.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
+   :language: python
 
 .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
    :align: center

docs/source/index.rst

@@ -1,6 +1,8 @@
 .. include:: ../../README.rst
+
 |
 |
+
 Table of Contents
 -----------------
 

docs/source/install.rst

@@ -47,8 +47,10 @@ you can install MiniConda, which is a smaller footprint (fewer packages and
 smaller size) than its big brother Anaconda, but it still contains all the 
 main packages needed. To install MiniConda, you can follow these steps:
 
-1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
-   for your Operating System.
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select either 
+   Python 3.6 (recommended) or Python 2.7 for your Operating System. The 
+   `Enigma Data Marketplace <https://enigmampc.github.io/marketplace/>`_ will 
+   require Python3, that's why we are recommending to opt for the newer version.
 2. Install MiniConda. See the `Installation Instructions 
    <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
 3. Ensure the correct installation by running ``conda list`` in a Terminal 
@@ -64,21 +66,30 @@ main packages needed. To install MiniConda, you can follow these steps:
 
 Once either Conda or MiniConda has been set up you can install Catalyst:
 
-1. Download the file `python2.7-environment.yml 
-   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+1. Download the file `python3.6-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python3.6-environment.yml>`_ 
+   (recommended) or `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_ 
+   matching your Conda installation from step #1 above.
 
      To download, simply click on the 'Raw' button and save the file locally 
      to a folder you can remember. Make sure that the file gets saved with the
      ``.yml`` extension, and nothing like a ``.txt`` file or anything else.
 
 2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
-   saved the above ``python2.7-environment.yml`` file.
+   saved the above ``.yml`` file.
 
 3. Install using this file. This step can take about 5-10 minutes to install.
 
    .. code-block:: bash
 
-      conda env create -f python2.7-environment.yml
+      conda env create -f python3.6-environment.yml
+
+  or
+
+   .. code-block:: bash
+
+    conda env create -f python2.7-environment.yml
 
 4. Activate the environment (which you need to do every time you start a new 
    session to run Catalyst):
@@ -298,7 +309,7 @@ Troubleshooting ``pip`` Install
 .. _pipenv:
 
 Installing with ``pipenv``
--------------------------
+--------------------------
 
 Installing Catalyst via ``pipenv`` is perhaps easier that installing it via
 ``pip`` itself but you need to install ``pipenv`` first via ``pip``.
@@ -443,12 +454,22 @@ about matplotlib backends, please refer to the
 Windows Requirements
 --------------------
 
-In Windows, you will first need to install the `Microsoft Visual C++ Compiler 
-for Python 2.7 
-<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
-package contains the compiler and the set of system headers necessary for 
-producing binary wheels for Python 2.7 packages. If it's not already in your 
-system, download it and install it before proceeding to the next step.
+In Windows, you will first need to install the Microsoft Visual C++ Compiler, 
+which is different depending on the version of Python that you plan to use:
+
+* Python 3.5, 3.6: `Visual C++ 2015 Build Tools 
+  <http://landinghub.visualstudio.com/visual-cpp-build-tools>`_, 
+  which installs Visual C++ version 14.0. **This is the recommended version**
+
+* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7 
+  <https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_, which 
+  installs version Visual C++ version 9.0
+
+This package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python packages. If it's not already in your 
+system, download it and install it before proceeding to the next step. If you 
+need additional help, or are looking for other versions of Visual C++ for 
+Windows (only advanced users), follow `this link <https://wiki.python.org/moin/WindowsCompilers>`_.
 
 Once you have the above compiler installed, the easiest and best supported way 
 to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't 
@@ -476,6 +497,7 @@ mentioned above are as follows:
     default you get 0 as the Value Data)
 
 |
+
 - **The installer has encountered an unexpected error installing this package. 
   This may indicate a problem with this package. The error code is 2503.**
 

docs/source/live-trading.rst

@@ -30,22 +30,24 @@ Paper Trading vs Live Trading modes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Catalyst currently supports three different modes in which you can execute your
-trading algorithm. The first is backtesting, which is covered extensively in the 
-tutorial, and uses historical data to run your algorithm. There is no 
+trading algorithm. The first is **backtesting**, which is covered extensively in
+the tutorial, and uses historical data to run your algorithm. There is no 
 interaction with the exchange in backtesting mode, and this is the first mode 
 that you should test any new algorithm.
 
 Once you are confident with the simulations that you have obtained with your
 algorithm in backtesting, you may switch to live trading, where you have two
 different modes:
-* *Paper Trading*: The simulated algorithm runs in real time, and fetches 
-pricing data in real time from the exchange, but the orders never reach the 
-exchange, and are instead kept within Catalyst and simulated. No real currency
-is bought or sold. Think of it as a `backtesting happening in real time`.
-* *Live Trading*: This is the proper live trading mode in which an algorithm
-runs in real time, fetching pricing data from live exchanges and placing orders 
-against the exchange. Real currency is transacted on the exchange driven by the 
-algorithm.
+
+* **Paper Trading**: The simulated algorithm runs in real time, and fetches 
+  pricing data in real time from the exchange, but the orders never reach the 
+  exchange, and are instead kept within Catalyst and simulated. No real currency
+  is bought or sold. Think of it as a `backtesting happening in real time`.
+
+* **Live Trading**: This is the proper live trading mode in which an algorithm
+  runs in real time, fetching pricing data from live exchanges and placing 
+  orders against the exchange. Real currency is transacted on the exchange 
+  driven by the algorithm.
 
 These three modes are controlled by the following variables:
 
@@ -113,7 +115,7 @@ Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
 
 Here are some examples:
 
-.. code-block:: json
+.. code:: python
 
   # With Bitfinex
   bitcoin_usd_asset = symbol('btc_usd')
@@ -174,6 +176,13 @@ Here is the breakdown of the new arguments:
 - ``simulate_orders``: Enables the paper trading mode, in which orders are
   simulated in Catalyst instead of processed on the exchange. It defaults to 
   ``True``.
+- ``end_date``: When setting the end_date to a time in the **future**,
+  it will schedule the live algo to finish gracefully at the specified date.
+- ``start_date``: (**Will be implemented in the future**)
+  The live algo starts by default in the present, as mentioned above.
+  by setting the start_date to a time in the future, the algorithm would
+  essentially sleep and when the predefined time comes, it would start executing.
+
 
 
 The `catalyst live` command offers additional parameters.

docs/source/releases.rst

@@ -2,6 +2,38 @@
 Release Notes
 =============
 
+Version 0.5.3
+^^^^^^^^^^^^^
+**Release Date**: 2018-02-09
+
+Bug Fixes
+~~~~~~~~~
+- Fixed an issue with last candle in backtesting :issue:`219`
+
+Version 0.5.2
+^^^^^^^^^^^^^
+**Release Date**: 2018-02-08
+
+Bug Fixes
+~~~~~~~~~
+- Fixed an issue with live candle values :issue:`216` and :issue:`199`
+
+Version 0.5.1
+^^^^^^^^^^^^^
+**Release Date**: 2018-02-07
+
+Bug Fixes
+~~~~~~~~~
+- Fixed an issue with orders that stay open :issue:`211`
+- Fixed Jupyter issues :issue:`179`
+- Fetching multiple tickers in one call to minimize rate limit risks :issue:`174`
+- Improved live state presentation :issue:`171`
+
+
+Build
+~~~~~
+- Introducing the Enigma Marketplace
+
 Version 0.4.7
 ^^^^^^^^^^^^^
 **Release Date**: 2018-01-19

docs/source/videos.rst

@@ -11,6 +11,7 @@ Installation: MacOS
 
 |
 |
+
 Installation: Windows
 ---------------------
 
@@ -21,6 +22,7 @@ Where things go smoothly:
    <iframe width="560" height="315" src="https://www.youtube.com/embed/H8HqcEbZmkk" frameborder="0" allowfullscreen></iframe>
 
 |
+
 Where things don't:
 
 .. raw:: html
@@ -29,6 +31,7 @@ Where things don't:
 
 |
 |
+
 Backtesting a Strategy
 ----------------------
 
@@ -44,6 +47,7 @@ sell. Hopefully, we’ll ride the waves.
 
 |
 |
+
 Live Trading a Strategy
 -----------------------
 
@@ -54,5 +58,6 @@ in the previous video, we now take it to trade live against the Bittrex exchange
 .. raw:: html
 
 	<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
+
+|
 |
-|

etc/docker_cmd.sh

@@ -16,4 +16,4 @@ fi
 
 jupyter notebook -y --no-browser --notebook-dir=${PROJECT_DIR} \
     --certfile=${SSL_CERT_PEM} --keyfile=${SSL_CERT_KEY} --ip='*' \
-    --config=${CONFIG_PATH}
+    --config=${CONFIG_PATH} --allow-root

etc/python2.7-environment.yml

@@ -20,7 +20,9 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
-  - ccxt==1.10.944
+  - ccxt==1.10.1094
+  - web3==4.0.0b7
+  - requests-toolbelt==0.8.0
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0
@@ -57,4 +59,4 @@ dependencies:
   - tables==3.4.2
   - toolz==0.8.2
   - urllib3==1.22
-  - enigma-catalyst>=0.3
+  - enigma-catalyst>=0.5

etc/python3.6-environment.yml

@@ -0,0 +1,95 @@
+name: catalyst
+channels:
+- defaults
+dependencies:
+- ca-certificates=2017.08.26=ha1e5d58_0
+- certifi=2018.1.18=py36_0
+- intel-openmp=2018.0.0=h8158457_8
+- libcxx=4.0.1=h579ed51_0
+- libcxxabi=4.0.1=hebd6815_0
+- libedit=3.1=hb4e282d_0
+- libffi=3.2.1=h475c297_4
+- libgfortran=3.0.1=h93005f0_2
+- mkl=2018.0.1=hfbd8650_4
+- ncurses=6.0=hd04f020_2
+- numpy=1.14.0=py36h8a80b8c_1
+- openssl=1.0.2n=hdbc3d79_0
+- pip=9.0.1=py36h1555ced_4
+- python=3.6.4=hc167b69_1
+- readline=7.0=hc1231fa_4
+- scipy=1.0.0=py36h1de22e9_0
+- setuptools=38.4.0=py36_0
+- sqlite=3.22.0=h3efe00b_0
+- tk=8.6.7=h35a86e2_3
+- wheel=0.30.0=py36h5eb2c71_1
+- xz=5.2.3=h0278029_2
+- zlib=1.2.11=hf3cbc9b_2
+- pip:
+  - aiodns==1.1.1
+  - aiohttp==3.0.1
+  - alembic==0.9.7
+  - async-timeout==2.0.0
+  - attrdict==2.0.0
+  - attrs==17.4.0
+  - bcolz==0.12.1
+  - boto3==1.5.27
+  - botocore==1.8.41
+  - bottleneck==1.2.1
+  - cchardet==2.1.1
+  - ccxt==1.10.1102
+  - chardet==3.0.4
+  - click==6.7
+  - contextlib2==0.5.5
+  - cyordereddict==1.0.0
+  - cython==0.27.3
+  - cytoolz==0.9.0
+  - decorator==4.2.1
+  - docutils==0.14
+  - empyrical==0.2.1
+  - enigma-catalyst>=0.5.3
+  - eth-abi==1.0.0b0
+  - eth-account==0.1.0a2
+  - eth-keyfile==0.5.1
+  - eth-keys==0.2.0b1
+  - eth-rlp==0.1.0a2
+  - eth-utils==1.0.0b1
+  - hexbytes==0.1.0b0
+  - idna==2.6
+  - idna-ssl==1.0.0
+  - intervaltree==2.1.0
+  - jmespath==0.9.3
+  - logbook==1.2.1
+  - lru-dict==1.1.6
+  - lxml==4.1.1
+  - mako==1.0.7
+  - markupsafe==1.0
+  - multidict==4.1.0
+  - multipledispatch==0.4.9
+  - networkx==2.1
+  - numexpr==2.6.4
+  - pandas==0.19.2
+  - pandas-datareader==0.6.0
+  - patsy==0.5.0
+  - pycares==2.3.0
+  - pycryptodome==3.4.11
+  - pysha3==1.0.2
+  - python-dateutil==2.6.1
+  - python-editor==1.0.3
+  - pytz==2018.3
+  - redo==1.6
+  - requests==2.18.4
+  - requests-file==1.4.3
+  - requests-ftp==0.3.1
+  - requests-toolbelt==0.8.0
+  - rlp==0.6.0
+  - s3transfer==0.1.12
+  - six==1.11.0
+  - sortedcontainers==1.5.9
+  - sqlalchemy==1.2.2
+  - statsmodels==0.8.0
+  - tables==3.4.2
+  - toolz==0.9.0
+  - urllib3==1.22
+  - web3==4.0.0b9
+  - wrapt==1.10.11
+  - yarl==1.1.0

etc/requirements.txt

@@ -81,6 +81,8 @@ empyrical==0.2.1
 tables==3.3.0
 
 #Catalyst dependencies
-ccxt==1.10.944
+ccxt==1.10.1094
 boto3==1.4.8
 redo==1.6
+web3==4.0.0b7
+requests-toolbelt==0.8.0

etc/requirements_dev.txt

@@ -16,7 +16,7 @@ babel==1.3
 docutils==0.12
 snowballstemmer==1.2.0
 sphinx-rtd-theme==0.1.8
-sphinx==1.3.4
+sphinx==1.6.7
 pbr==1.10.0
 
 mock==2.0.0

etc/requirements_docs.txt

@@ -1,4 +1,4 @@
-Sphinx>=1.3.2
+Sphinx==1.6.7
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
 docutils==0.12

etc/requirements_marketplace.txt

@@ -0,0 +1,2 @@
+web3==4.0.0b7
+requests-toolbelt==0.8.0

tests/exchange/test_ccxt.py

@@ -1,8 +1,7 @@
 import pandas as pd
 from logbook import Logger
 
-from catalyst.testing import ZiplineTestCase
-from catalyst.testing.fixtures import WithLogger
+from catalyst.exchange.utils.stats_utils import set_print_settings
 from .base import BaseExchangeTestCase
 from catalyst.exchange.ccxt.ccxt_exchange import CCXT
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
@@ -15,7 +14,7 @@ log = Logger('test_ccxt')
 class TestCCXT(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        exchange_name = 'binance'
+        exchange_name = 'bittrex'
         auth = get_exchange_auth(exchange_name)
         self.exchange = CCXT(
             exchange_name=exchange_name,
@@ -58,15 +57,20 @@ class TestCCXT(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         candles = self.exchange.get_candles(
-            freq='30T',
+            freq='1T',
             assets=[self.exchange.get_asset('eth_btc')],
             bar_count=200,
-            start_dt=pd.to_datetime('2017-09-01', utc=True)
+            # start_dt=pd.to_datetime('2017-09-01', utc=True),
         )
 
         for asset in candles:
             df = pd.DataFrame(candles[asset])
             df.set_index('last_traded', drop=True, inplace=True)
+
+        set_print_settings()
+        print('got {} candles'.format(len(df)))
+        print(df.head(10))
+        print(df.tail(10))
         pass
 
     def test_tickers(self):

tests/exchange/test_suites/test_suite_bundle.py

@@ -2,6 +2,7 @@ import random
 
 import os
 import pandas as pd
+from datetime import timedelta
 from logbook import TestHandler
 from pandas.util.testing import assert_frame_equal
 
@@ -12,6 +13,7 @@ from catalyst.exchange.utils.exchange_utils import get_candles_df
 from catalyst.exchange.utils.factory import get_exchange
 from catalyst.exchange.utils.test_utils import output_df, \
     select_random_assets
+from catalyst.exchange.utils.stats_utils import set_print_settings
 
 pd.set_option('display.expand_frame_repr', False)
 pd.set_option('precision', 8)
@@ -35,7 +37,7 @@ class TestSuiteBundle:
         return data_portal
 
     def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     freq, data_frequency, data_portal):
+                                     freq, data_frequency, data_portal, field):
         """
         Creates DataFrames from the bundle and exchange for the specified
         data set.
@@ -58,14 +60,26 @@ class TestSuiteBundle:
 
         log_catcher = TestHandler()
         with log_catcher:
+            symbols = [asset.symbol for asset in assets]
+            print(
+                'comparing {} for {}/{} with {} timeframe until {}'.format(
+                    field, exchange.name, symbols, freq, end_dt
+                )
+            )
             data['bundle'] = data_portal.get_history_window(
                 assets=assets,
                 end_dt=end_dt,
                 bar_count=bar_count,
                 frequency=freq,
-                field='close',
+                field=field,
                 data_frequency=data_frequency,
             )
+            set_print_settings()
+            print(
+                'the bundle data:\n{}'.format(
+                    data['bundle']
+                )
+            )
             candles = exchange.get_candles(
                 end_dt=end_dt,
                 freq=freq,
@@ -74,11 +88,16 @@ class TestSuiteBundle:
             )
             data['exchange'] = get_candles_df(
                 candles=candles,
-                field='close',
+                field=field,
                 freq=freq,
                 bar_count=bar_count,
                 end_dt=end_dt,
             )
+            print(
+                'the exchange data:\n{}'.format(
+                    data['exchange']
+                )
+            )
             for source in data:
                 df = data[source]
                 path, folder = output_df(
@@ -99,13 +118,74 @@ class TestSuiteBundle:
                     check_less_precise=min([a.decimals for a in assets]),
                 )
             except Exception as e:
-                print('Some differences were found within a 1 decimal point '
-                      'interval of confidence: {}'.format(e))
+                print(
+                    'Some differences were found within a 1 decimal point '
+                    'interval of confidence: {}'.format(e)
+                )
                 with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
                     handle.write(e.args[0])
 
             pass
 
+    def compare_current_with_last_candle(self, exchange, assets, end_dt,
+                                         freq, data_frequency, data_portal):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        freq
+        data_frequency
+        data_portal
+
+        Returns
+        -------
+
+        """
+        data = dict()
+
+        assets = sorted(assets, key=lambda a: a.symbol)
+        log_catcher = TestHandler()
+        with log_catcher:
+            symbols = [asset.symbol for asset in assets]
+            print(
+                'comparing data for {}/{} with {} timeframe on {}'.format(
+                    exchange.name, symbols, freq, end_dt
+                )
+            )
+            data['candle'] = data_portal.get_history_window(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=1,
+                frequency=freq,
+                field='close',
+                data_frequency=data_frequency,
+            )
+            set_print_settings()
+            print(
+                'the bundle first / last row:\n{}'.format(
+                    data['candle'].iloc[[-1]]
+                )
+            )
+            current = data_portal.get_spot_value(
+                assets=assets,
+                field='close',
+                dt=end_dt,
+                data_frequency=data_frequency,
+            )
+            data['current'] = pd.Series(data=current, index=assets)
+            print(
+                'the current price:\n{}'.format(
+                    data['current']
+                )
+            )
+            pass
+
     def test_validate_bundles(self):
         # exchange_population = 3
         asset_population = 3
@@ -125,8 +205,11 @@ class TestSuiteBundle:
 
             frequencies = exchange.get_candle_frequencies(data_frequency)
             freq = random.sample(frequencies, 1)[0]
+            rnd = random.SystemRandom()
+            # field = rnd.choice(['open', 'high', 'low', 'close', 'volume'])
+            field = rnd.choice(['volume'])
 
-            bar_count = random.randint(1, 10)
+            bar_count = random.randint(3, 6)
 
             assets = select_random_assets(
                 exchange.assets, asset_population
@@ -139,6 +222,7 @@ class TestSuiteBundle:
                 if end_dt is None or asset_end_dt < end_dt:
                     end_dt = asset_end_dt
 
+            end_dt = end_dt + timedelta(minutes=3)
             dt_range = pd.date_range(
                 end=end_dt, periods=bar_count, freq=freq
             )
@@ -150,5 +234,48 @@ class TestSuiteBundle:
                 freq=freq,
                 data_frequency=data_frequency,
                 data_portal=data_portal,
+                field=field,
+            )
+        pass
+
+    def test_validate_last_candle(self):
+        # exchange_population = 3
+        asset_population = 3
+        data_frequency = random.choice(['minute'])
+
+        # bundle = 'dailyBundle' if data_frequency
+        #  == 'daily' else 'minuteBundle'
+        # exchanges = select_random_exchanges(
+        #     population=exchange_population,
+        #     features=[bundle],
+        # )  # Type: list[Exchange]
+        exchanges = [get_exchange('poloniex', skip_init=True)]
+
+        data_portal = TestSuiteBundle.get_data_portal(exchanges)
+        for exchange in exchanges:
+            exchange.init()
+
+            frequencies = exchange.get_candle_frequencies(data_frequency)
+            freq = random.sample(frequencies, 1)[0]
+
+            assets = select_random_assets(
+                exchange.assets, asset_population
+            )
+            end_dt = None
+            for asset in assets:
+                attribute = 'end_{}'.format(data_frequency)
+                asset_end_dt = getattr(asset, attribute)
+
+                if end_dt is None or asset_end_dt < end_dt:
+                    end_dt = asset_end_dt
+
+            end_dt = end_dt + timedelta(minutes=3)
+            self.compare_current_with_last_candle(
+                exchange=exchange,
+                assets=assets,
+                end_dt=end_dt,
+                freq=freq,
+                data_frequency=data_frequency,
+                data_portal=data_portal,
             )
         pass

tests/exchange/test_suites/test_suite_exchange.py

@@ -15,7 +15,7 @@ from catalyst.exchange.utils.test_utils import select_random_exchanges, \
     handle_exchange_error, select_random_assets
 from catalyst.testing import ZiplineTestCase
 from catalyst.testing.fixtures import WithLogger
-from catalyst.exchange.utils.factory import get_exchanges
+from catalyst.exchange.utils.factory import get_exchanges, get_exchange
 
 log = Logger('TestSuiteExchange')
 
@@ -90,7 +90,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
         #     exchange_population,
         #     features=['fetchTickers'],
         # )  # Type: list[Exchange]
-        exchanges = list(get_exchanges(['bitfinex']).values())
+        exchanges = list(get_exchanges(['binance']).values())
         for exchange in exchanges:
             exchange.init()
 
@@ -113,10 +113,11 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
         exchange_population = 3
         asset_population = 3
 
-        exchanges = select_random_exchanges(
-            population=exchange_population,
-            features=['fetchOHLCV'],
-        )  # Type: list[Exchange]
+        # exchanges = select_random_exchanges(
+        #     population=exchange_population,
+        #     features=['fetchOHLCV'],
+        # )  # Type: list[Exchange]
+        exchanges = list(get_exchanges(['binance']).values())
         for exchange in exchanges:
             exchange.init()
 
@@ -154,13 +155,20 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
         quote_currency = 'eth'
         order_amount = 0.1
 
-        exchanges = select_random_exchanges(
-            population=population,
-            features=['fetchOrder'],
-            is_authenticated=True,
-            base_currency=quote_currency,
-        )  # Type: list[Exchange]
+        # exchanges = select_random_exchanges(
+        #     population=population,
+        #     features=['fetchOrder'],
+        #     is_authenticated=True,
+        #     base_currency=quote_currency,
+        # )  # Type: list[Exchange]
 
+        exchanges = [
+            get_exchange(
+                'binance',
+                base_currency=quote_currency,
+                must_authenticate=True,
+            )
+        ]
         log_catcher = TestHandler()
         with log_catcher:
             for exchange in exchanges:

tests/marketplace/test_marketplace.py

@@ -0,0 +1,36 @@
+from catalyst.marketplace.marketplace import Marketplace
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+import pandas as pd
+
+
+class TestMarketplace(WithLogger, ZiplineTestCase):
+    def test_list(self):
+        marketplace = Marketplace()
+        marketplace.list()
+        pass
+
+    def test_register(self):
+        marketplace = Marketplace()
+        marketplace.register()
+        pass
+
+    def test_subscribe(self):
+        marketplace = Marketplace()
+        marketplace.subscribe('marketcap2222')
+        pass
+
+    def test_ingest(self):
+        marketplace = Marketplace()
+        ds_def = marketplace.ingest('github')
+        pass
+
+    def test_publish(self):
+        marketplace = Marketplace()
+        datadir = '/Users/fredfortier/Downloads/marketcap_test_single'
+        marketplace.publish('marketcap1234', datadir, False)
+        pass
+
+    def test_clean(self):
+        marketplace = Marketplace()
+        marketplace.clean('marketcap')
+        pass