BLD: adjusting unit tests

# Conflicts:
#	tests/exchange/test_bundle.py

.gitignore

@@ -40,6 +40,7 @@ develop-eggs
 coverage.xml
 htmlcov
 nosetests.xml
+.python-version
 
 # C Extensions
 *.o

Dockerfile

@@ -11,13 +11,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

Dockerfile-dev

@@ -5,7 +5,7 @@
 #
 # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile
+#    docker build -t quantopian/catalyst -f Dockerfile .
 #
 # To run the container:
 #
@@ -15,13 +15,13 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
-# You can also run an algo using the docker exec command.  For example:
+# You can also run an algo using the docker exec command. For example:
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #

README.rst

@@ -1,4 +1,4 @@
-.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png
     :target: https://enigmampc.github.io/catalyst
     :align: center
     :alt: Enigma | Catalyst
@@ -17,9 +17,7 @@ insights regarding a particular strategy's performance. Catalyst also supports
 live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
 and Poloniex) with more being added over time. Catalyst empowers users to share 
 and curate data and build profitable, data-driven investment strategies. Please 
-visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
-refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
-further technical details.
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
 
 Catalyst builds on top of the well-established 
 `Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 

catalyst/__main__.py

@@ -3,13 +3,15 @@ import os
 from functools import wraps
 
 import click
+import sys
 import logbook
 import pandas as pd
+from catalyst.marketplace.marketplace import Marketplace
 from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_utils import delete_algo_folder
+from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -257,7 +259,7 @@ def run(ctx,
     if capital_base is None:
         ctx.fail("must specify a capital base with '--capital-base'")
 
-    click.echo('Running in backtesting mode.')
+    click.echo('Running in backtesting mode.', sys.stdout)
 
     perf = _run(
         initialize=None,
@@ -282,13 +284,15 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
+        analyze_live=None,
         live_graph=False,
         simulate_orders=True,
+        auth_aliases=None,
         stats_output=None,
     )
 
     if output == '-':
-        click.echo(str(perf))
+        click.echo(str(perf), sys.stdout)
     elif output != os.devnull:  # make the catalyst magic not write any data
         perf.to_pickle(output)
 
@@ -312,11 +316,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                # these options are set when running in line magic mode
-                # set a non None algo text to use the ipython user_ns
-                '--algotext', '',
-                '--local-namespace',
-            ] if cell is None else []) + line.split(),
+                     # these options are set when running in line magic mode
+                     # set a non None algo text to use the ipython user_ns
+                     '--algotext', '',
+                     '--local-namespace',
+                 ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,
@@ -393,6 +397,12 @@ def catalyst_magic(line, cell=None):
     help='The base currency used to calculate statistics '
          '(e.g. usd, btc, eth).',
 )
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
 @click.option(
     '--live-graph/--no-live-graph',
     is_flag=True,
@@ -406,6 +416,15 @@ def catalyst_magic(line, cell=None):
     help='Simulating orders enable the paper trading mode. No orders will be '
          'sent to the exchange unless set to false.',
 )
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
 @click.pass_context
 def live(ctx,
          algofile,
@@ -418,7 +437,9 @@ def live(ctx,
          exchange_name,
          algo_namespace,
          base_currency,
+         end,
          live_graph,
+         auth_aliases,
          simulate_orders):
     """Trade live with the given algorithm.
     """
@@ -441,10 +462,10 @@ def live(ctx,
         ctx.fail("must specify a capital base with '--capital-base'")
 
     if simulate_orders:
-        click.echo('Running in paper trading mode.')
+        click.echo('Running in paper trading mode.', sys.stdout)
 
     else:
-        click.echo('Running in live trading mode.')
+        click.echo('Running in live trading mode.', sys.stdout)
 
     perf = _run(
         initialize=None,
@@ -460,7 +481,7 @@ def live(ctx,
         bundle=None,
         bundle_timestamp=None,
         start=None,
-        end=None,
+        end=end,
         output=output,
         print_algo=print_algo,
         local_namespace=local_namespace,
@@ -470,12 +491,14 @@ def live(ctx,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=None,
         simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
         stats_output=None,
     )
 
     if output == '-':
-        click.echo(str(perf))
+        click.echo(str(perf), sys.stdout)
     elif output != os.devnull:  # make the catalyst magic not write any data
         perf.to_pickle(output)
 
@@ -557,7 +580,8 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
+               sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,
@@ -580,10 +604,11 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 @click.pass_context
 def clean_algo(ctx, algo_namespace):
     click.echo(
-        'Cleaning algo state: {}'.format(algo_namespace)
+        'Cleaning algo state: {}'.format(algo_namespace),
+        sys.stdout
     )
     delete_algo_folder(algo_namespace)
-    click.echo('Done')
+    click.echo('Done', sys.stdout)
 
 
 @main.command(name='clean-exchange')
@@ -610,11 +635,12 @@ def clean_exchange(ctx, exchange_name, data_frequency):
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name),
+               sys.stdout)
     exchange_bundle.clean(
         data_frequency=data_frequency,
     )
-    click.echo('Done')
+    click.echo('Done', sys.stdout)
 
 
 @main.command()
@@ -735,7 +761,132 @@ def bundles():
         # because there were no entries, print a single message indicating that
         # no ingestions have yet been made.
         for timestamp in ingestions or ["<no ingestions>"]:
-            click.echo("%s %s" % (bundle, timestamp))
+            click.echo("%s %s" % (bundle, timestamp), sys.stdout)
+
+
+@main.group()
+@click.pass_context
+def marketplace(ctx):
+    """Access the Enigma Data Marketplace to:\n
+    - Register and Publish new datasets (seller-side)\n
+    - Subscribe and Ingest premium datasets (buyer-side)\n
+    """
+    pass
+
+
+@marketplace.command()
+@click.pass_context
+def ls(ctx):
+    """List all available datasets.
+    """
+    click.echo('Listing of available data sources on the marketplace:',
+               sys.stdout)
+    marketplace = Marketplace()
+    marketplace.list()
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def subscribe(ctx, dataset):
+    """Subscribe to an exisiting dataset.
+    """
+    marketplace = Marketplace()
+    marketplace.subscribe(dataset)
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.option(
+    '-f',
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}),
+    default='daily',
+    show_default=True,
+    help='The data frequency of the desired OHLCV bars.',
+)
+@click.option(
+    '-s',
+    '--start',
+    default=None,
+    type=Date(tz='utc', as_timestamp=True),
+    help='The start date of the data range. (default: one year from end date)',
+)
+@click.option(
+    '-e',
+    '--end',
+    default=None,
+    type=Date(tz='utc', as_timestamp=True),
+    help='The end date of the data range. (default: today)',
+)
+@click.pass_context
+def ingest(ctx, dataset, data_frequency, start, end):
+    """Ingest a dataset (requires subscription).
+    """
+    marketplace = Marketplace()
+    marketplace.ingest(dataset, data_frequency, start, end)
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the dataset to ingest from the Data Marketplace.',
+)
+@click.pass_context
+def clean(ctx, dataset):
+    """Clean/Remove local data for a given dataset.
+    """
+    marketplace = Marketplace()
+    marketplace.clean(dataset)
+
+
+@marketplace.command()
+@click.pass_context
+def register(ctx):
+    """Register a new dataset.
+    """
+    marketplace = Marketplace()
+    marketplace.register()
+
+
+@marketplace.command()
+@click.option(
+    '--dataset',
+    default=None,
+    help='The name of the Marketplace dataset to publish data for.',
+)
+@click.option(
+    '--datadir',
+    default=None,
+    help='The folder that contains the CSV data files to publish.',
+)
+@click.option(
+    '--watch/--no-watch',
+    is_flag=True,
+    default=False,
+    help='Whether to watch the datadir for live data.',
+)
+@click.pass_context
+def publish(ctx, dataset, datadir, watch):
+    """Publish data for a registered dataset.
+    """
+    marketplace = Marketplace()
+    if dataset is None:
+        ctx.fail("must specify a dataset to publish data for "
+                 " with '--dataset'\n")
+    if datadir is None:
+        ctx.fail("must specify a datadir where to find the files to publish "
+                 " with '--datadir'\n")
+    marketplace.publish(dataset, datadir, watch)
 
 
 if __name__ == '__main__':

catalyst/algorithm.py

@@ -939,7 +939,7 @@ class TradingAlgorithm(object):
             The field to query. The options have the following meanings:
               arena : str
                   The arena from the simulation parameters. This will normally
-                  be ``'backtest'`` but some systems may use this distinguish
+                  be ``backtest`` but some systems may use this distinguish
                   live trading from backtesting.
               data_frequency : {'daily', 'minute'}
                   data_frequency tells the algorithm if it is running with
@@ -954,7 +954,7 @@ class TradingAlgorithm(object):
                   The platform that the code is running on. By default this
                   will be the string 'catalyst'. This can allow algorithms to
                   know if they are running on the Quantopian platform instead.
-              * : dict[str -> any]
+              \* : dict[str -> any]
                   Returns all of the fields in a dictionary.
 
         Returns
@@ -1032,7 +1032,7 @@ class TradingAlgorithm(object):
             argument is the name of the column in the preprocessed dataframe
             containing the symbols. This will be used along with the date
             information to map the sids in the asset finder.
-        **kwargs
+        \*\*kwargs
             Forwarded to :func:`pandas.read_csv`.
 
         Returns
@@ -1156,7 +1156,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        **kwargs
+        \*\*kwargs
             The names and values to record.
 
         Notes
@@ -1273,7 +1273,7 @@ class TradingAlgorithm(object):
 
         Parameters
         ----------
-        *args : iterable[str]
+        \*args : iterable[str]
             The ticker symbols to lookup.
 
         Returns

catalyst/api.pyi

@@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None):
     :func:`catalyst.api.pipeline_output`
     """
 
+
 def batch_market_order(share_counts):
     """Place a batch market order for multiple assets.
 
@@ -48,6 +49,7 @@ def batch_market_order(share_counts):
         Index of ids for newly-created orders.
     """
 
+
 def cancel_order(order_param):
     """Cancel an open order.
 
@@ -57,7 +59,9 @@ def cancel_order(order_param):
         The order_id or order object to cancel.
     """
 
-def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
+
+def continuous_future(root_symbol_str, offset=0, roll='volume',
+                      adjustment='mul'):
     """Create a specifier for a continuous contract.
 
     Parameters
@@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'
         The continuous future specifier.
     """
 
-def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
+
+def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
+              date_format=None, timezone='UTC', symbol=None, mask=True,
+              symbol_column=None, special_params_checker=None, **kwargs):
     """Fetch a csv from a remote url and register the data so that it is
     queryable from the ``data`` object.
 
@@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma
         A requests source that will pull data from the url specified.
     """
 
+
 def future_symbol(symbol):
     """Lookup a futures contract with a given symbol.
 
@@ -144,6 +152,7 @@ def future_symbol(symbol):
         Raised when no contract named 'symbol' is found.
     """
 
+
 def get_datetime(tz=None):
     """
 Returns the current simulation datetime.
@@ -159,6 +168,7 @@ dt : datetime
     The current simulation datetime converted to ``tz``.
     """
 
+
 def get_environment(field='platform'):
     """Query the execution environment.
 
@@ -198,6 +208,7 @@ def get_environment(field='platform'):
         Raised when ``field`` is not a valid option.
     """
 
+
 def get_order(order_id):
     """Lookup an order based on the order id returned from one of the
     order functions.
@@ -213,10 +224,12 @@ def get_order(order_id):
         The order object.
     """
 
+
 def history(bar_count, frequency, field, ffill=True):
     """DEPRECATED: use ``data.history`` instead.
     """
 
+
 def order(asset, amount, limit_price=None, stop_price=None, style=None):
     """Place an order.
 
@@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
-def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
+
+def order_percent(asset, percent, limit_price=None, stop_price=None,
+                  style=None):
     """Place an order in the specified asset corresponding to the given
     percent of the current portfolio value.
 
@@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None)
     :func:`catalyst.api.order_value`
     """
 
+
 def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     """Place an order to adjust a position to a target number of shares. If
     the position doesn't already exist, this is equivalent to placing a new
@@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_target_value`
     """
 
-def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
+
+def order_target_percent(asset, target, limit_price=None, stop_price=None,
+                         style=None):
     """Place an order to adjust a position to a target percent of the
     current portfolio value. If the position doesn't already exist, this is
     equivalent to placing a new order. If the position does exist, this is
@@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style
     :func:`catalyst.api.order_target_value`
     """
 
-def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
+
+def order_target_value(asset, target, limit_price=None, stop_price=None,
+                       style=None):
     """Place an order to adjust a position to a target value. If
     the position doesn't already exist, this is equivalent to placing a new
     order. If the position does exist, this is equivalent to placing an
@@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N
     :func:`catalyst.api.order_target_percent`
     """
 
+
 def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     """Place an order by desired value rather than desired number of
     shares.
@@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
     :func:`catalyst.api.order_percent`
     """
 
+
 def pipeline_output(name):
     """Get the results of the pipeline that was attached with the name:
     ``name``.
@@ -514,6 +536,7 @@ def pipeline_output(name):
     :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
     """
 
+
 def record(*args, **kwargs):
     """Track and record values each day.
 
@@ -529,7 +552,9 @@ def record(*args, **kwargs):
     :func:`~catalyst.run_algorithm`.
     """
 
-def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
+
+def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
+                      calendar=None):
     """Schedules a function to be called according to some timed rules.
 
     Parameters
@@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale
     :class:`catalyst.api.time_rules`
     """
 
+
 def set_asset_restrictions(restrictions, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'):
     catalyst.finance.asset_restrictions.Restrictions
     """
 
+
 def set_benchmark(benchmark):
     """Set the benchmark asset.
 
@@ -576,6 +603,7 @@ def set_benchmark(benchmark):
     automatically reinvested.
     """
 
+
 def set_cancel_policy(cancel_policy):
     """Sets the order cancellation policy for the simulation.
 
@@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy):
     :class:`catalyst.api.NeverCancel`
     """
 
+
 def set_commission(commission):
     """Sets the commission model for the simulation.
 
@@ -605,6 +634,7 @@ def set_commission(commission):
     :class:`catalyst.finance.commission.PerDollar`
     """
 
+
 def set_do_not_order_list(restricted_list, on_error='fail'):
     """Set a restriction on which assets can be ordered.
 
@@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
         The assets that cannot be ordered.
     """
 
+
 def set_long_only(on_error='fail'):
     """Set a rule specifying that this algorithm cannot take short
     positions.
     """
 
+
 def set_max_leverage(max_leverage):
     """Set a limit on the maximum leverage of the algorithm.
 
@@ -629,6 +661,7 @@ def set_max_leverage(max_leverage):
         be no maximum.
     """
 
+
 def set_max_order_count(max_count, on_error='fail'):
     """Set a limit on the number of orders that can be placed in a single
     day.
@@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'):
         The maximum number of orders that can be placed on any single day.
     """
 
-def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
+
+def set_max_order_size(asset=None, max_shares=None, max_notional=None,
+                       on_error='fail'):
     """Set a limit on the number of shares and/or dollar value of any single
     order placed for sid.  Limits are treated as absolute values and are
     enforced at the time that the algo attempts to place an order for sid.
@@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error=
         The maximum value that can be ordered at one time.
     """
 
-def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
+
+def set_max_position_size(asset=None, max_shares=None, max_notional=None,
+                          on_error='fail'):
     """Set a limit on the number of shares and/or dollar value held for the
     given sid. Limits are treated as absolute values and are enforced at
     the time that the algo attempts to place an order for sid. This means
@@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err
         The maximum value to hold for an asset.
     """
 
+
 def set_slippage(slippage):
     """Set the slippage model for the simulation.
 
@@ -694,6 +732,7 @@ def set_slippage(slippage):
     :class:`catalyst.finance.slippage.SlippageModel`
     """
 
+
 def set_symbol_lookup_date(dt):
     """Set the date for which symbols will be resolved to their assets
     (symbols may map to different firms or underlying assets at
@@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt):
         The new symbol lookup date.
     """
 
+
 def sid(sid):
     """Lookup an Asset by its unique asset identifier.
 
@@ -724,6 +764,7 @@ def sid(sid):
         When a requested ``sid`` does not map to any asset.
     """
 
+
 def symbol(symbol_str):
     """Lookup an Equity by its ticker symbol.
 
@@ -748,6 +789,7 @@ def symbol(symbol_str):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
+
 def symbols(*args):
     """Lookup multuple Equities as a list.
 
@@ -773,3 +815,18 @@ def symbols(*args):
     :func:`catalyst.api.set_symbol_lookup_date`
     """
 
+
+def get_dataset(ds_name, start=None, end=None):
+    """
+    Lookup a data source from the marketplace
+
+    Parameters
+    ----------
+    ds_name: str
+    start: pd.Timestamp
+    end: pd.Timestamp
+
+    Returns
+    -------
+
+    """

catalyst/assets/_assets.pyx

@@ -17,6 +17,7 @@
 """
 Cythonized Asset object.
 """
+
 import hashlib
 
 cimport cython
@@ -38,7 +39,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
-from catalyst.exchange.exchange_utils import get_sid
+from catalyst.exchange.utils.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
@@ -432,7 +433,7 @@ cdef class TradingPair(Asset):
         'taker',
         'trading_state',
         'data_source',
-        'decimals'
+        'decimals',
     })
     def __init__(self,
                  object symbol,
@@ -454,7 +455,7 @@ cdef class TradingPair(Asset):
                  float taker=0.0025,
                  float lot=0,
                  int decimals = 8,
-                 int trading_state=0,
+                 int trading_state=1,
                  object data_source='catalyst'):
         """
         Replicates the Asset constructor with some built-in conventions
@@ -599,14 +600,51 @@ cdef class TradingPair(Asset):
     cpdef to_dict(self):
         """
         Convert to a python dict.
+        
+        Repeat constructor params:
+             object symbol,
+             object exchange,
+             object start_date=None,
+             object asset_name=None,
+             int sid=0,
+             float leverage=1.0,
+             object end_daily=None,
+             object end_minute=None,
+             object end_date=None,
+             object exchange_symbol=None,
+             object first_traded=None,
+             object auto_close_date=None,
+             object exchange_full=None,
+             float min_trade_size=0.0001,
+             float max_trade_size=1000000,
+             float maker=0.0015,
+             float taker=0.0025,
+             float lot=0,
+             int decimals = 8,
+             int trading_state=1,
+             object data_source='catalyst',
         """
-        #TODO: missing fields
-        super_dict = super(TradingPair, self).to_dict()
-        super_dict['end_daily'] = self.end_daily
-        super_dict['end_minute'] = self.end_minute
-        super_dict['leverage'] = self.leverage
-        super_dict['min_trade_size'] = self.min_trade_size
-        return super_dict
+        trading_pair_dict = dict(
+            symbol=self.symbol,
+            exchange=self.exchange,
+            start_date=self.start_date,
+            asset_name=self.asset_name,
+            leverage=self.leverage,
+            end_daily=self.end_daily,
+            end_minute=self.end_minute,
+            end_date=self.end_date,
+            exchange_symbol=self.exchange_symbol,
+            exchange_full=self.exchange_full,
+            min_trade_size=self.min_trade_size,
+            max_trade_size=self.max_trade_size,
+            maker=self.maker,
+            taker=self.taker,
+            lot=self.lot,
+            decimals=self.decimals,
+            trading_state=self.trading_state,
+            data_source=self.data_source,
+        )
+        return trading_pair_dict
 
     def is_exchange_open(self, dt_minute):
         """
@@ -622,6 +660,16 @@ cdef class TradingPair(Asset):
         #TODO: make more dymanic to catch holds
         return True
 
+    def set_end_date(self, dt, data_frequency):
+        if data_frequency == 'minute':
+            self.end_minute = dt
+
+        else:
+            self.end_daily = dt
+
+    def set_start_date(self, dt):
+        self.start_date = dt
+
     cpdef __reduce__(self):
         """
         Function used by pickle to determine how to serialize/deserialize this
@@ -645,7 +693,9 @@ cdef class TradingPair(Asset):
                                  self.lot,
                                  self.decimals,
                                  self.taker,
-                                 self.maker))
+                                 self.maker,
+                                 self.trading_state,
+                                 self.data_source))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -11,8 +11,39 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
-
+EXCHANGE_CONFIG_URL = 'https://s3.amazonaws.com/enigmaco/ohlcv/' \
+                      '{exchange}/config.json'
+BUNDLE_URL = 'https://s3.amazonaws.com/enigmaco/ohlcv/' \
+              '{exchange}/{data_frequency}/{name}.tar.gz'
 DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
 DATE_FORMAT = '%Y-%m-%d'
 
+try:
+    ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+except Exception as e:
+    print('unable to get catalyst path: {}'.format(e))
+
 AUTO_INGEST = False
+
+AUTH_SERVER = 'https://data.enigma.co'
+
+# TODO: switch to mainnet
+ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
+
+
+MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
+                       'catalyst/master/catalyst/marketplace/' \
+                       'contract_marketplace_address.txt'
+
+MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
+                           'catalyst/master/catalyst/marketplace/' \
+                           'contract_marketplace_abi.json'
+
+# TODO: switch to mainnet
+ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
+                  'master/catalyst/marketplace/' \
+                  'contract_enigma_address.txt'
+
+ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
+                      'catalyst/master/catalyst/marketplace/' \
+                      'contract_enigma_abi.json'

catalyst/curate/poloniex.py

@@ -1,16 +1,16 @@
-import os
-import time
-import shutil
-import json
 import csv
+import json
+import os
+import shutil
+import time
 from datetime import datetime
 
+import logbook
 import pandas as pd
 import requests
-import logbook
-
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
+from catalyst.exchange.utils.exchange_utils import \
+    get_exchange_symbols_filename
 
 DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
 DT_END = pd.to_datetime('today').value // 10 ** 9
@@ -193,7 +193,8 @@ class PoloniexCurator(object):
                    for this currencyPair
         '''
         try:
-            if('end_file' in locals() and end_file + 3600 < end):
+            if(temp is not None
+                    or ('end_file' in locals() and end_file + 3600 < end)):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
@@ -261,7 +262,7 @@ class PoloniexCurator(object):
         vol = df['total'].to_frame('volume')           # set Vol aside
         df.drop('total', axis=1, inplace=True)         # Drop volume data
         ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
-        ohlc.cols = ohlc.cols.map(lambda t: t[1])      # Raname cols
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Rename cols
         closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
         ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
         vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin

catalyst/data/dispatch_bar_reader.py

@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         if self._last_available_dt is not None:
             return self._last_available_dt
         else:
-            return min(r.last_available_dt for r in self._readers.values())
+            return min(r.last_available_dt for r in list(self._readers.values()))
 
     @lazyval
     def first_trading_day(self):
-        return max(r.first_trading_day for r in self._readers.values())
+        return max(r.first_trading_day for r in list(self._readers.values()))
 
     def get_value(self, sid, dt, field):
         asset = self._asset_finder.retrieve_asset(sid)

catalyst/data/loader.py

@@ -22,6 +22,7 @@ from pandas_datareader.data import DataReader
 from six import iteritems
 from six.moves.urllib_error import HTTPError
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.utils.calendars import get_calendar
 from . import treasuries, treasuries_can
 from .benchmarks import get_benchmark_returns
@@ -31,8 +32,6 @@ from ..utils.paths import (
     data_root,
 )
 
-from catalyst.constants import LOG_LEVEL
-
 logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
@@ -102,7 +101,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         trading_day = get_calendar('OPEN').trading_day
 
     # TODO: consider making configurable
-    bm_symbol = 'btc_usdt'
+    bm_symbol = 'btc_usd'
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
@@ -143,10 +142,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.factory import get_exchange
+        from catalyst.exchange.utils.factory import get_exchange
         exchange = get_exchange(
-            exchange_name='poloniex', base_currency='usdt'
+            exchange_name='bitfinex', base_currency='usd'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 

catalyst/examples/arbitrage_with_interface.py

@@ -6,7 +6,7 @@ from catalyst.api import (
     symbol,
     get_open_orders
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'arbitrage_eth_btc'

catalyst/examples/buy_and_hodl.py

@@ -23,7 +23,7 @@ from catalyst.api import (order_target_value, symbol, record,
 
 
 def initialize(context):
-    context.ASSET_NAME = 'btc_usd'
+    context.ASSET_NAME = 'btc_usdt'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
@@ -140,9 +140,9 @@ if __name__ == '__main__':
         initialize=initialize,
         handle_data=handle_data,
         analyze=analyze,
-        exchange_name='bitfinex',
+        exchange_name='poloniex',
         algo_namespace='buy_and_hodl',
-        base_currency='usd',
+        base_currency='usdt',
         start=pd.to_datetime('2015-03-01', utc=True),
         end=pd.to_datetime('2017-10-31', utc=True),
     )

catalyst/examples/buy_btc_simple.py

@@ -27,7 +27,7 @@ import pandas as pd
 
 
 def initialize(context):
-    context.asset = symbol('btc_usd')
+    context.asset = symbol('btc_usdt')
 
 
 def handle_data(context, data):
@@ -41,9 +41,9 @@ if __name__ == '__main__':
         data_frequency='daily',
         initialize=initialize,
         handle_data=handle_data,
-        exchange_name='bitfinex',
+        exchange_name='poloniex',
         algo_namespace='buy_and_hodl',
-        base_currency='usd',
+        base_currency='usdt',
         start=pd.to_datetime('2015-03-01', utc=True),
         end=pd.to_datetime('2017-10-31', utc=True),
     )

catalyst/examples/buy_low_sell_high.py

@@ -1,85 +1,65 @@
-'''
-This algorithm requires an additional library (ta-lib) beyond those
-required by catalyst. Install it first by running:
-$ pip install TA-Lib
-
-If you get build errors like:
-    "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and it
-needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
-instructions on how to install the required dependencies.
-'''
-
 import talib
+import pandas as pd
 from logbook import Logger
 
-from catalyst import run_algorithm
 from catalyst.api import (
     order,
     order_target_percent,
     symbol,
     record,
-    get_open_orders,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
-import pandas as pd
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+from catalyst.utils.run_algo import run_algorithm
 
-algo_namespace = 'buy_low_sell_high_xrp'
-log = Logger(algo_namespace)
+algo_namespace = 'buy_the_dip_live'
+log = Logger('buy low sell high')
 
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USDT'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 5000
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.05
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.swallow_errors = True
+    context.SLIPPAGE_ALLOWED = 0.02
 
     context.errors = []
     pass
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1D'
     )
-
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
     elif rsi <= 70:
-        buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
     )
 
-    orders = get_open_orders(context.asset)
+    orders = context.blotter.open_orders
     if orders:
         log.info('skipping bar until all open orders execute')
         return
@@ -141,11 +121,11 @@ def _handle_data(context, data):
 
 def handle_data(context, data):
     log.info('handling bar {}'.format(data.current_dt))
-    try:
-        _handle_data(context, data)
-    except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
+    # try:
+    _handle_data(context, data)
+    # except Exception as e:
+    #     log.warn('aborting the bar on error {}'.format(e))
+    #     context.errors.append(e)
 
     log.info('completed bar {}, total execution errors {}'.format(
         data.current_dt,
@@ -162,15 +142,29 @@ def analyze(context, stats):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=10000,
-        data_frequency='daily',
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='poloniex',
-        algo_namespace='buy_and_hodl',
-        base_currency='usd',
-        start=pd.to_datetime('2015-03-01', utc=True),
-        end=pd.to_datetime('2017-10-31', utc=True),
-    )
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=1000,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bittrex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='buy_and_hodl',
+            base_currency='usdt',
+            start=pd.to_datetime('2015-03-01', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )

catalyst/examples/buy_low_sell_high_live.py

@@ -1,160 +0,0 @@
-import talib
-from logbook import Logger
-
-import pandas as pd
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_the_dip_live'
-log = Logger('buy low sell high')
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.ASSET_NAME = 'btc_usdt'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.TARGET_POSITIONS = 30
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='1D'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.2
-    else:
-        buy_increment = 0.1
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(
-        price=price,
-        rsi=rsi,
-    )
-
-    orders = get_open_orders(context.asset)
-    if orders:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif (position.amount > 0
-              and price > cost_basis * (1 + context.PROFIT_TARGET)):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            log.info('the rsi is too high to consider buying {}'.format(rsi))
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
-        )
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-    pass
-
-
-if __name__ == '__main__':
-    run_algorithm(
-        capital_base=0.001,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=analyze,
-        exchange_name='binance',
-        live=True,
-        algo_namespace=algo_namespace,
-        base_currency='btc',
-        simulate_orders=True,
-    )

catalyst/examples/dual_moving_average.py

@@ -1,12 +1,11 @@
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from logbook import Logger
-import matplotlib.pyplot as plt
 
 from catalyst import run_algorithm
-from catalyst.api import (record, symbol, order_target_percent,
-                          get_open_orders)
-from catalyst.exchange.stats_utils import extract_transactions
+from catalyst.api import (record, symbol, order_target_percent,)
+from catalyst.exchange.utils.stats_utils import extract_transactions
 
 NAMESPACE = 'dual_moving_average'
 log = Logger(NAMESPACE)
@@ -32,16 +31,18 @@ def handle_data(context, data):
     # moving average with the appropriate parameters. We choose to use
     # minute bars for this simulation -> freq="1m"
     # Returns a pandas dataframe.
-    short_mavg = data.history(context.asset,
+    short_data = data.history(context.asset,
                               'price',
                               bar_count=short_window,
-                              frequency="1m",
-                              ).mean()
-    long_mavg = data.history(context.asset,
+                              frequency="1T",
+                              )
+    short_mavg = short_data.mean()
+    long_data = data.history(context.asset,
                              'price',
                              bar_count=long_window,
-                             frequency="1m",
-                             ).mean()
+                             frequency="1T",
+                             )
+    long_mavg = long_data.mean()
 
     # Let's keep the price of our asset in a more handy variable
     price = data.current(context.asset, 'price')
@@ -61,7 +62,7 @@ def handle_data(context, data):
 
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.asset)
+    orders = context.blotter.open_orders
     if len(orders) > 0:
         return
 
@@ -82,9 +83,9 @@ def handle_data(context, data):
 
 
 def analyze(context, perf):
-
     # Get the base_currency that was passed as a parameter to the simulation
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    exchange = list(context.exchanges.values())[0]
+    base_currency = exchange.base_currency.upper()
 
     # First chart: Plot portfolio value using base_currency
     ax1 = plt.subplot(411)
@@ -92,7 +93,7 @@ def analyze(context, perf):
     ax1.legend_.remove()
     ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
     start, end = ax1.get_ylim()
-    ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     # Second chart: Plot asset price, moving averages and buys/sells
     ax2 = plt.subplot(412, sharex=ax1)
@@ -103,9 +104,9 @@ def analyze(context, perf):
     ax2.set_ylabel('{asset}\n({base})'.format(
         asset=context.asset.symbol,
         base=base_currency
-        ))
+    ))
     start, end = ax2.get_ylim()
-    ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     transaction_df = extract_transactions(perf)
     if not transaction_df.empty:
@@ -135,19 +136,20 @@ def analyze(context, perf):
     ax3.legend_.remove()
     ax3.set_ylabel('Percent Change')
     start, end = ax3.get_ylim()
-    ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+    ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
 
     # Fourth chart: Plot our cash
     ax4 = plt.subplot(414, sharex=ax1)
     perf.cash.plot(ax=ax4)
     ax4.set_ylabel('Cash\n({})'.format(base_currency))
     start, end = ax4.get_ylim()
-    ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+    ax4.yaxis.set_ticks(np.arange(0, end, end / 5))
 
     plt.show()
 
 
 if __name__ == '__main__':
+    
     run_algorithm(
             capital_base=1000,
             data_frequency='minute',

catalyst/examples/marketplace/github-research.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, get_dataset
+
+START = '2017-01-01'
+END = '2017-12-31'
+
+
+def initialize(context):
+    pass
+
+
+def handle_data(context, data):
+    context.github = get_dataset('github')
+    context.github.sort_index(level=0, inplace=True)
+
+    context.zec = data.history(symbol('zec_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+    context.xmr = data.history(symbol('xmr_usdt'),
+                               ['price', ],
+                               bar_count=365,
+                               frequency="1d")
+
+
+def analyze(context=None, results=None):
+    ax1 = plt.subplot(211)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'ZEC']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax1, color='blue')
+    ax1.legend(loc=2)
+    ax1.set_title('Zcash')
+    ax2 = ax1.twinx()
+    context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
+    ax2.legend(loc=1)
+
+    ax3 = plt.subplot(212)
+    idx = pd.IndexSlice
+    df = context.github.loc[START:END].loc[
+            idx[:, [b'XMR']], ['commits']].reset_index(
+                level='symbol', drop=True)
+    df.plot(ax=ax3, color='blue')
+    ax3.legend(loc=2)
+    ax3.set_title('Monero')
+    ax4 = ax3.twinx()
+    context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
+    ax4.legend(loc=1)
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='algo-github',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime(END, utc=True),
+            end=pd.to_datetime(END, utc=True),
+        )

catalyst/examples/marketplace/mean_reversion_by_marketcap.py

@@ -0,0 +1,237 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_dataset
+from catalyst.exchange.utils.stats_utils import set_print_settings, \
+    get_pretty_stats
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    df = get_dataset('testmarketcap2')  # type: pd.DataFrame
+
+    # Picking a specific date in our DataFrame
+    first_dt = df.index.get_level_values(0)[0]
+    # Since we use a MultiIndex with date / symbol, picking a date will
+    # result in a new DataFrame for the selected date with a single
+    # symbol index
+    df = df.xs(first_dt, level=0)
+    # Keep only the top coins by market cap
+    df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
+
+    set_print_settings()
+
+    df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
+    print('the marketplace data:\n{}'.format(df))
+
+    # Pick the 5 assets with the lowest market cap for trading
+    quote_currency = 'eth'
+    exchange = context.exchanges[next(iter(context.exchanges))]
+    symbols = [a.symbol for a in exchange.assets
+               if a.start_date < context.datetime]
+    context.assets = []
+    for currency, price in df['market_cap_usd'].iteritems():
+        if len(context.assets) >= 5:
+            break
+
+        s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
+        if s in symbols:
+            context.assets.append(symbol(s))
+
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = dict()
+        context.current_day = today
+
+    # Preparing dictionaries for asset-level data points
+    volumes = dict()
+    rsis = dict()
+    price_values = dict()
+    cash = context.portfolio.cash
+
+    for asset in context.assets:
+        # We're computing the volume-weighted-average-price of the security
+        # defined above, in the context.assets variable.  For this example,
+        #  we're using three bars on the 15 min bars.
+
+        # The frequency attribute determine the bar size. We use this
+        # convention for the frequency alias:
+        # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+        prices = data.history(
+            asset,
+            fields='close',
+            bar_count=50,
+            frequency=context.CANDLE_SIZE
+        )
+
+        # Ta-lib calculates various technical indicator based on price and
+        # volume arrays.
+
+        # In this example, we are comp
+        rsi = talib.RSI(prices.values, timeperiod=14)
+
+        # We need a variable for the current price of the security to compare
+        #  to the average. Since we are requesting two fields, data.current()
+        # returns a DataFrame with
+        current = data.current(asset, fields=['close', 'volume'])
+        price = current['close']
+
+        # If base_price is not set, we use the current value. This is the
+        # price at the first bar which we reference to calculate price_change.
+        # if asset not in context.base_price:
+        #     context.base_price[asset] = price
+        #
+        # base_price = context.base_price[asset]
+        # price_change = (price - base_price) / base_price
+
+        # Tracking the relevant data
+        volumes[asset] = current['volume']
+        rsis[asset] = rsi[-1]
+        price_values[asset] = price
+        # price_changes[asset] = price_change
+
+        # We are trying to avoid over-trading by limiting our trades to
+        # one per day.
+        if asset in context.traded_today:
+            continue
+
+        # Exit if we cannot trade
+        if not data.can_trade(asset):
+            continue
+
+        # Another powerful built-in feature of the Catalyst backtester is the
+        # portfolio object.  The portfolio object tracks your positions, cash,
+        # cost basis of specific holdings, and more.  In this line, we
+        # calculate how long or short our position is at this minute.
+        pos_amount = context.portfolio.positions[asset].amount
+
+        if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+            log.info(
+                '{}: buying - price: {}, rsi: {}'.format(
+                    data.current_dt, price, rsi[-1]
+                )
+            )
+            # Set a style for limit orders,
+            limit_price = price * 1.005
+            target = 1.0 / len(context.assets)
+            order_target_percent(
+                asset, target, limit_price=limit_price
+            )
+            context.traded_today[asset] = True
+
+        elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+            log.info(
+                '{}: selling - price: {}, rsi: {}'.format(
+                    data.current_dt, price, rsi[-1]
+                )
+            )
+            limit_price = price * 0.995
+            order_target_percent(
+                asset, 0, limit_price=limit_price
+            )
+            context.traded_today[asset] = True
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        current_price=price_values,
+        volume=volumes,
+        rsi=rsis,
+        cash=cash,
+    )
+
+
+def analyze(context=None, perf=None):
+    stats = get_pretty_stats(perf)
+    print('the algo stats:\n{}'.format(stats))
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=100,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-10-15', utc=True),
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/examples/mean_reversion_simple.py

@@ -12,8 +12,7 @@ from logbook import Logger
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.stats_utils import extract_transactions
-
+from catalyst.exchange.utils.stats_utils import extract_transactions
 # We give a name to the algorithm which Catalyst will use to persist its state.
 # In this example, Catalyst will create the `.catalyst/data/live_algos`
 # directory. If we stop and start the algorithm, Catalyst will resume its
@@ -34,18 +33,18 @@ def initialize(context):
     # parameters or values you're going to use.
 
     # In our example, we're looking at Neo in Ether.
-    context.market = symbol('neo_eth')
+    context.market = symbol('eth_btc')
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 30
-    context.RSI_OVERBOUGHT = 80
-    context.CANDLE_SIZE = '5T'
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '15T'
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.1, taker=0.2)
-    context.set_slippage(spread=0.0001)
+    context.set_commission(maker=0.001, taker=0.002)
+    context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -115,7 +114,7 @@ def handle_data(context, data):
     # TODO: retest with open orders
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.market)
+    orders = context.blotter.open_orders
     if len(orders) > 0:
         log.info('exiting because orders are open: {}'.format(orders))
         return
@@ -162,7 +161,7 @@ def analyze(context=None, perf=None):
 
     import matplotlib.pyplot as plt
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -245,9 +244,25 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    MODE = 'backtest'
+    live = True
 
-    if MODE == 'backtest':
+    if live:
+        run_algorithm(
+            capital_base=0.03,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+            # auth_aliases=dict(poloniex='auth2')
+        )
+
+    else:
         folder = os.path.join(
             tempfile.gettempdir(), 'catalyst', NAMESPACE
         )
@@ -266,24 +281,9 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out
         )
         log.info('saved perf stats: {}'.format(out))
-
-    elif MODE == 'live':
-        run_algorithm(
-            capital_base=0.05,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='binance',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            live_graph=False,
-            simulate_orders=True,
-            stats_output=None
-        )

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/examples/portfolio_optimization.py

@@ -66,7 +66,7 @@ def handle_data(context, data):
         # Define portfolio optimization parameters
         n_portfolios = 50000
         results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in xrange(n_portfolios):
+        for p in range(n_portfolios):
             weights = np.random.random(context.nassets)
             weights /= np.sum(weights)
             w = np.asmatrix(weights)
@@ -146,4 +146,5 @@ if __name__ == '__main__':
                             start=start,
                             end=end,
                             exchange_name='poloniex',
-                            capital_base=100000, )
+                            capital_base=100000,
+                            base_currency='usdt', )

catalyst/examples/rsi_profit_target.py

@@ -175,7 +175,7 @@ def handle_data(context, data):
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
 
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results.loc[:, 'portfolio_value'].plot(ax=ax1)

catalyst/examples/simple_loop.py

@@ -1,35 +1,38 @@
-import talib
 import pandas as pd
+import talib
+from logbook import Logger, INFO
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record
-from catalyst.exchange.stats_utils import get_pretty_stats, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
     extract_transactions
 
+log = Logger('simple_loop', level=INFO)
+
 
 def initialize(context):
-    print('initializing')
+    log.info('initializing')
     context.asset = symbol('eth_btc')
     context.base_price = None
 
 
 def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
+    log.info('handling bar: {}'.format(data.current_dt))
 
     price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
+    log.info('got price {price}'.format(price=price))
 
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='30T'
+        frequency='2H'
     )
     last_traded = prices.index[-1]
-    print('last candle date: {}'.format(last_traded))
+    log.info('last candle date: {}'.format(last_traded))
 
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    print('got rsi: {}'.format(rsi))
+    log.info('got rsi: {}'.format(rsi))
 
     # If base_price is not set, we use the current value. This is the
     # price at the first bar which we reference to calculate price_change.
@@ -51,10 +54,10 @@ def handle_data(context, data):
 
 def analyze(context, perf):
     import matplotlib.pyplot as plt
-    print('the stats: {}'.format(get_pretty_stats(perf)))
+    log.info('the stats: {}'.format(get_pretty_stats(perf)))
 
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -111,15 +114,31 @@ def analyze(context, perf):
 
 
 if __name__ == '__main__':
-    run_algorithm(
-        capital_base=1,
-        initialize=initialize,
-        handle_data=handle_data,
-        analyze=None,
-        exchange_name='poloniex',
-        live=True,
-        algo_namespace='simple_loop',
-        base_currency='eth',
-        live_graph=False,
-        simulate_orders=True
-    )
+    mode = 'live'
+
+    if mode == 'backtest':
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='poloniex',
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            data_frequency='minute',
+            start=pd.to_datetime('2017-9-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='binance',
+            live=True,
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True
+        )

catalyst/examples/simple_universe.py

@@ -35,14 +35,14 @@ import numpy as np
 import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.api import (symbols, )
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
 
 
 def initialize(context):
     context.i = -1  # minute counter
-    context.exchange = context.exchanges.values()[0].name.lower()
-    context.base_currency = context.exchanges.values()[0].base_currency.lower()
+    context.exchange = list(context.exchanges.values())[0].name.lower()
+    context.base_currency = list(context.exchanges.values())[0].base_currency.lower()
 
 
 def handle_data(context, data):
@@ -65,7 +65,7 @@ def handle_data(context, data):
     minutes = 30
 
     # get lookback_days of history data: that is 'lookback' number of bins
-    lookback = one_day_in_minutes / minutes * lookback_days
+    lookback = int(one_day_in_minutes / minutes * lookback_days)
     if not context.i % minutes and context.universe:
         # we iterate for every pair in the current universe
         for coin in context.coins:

catalyst/examples/talib_simple.py

@@ -23,7 +23,7 @@ from catalyst.api import (
     order_target_percent,
     symbol,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 
 algo_namespace = 'talib_sample'
 log = Logger(algo_namespace)

catalyst/exchange/asset_finder_exchange.py

@@ -1,99 +0,0 @@
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-
-log = Logger('AssetFinderExchange', level=LOG_LEVEL)
-
-
-class AssetFinderExchange(object):
-    def __init__(self):
-        self._asset_cache = {}
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        return list()
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        # for sid in sids:
-        #     if sid in self._asset_cache:
-        #         log.debug('got asset from cache: {}'.format(sid))
-        #     else:
-        #         log.debug('fetching asset: {}'.format(sid))
-        return list()
-
-    def lookup_symbol(self, symbol, exchange, data_frequency=None,
-                      as_of_date=None, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
-
-        if data_frequency is not None:
-            key = ','.join([exchange.name, symbol, data_frequency])
-
-        else:
-            key = ','.join([exchange.name, symbol])
-
-        if key in self._asset_cache:
-            return self._asset_cache[key]
-        else:
-            asset = exchange.get_asset(symbol, data_frequency)
-            self._asset_cache[key] = asset
-            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,709 +0,0 @@
-import base64
-import datetime
-import hashlib
-import hmac
-import json
-import re
-import time
-
-import numpy as np
-import pandas as pd
-import pytz
-import requests
-import six
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
-
-# Trying to account for REST api instability
-# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-from catalyst.utils.deprecate import deprecated
-
-requests.adapters.DEFAULT_RETRIES = 20
-
-BITFINEX_URL = 'https://api.bitfinex.com'
-
-log = Logger('Bitfinex', level=LOG_LEVEL)
-warning_logger = Logger('AlgoWarning')
-
-
-@deprecated
-class Bitfinex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.url = BITFINEX_URL
-        self.key = key
-        self.secret = secret.encode('UTF-8')
-        self.name = 'bitfinex'
-        self.color = 'green'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-
-        # The candle limit for each request
-        self.num_candles_limit = 1000
-
-        # Max is 90 but playing it safe
-        # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 80
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def _request(self, operation, data, version='v1'):
-        payload_object = {
-            'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 1000000),
-            # convert to string
-            'options': {}
-        }
-
-        if data is None:
-            payload_dict = payload_object
-        else:
-            payload_dict = payload_object.copy()
-            payload_dict.update(data)
-
-        payload_json = json.dumps(payload_dict)
-        if six.PY3:
-            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
-        else:
-            payload = base64.b64encode(payload_json)
-
-        m = hmac.new(self.secret, payload, hashlib.sha384)
-        m = m.hexdigest()
-
-        # headers
-        headers = {
-            'X-BFX-APIKEY': self.key,
-            'X-BFX-PAYLOAD': payload,
-            'X-BFX-SIGNATURE': m
-        }
-
-        if data is None:
-            request = requests.get(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ), data={},
-                headers=headers)
-        else:
-            request = requests.post(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ),
-                headers=headers)
-
-        return request
-
-    def _get_v2_symbol(self, asset):
-        pair = asset.symbol.split('_')
-        symbol = 't' + pair[0].upper() + pair[1].upper()
-        return symbol
-
-    def _get_v2_symbols(self, assets):
-        """
-        Workaround to support Bitfinex v2
-        TODO: Might require a separate asset dictionary
-
-        :param assets:
-        :return:
-        """
-
-        v2_symbols = []
-        for asset in assets:
-            v2_symbols.append(self._get_v2_symbol(asset))
-
-        return v2_symbols
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from a Bitfinex order dictionary
-        :param order_status:
-        :return: Order
-        """
-        if order_status['is_cancelled']:
-            status = ORDER_STATUS.CANCELLED
-        elif not order_status['is_live']:
-            log.info('found executed order {}'.format(order_status))
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['original_amount'])
-        filled = float(order_status['executed_amount'])
-
-        if order_status['side'] == 'sell':
-            amount = -amount
-            filled = -filled
-
-        price = float(order_status['price'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        if order_type.endswith('limit'):
-            limit_price = price
-        elif order_type.endswith('stop'):
-            stop_price = price
-
-        executed_price = float(order_status['avg_execution_price'])
-
-        # TODO: bitfinex does not specify comission.
-        # I could calculate it but not sure if it's worth it.
-        commission = None
-
-        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        date = pytz.utc.localize(date)
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['id']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        log.debug('retrieving wallets balances')
-        try:
-            self.ask_request()
-            response = self._request('balances', None)
-            balances = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['message'])
-            )
-
-        std_balances = dict()
-        for balance in balances:
-            currency = balance['currency'].lower()
-            std_balances[currency] = float(balance['available'])
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Bitfinex
-
-        :param data_frequency:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
-         '1M'
-        """
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
-                               '360T', '720T', '1D', '7D', '14D', '30D']
-        if freq not in allowed_frequencies:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
-        if freq_match:
-            number = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-            if unit == 'T':
-                if number in [60, 180, 360, 720]:
-                    number = number / 60
-                    converted_unit = 'h'
-                else:
-                    converted_unit = 'm'
-            else:
-                converted_unit = unit
-
-            frequency = '{}{}'.format(number, converted_unit)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-        for asset in asset_list:
-            symbol = self._get_v2_symbol(asset)
-            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
-                url=self.url,
-                frequency=frequency,
-                symbol=symbol
-            )
-
-            if bar_count:
-                is_list = True
-                url += '/hist?limit={}'.format(int(bar_count))
-
-                def get_ms(date):
-                    epoch = datetime.datetime.utcfromtimestamp(0)
-                    epoch = epoch.replace(tzinfo=pytz.UTC)
-
-                    return (date - epoch).total_seconds() * 1000.0
-
-                if start_dt is not None:
-                    start_ms = get_ms(start_dt)
-                    url += '&start={0:f}'.format(start_ms)
-
-                if end_dt is not None:
-                    end_ms = get_ms(end_dt)
-                    url += '&end={0:f}'.format(end_ms)
-
-            else:
-                is_list = False
-                url += '/last'
-
-            try:
-                self.ask_request()
-                response = requests.get(url)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response.content:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            candles = response.json()
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(
-                    candle[0] / 1000.0)
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-                ohlc = dict(
-                    open=np.float64(candle[1]),
-                    high=np.float64(candle[3]),
-                    low=np.float64(candle[4]),
-                    close=np.float64(candle[2]),
-                    volume=np.float64(candle[5]),
-                    price=np.float64(candle[2]),
-                    last_traded=last_traded
-                )
-                return ohlc
-
-            if is_list:
-                ohlc_bars = []
-                # We can to list candles from old to new
-                for candle in reversed(candles):
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-            else:
-                ohlc = ohlc_from_candle(candles)
-                ohlc_map[asset] = ohlc
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-        if isinstance(style, ExchangeLimitOrder) \
-                or isinstance(style, ExchangeStopLimitOrder):
-            price = style.get_limit_price(is_buy)
-            order_type = 'limit'
-
-        elif isinstance(style, ExchangeStopOrder):
-            price = style.get_stop_price(is_buy)
-            order_type = 'stop'
-
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-        req = dict(
-            symbol=exchange_symbol,
-            amount=str(float(abs(amount))),
-            price="{:.20f}".format(float(price)),
-            side='buy' if is_buy else 'sell',
-            type='exchange ' + order_type,  # TODO: support margin trades
-            exchange=self.name,
-            is_hidden=False,
-            is_postonly=False,
-            use_all_available=0,
-            ocoorder=False,
-            buy_price_oco=0,
-            sell_price_oco=0
-        )
-
-        date = pd.Timestamp.utcnow()
-        try:
-            self.ask_request()
-            response = self._request('order/new', req)
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='unable to create Bitfinex order {}'.format(
-                    order_status['message'])
-            )
-
-        order_id = str(order_status['id'])
-        order = Order(
-            dt=date,
-            asset=asset,
-            amount=amount,
-            stop=style.get_stop_price(is_buy),
-            limit=style.get_limit_price(is_buy),
-            id=order_id
-        )
-
-        return order
-
-    def get_open_orders(self, asset=None):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        try:
-            self.ask_request()
-            response = self._request('orders', None)
-            order_statuses = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_statuses:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        orders = []
-        for order_status in order_statuses:
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-        try:
-            self.ask_request()
-            response = self._request(
-                'order/status', {'order_id': int(order_id)})
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='Unable to retrieve order status: {}'.format(
-                    order_status['message'])
-            )
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-
-        try:
-            self.ask_request()
-            response = self._request('order/cancel', {'order_id': order_id})
-            status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self._get_v2_symbols(assets)
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            self.ask_request()
-            response = requests.get(
-                '{url}/v2/tickers?symbols={symbols}'.format(
-                    url=self.url,
-                    symbols=','.join(symbols),
-                )
-            )
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response.content:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response.content)
-            )
-
-        try:
-            tickers = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        ticks = dict()
-        for index, ticker in enumerate(tickers):
-            if not len(ticker) == 11:
-                raise ExchangeRequestError(
-                    error='Invalid ticker in response: {}'.format(ticker)
-                )
-
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker[1],
-                ask=ticker[3],
-                last_price=ticker[7],
-                low=ticker[10],
-                high=ticker[9],
-                volume=ticker[8],
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self._request('symbols', None)
-
-        for symbol in response.json():
-            if (source_dates):
-                start_date = self.get_symbol_start_date(symbol)
-            else:
-                try:
-                    start_date = cached_symbols[symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[symbol] = dict(
-                symbol=symbol[:-3] + '_' + symbol[-3:],
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-
-        print(symbol)
-        symbol_v2 = 't' + symbol.upper()
-
-        """
-            For each symbol we retrieve candles with Monhtly resolution
-            We get the first month, and query again with daily resolution
-            around that date, and we get the first date
-        """
-        url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
-            url=self.url,
-            symbol=symbol_v2
-        )
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        """
-            If we don't get any data back for our monthly-resolution query
-            it means that symbol started trading less than a month ago, so
-            arbitrarily set the ref. date to 15 days ago to be safe with
-            +/- 31 days
-        """
-        if (len(response.json())):
-            startmonth = response.json()[-1][0]
-        else:
-            startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
-
-        """
-            Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than
-            now: time.time()
-        """
-        url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
-               '&end={end}').format(
-            url=self.url,
-            symbol=symbol_v2,
-            start=startmonth - 3600 * 24 * 31 * 1000,
-            end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000)))
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d',
-                             time.gmtime(int(response.json()[-1][0] / 1000)))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        exchange_symbol = asset.exchange_symbol
-        try:
-            self.ask_request()
-            # TODO: implement limit
-            response = self._request(
-                'book/{}'.format(exchange_symbol), None)
-            data = response.json()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        # TODO: filter by type
-        result = dict()
-        for order_type in data:
-            result[order_type] = []
-
-            for entry in data[order_type]:
-                result[order_type].append(dict(
-                    rate=float(entry['price']),
-                    quantity=float(entry['amount'])
-                ))
-
-        return result

catalyst/exchange/bitfinex/symbols.json

@@ -1,127 +0,0 @@
-{
-  "neobtc": {
-    "symbol": "neo_btc",
-    "start_date": "2017-09-07",
-    "precision": 5
-  },
-  "neousd": {
-    "symbol": "neo_usd",
-    "start_date": "2017-09-07"
-  },
-  "neoeth": {
-    "symbol": "neo_eth",
-    "start_date": "2017-09-07"
-  },
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bchusd": {
-    "symbol": "bch_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2017-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcbtc": {
-    "symbol": "etc_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcusd": {
-    "symbol": "etc_usd",
-    "start_date": "2017-01-01"
-  },
-  "rrtusd": {
-    "symbol": "rrt_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtbtc": {
-    "symbol": "rrt_btc",
-    "start_date": "2010-01-01"
-  },
-  "zecusd": {
-    "symbol": "zec_usd",
-    "start_date": "2010-01-01"
-  },
-  "zecbtc": {
-    "symbol": "zec_btc",
-    "start_date": "2010-01-01"
-  },
-  "xmrusd": {
-    "symbol": "xmr_usd",
-    "start_date": "2010-01-01"
-  },
-  "xmrbtc": {
-    "symbol": "xmr_btc",
-    "start_date": "2010-01-01"
-  },
-  "dshusd": {
-    "symbol": "dsh_usd",
-    "start_date": "2010-01-01"
-  },
-  "dshbtc": {
-    "symbol": "dsh_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccbtc": {
-    "symbol": "bcc_btc",
-    "start_date": "2010-01-01"
-  },
-  "bcubtc": {
-    "symbol": "bcu_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccusd": {
-    "symbol": "bcc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bcuusd": {
-    "symbol": "bcu_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpusd": {
-    "symbol": "xrp_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpbtc": {
-    "symbol": "xrp_btc",
-    "start_date": "2010-01-01"
-  },
-  "iotusd": {
-    "symbol": "iot_usd",
-    "start_date": "2010-01-01"
-  },
-  "iotbtc": {
-    "symbol": "iot_btc",
-    "start_date": "2010-01-01"
-  },
-  "ioteth": {
-    "symbol": "iot_eth",
-    "start_date": "2010-01-01"
-  },
-  "eosusd": {
-    "symbol": "eos_usd",
-    "start_date": "2010-01-01"
-  },
-  "eosbtc": {
-    "symbol": "eos_btc",
-    "start_date": "2010-01-01"
-  },
-  "eoseth": {
-    "symbol": "eos_eth",
-    "start_date": "2010-01-01"
-  }
-}

catalyst/exchange/bittrex/bittrex.py

@@ -1,417 +0,0 @@
-import json
-import time
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from six.moves import urllib
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-    ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
-    CreateOrderError
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-
-# TODO: consider using this: https://github.com/mondeja/bittrex_v2
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Bittrex', level=LOG_LEVEL)
-
-URL2 = 'https://bittrex.com/Api/v2.0'
-
-
-@deprecated
-class Bittrex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret)
-        self.name = 'bittrex'
-        self.color = 'blue'
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-
-        self.num_candles_limit = 2000
-
-        # Not sure what the rate limit is but trying to play it safe
-        # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.minute_writer = None
-        self.minute_reader = None
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.bundle = ExchangeBundle(self.name)
-
-    @property
-    def account(self):
-        pass
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def get_balances(self):
-        balances = self.api.getbalances()
-        try:
-            log.debug('retrieving wallet balances')
-            self.ask_request()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        std_balances = dict()
-        try:
-            for balance in balances:
-                currency = balance['Currency'].lower()
-                std_balances[currency] = balance['Available']
-
-        except TypeError:
-            raise ExchangeRequestError(error=balances)
-
-        return std_balances
-
-    def create_order(self, asset, amount, is_buy, style):
-        log.info('creating {} order'.format('buy' if is_buy else 'sell'))
-        exchange_symbol = self.get_symbol(asset)
-
-        if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
-            if isinstance(style, StopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-            try:
-                self.ask_request()
-                if is_buy:
-                    order_status = self.api.buylimit(exchange_symbol, amount,
-                                                     price)
-                else:
-                    order_status = self.api.selllimit(exchange_symbol,
-                                                      abs(amount), price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'uuid' in order_status:
-                order_id = order_status['uuid']
-                order = Order(
-                    dt=pd.Timestamp.utcnow(),
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                if order_status == 'INSUFFICIENT_FUNDS':
-                    log.warn('not enough funds to create order')
-                    return None
-                elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
-                    log.warn('Your order is too small, order at least 50K'
-                             ' Satoshi')
-                    return None
-                else:
-                    raise CreateOrderError(
-                        exchange=self.name,
-                        error=order_status
-                    )
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset):
-        symbol = self.get_symbol(asset)
-        try:
-            self.ask_request()
-            open_orders = self.api.getopenorders(symbol)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        orders = list()
-        for order_status in open_orders:
-            order = self._create_order(order_status)
-            orders.append(order)
-
-        return orders
-
-    def _create_order(self, order_status):
-        log.info(
-            'creating catalyst order from Bittrex {}'.format(order_status))
-        if order_status['CancelInitiated']:
-            status = ORDER_STATUS.CANCELLED
-        elif order_status['Closed'] is not None:
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        date = pd.to_datetime(order_status['Opened'], utc=True)
-        amount = order_status['Quantity']
-        filled = amount - order_status['QuantityRemaining']
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['Exchange']],
-            amount=amount,
-            stop=None,  # Not yet supported by Bittrex
-            limit=order_status['Limit'],
-            filled=filled,
-            id=order_status['OrderUuid'],
-            commission=order_status['CommissionPaid']
-        )
-        order.status = status
-
-        executed_price = order_status['PricePerUnit']
-
-        return order, executed_price
-
-    def get_order(self, order_id):
-        log.info('retrieving order {}'.format(order_id))
-        try:
-            self.ask_request()
-            order_status = self.api.getorder(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if order_status is None:
-            raise OrderNotFound(order_id=order_id, exchange=self.name)
-
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-        log.info('cancelling order {}'.format(order_id))
-
-        try:
-            self.ask_request()
-            status = self.api.cancel(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Supported Intervals
-        -------------------
-        day, oneMin, fiveMin, thirtyMin, hour
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :param start_dt
-        :param end_dt
-        :return:
-        """
-
-        # TODO: this has no effect at the moment
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T':
-            frequency = 'oneMin'
-        elif freq == '5T':
-            frequency = 'fiveMin'
-        elif freq == '30T':
-            frequency = 'thirtyMin'
-        elif freq == '60T':
-            frequency = 'hour'
-        elif freq == '1D':
-            frequency = 'day'
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        for asset in asset_list:
-            end = int(time.mktime(end_dt.timetuple()))
-            url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_={end}'.format(
-                    url=URL2,
-                    symbol=self.get_symbol(asset),
-                    frequency=frequency,
-                    end=end, )
-
-            try:
-                data = json.loads(urllib.request.urlopen(url).read().decode())
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if data['message']:
-                raise ExchangeRequestError(
-                    error='Unable to fetch candles {}'.format(data['message'])
-                )
-
-            candles = data['result']
-
-            def ohlc_from_candle(candle):
-                ohlc = dict(
-                    open=candle['O'],
-                    high=candle['H'],
-                    low=candle['L'],
-                    close=candle['C'],
-                    volume=candle['V'],
-                    price=candle['C'],
-                    last_traded=pd.to_datetime(candle['T'], utc=True)
-                )
-                return ohlc
-
-            ordered_candles = list(reversed(candles))
-            ohlc_map = dict()
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
-            else:
-                # TODO: optimize
-                ohlc_bars = []
-                for candle in ordered_candles[:bar_count]:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def tickers(self, assets):
-        """
-        As of v1.1, Bittrex only allows one ticker at the time.
-        So we have to make multiple calls to fetch multiple assets.
-
-        :param assets:
-        :return:
-        """
-        log.info('retrieving tickers')
-
-        ticks = dict()
-        for asset in assets:
-            symbol = self.get_symbol(asset)
-            try:
-                self.ask_request()
-                ticker = self.api.getticker(symbol)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            # TODO: catch invalid ticker
-            ticks[asset] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker['Bid'],
-                ask=ticker['Ask'],
-                last_price=ticker['Last']
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def generate_symbols_json(self, filename=None):
-        symbol_map = {}
-
-        fn, r = download_exchange_symbols(self.name)
-        with open(fn) as data_file:
-            cached_symbols = json.load(data_file)
-
-        markets = self.api.getmarkets()
-        for market in markets:
-            exchange_symbol = market['MarketName']
-            symbol = '{market}_{base}'.format(
-                market=self.sanitize_curency_symbol(market['MarketCurrency']),
-                base=self.sanitize_curency_symbol(market['BaseCurrency'])
-            )
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=pd.to_datetime(market['Created'],
-                                          utc=True).strftime("%Y-%m-%d"),
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        if order_type == 'all':
-            order_type = 'both'
-        elif order_type == 'bid':
-            order_type = 'buy'
-        elif order_type == 'ask':
-            order_type = 'sell'
-        else:
-            raise ValueError('invalid type')
-
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(
-            market=exchange_symbol,
-            type=order_type,
-            depth=100
-        )
-
-        result = dict()
-        for exchange_type in data:
-            if exchange_type == 'buy':
-                order_type = 'bids'
-            elif exchange_type == 'sell':
-                order_type = 'asks'
-
-            result[order_type] = []
-            for entry in data[exchange_type]:
-                result[order_type].append(dict(
-                    rate=entry['Rate'],
-                    quantity=entry['Quantity']
-                ))
-
-        return result

catalyst/exchange/bittrex/bittrex_api.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-from six.moves import urllib
-
-urlopen = urllib.request.urlopen
-
-
-class Bittrex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-        self.public = ['getmarkets', 'getcurrencies', 'getticker',
-                       'getmarketsummaries', 'getmarketsummary',
-                       'getorderbook', 'getmarkethistory']
-        self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
-                       'cancel', 'getopenorders']
-        self.account = ['getbalances', 'getbalance', 'getdepositaddress',
-                        'withdraw', 'getorder', 'getorderhistory',
-                        'getwithdrawalhistory', 'getdeposithistory']
-
-    def query(self, method, values={}):
-        if method in self.public:
-            url = 'https://bittrex.com/api/v1.1/public/'
-        elif method in self.market:
-            url = 'https://bittrex.com/api/v1.1/market/'
-        elif method in self.account:
-            url = 'https://bittrex.com/api/v1.1/account/'
-        else:
-            return 'Something went wrong, sorry.'
-
-        url += method + '?' + urllib.parse.urlencode(values)
-
-        if method not in self.public:
-            url += '&apikey=' + self.key
-            url += '&nonce=' + str(int(time.time()))
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 url.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'apisign': signature}
-        else:
-            headers = {}
-
-        req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(
-            req, context=ssl._create_unverified_context()).read())
-
-        if response["result"]:
-            return response["result"]
-        else:
-            return response["message"]
-
-    def getmarkets(self):
-        return self.query('getmarkets')
-
-    def getcurrencies(self):
-        return self.query('getcurrencies')
-
-    def getticker(self, market):
-        return self.query('getticker', {'market': market})
-
-    def getmarketsummaries(self):
-        return self.query('getmarketsummaries')
-
-    def getmarketsummary(self, market):
-        return self.query('getmarketsummary', {'market': market})
-
-    def getorderbook(self, market, type, depth=20):
-        return self.query('getorderbook',
-                          {'market': market, 'type': type, 'depth': depth})
-
-    def getmarkethistory(self, market, count=20):
-        return self.query('getmarkethistory',
-                          {'market': market, 'count': count})
-
-    def buylimit(self, market, quantity, rate):
-        return self.query('buylimit', {'market': market, 'quantity': quantity,
-                                       'rate': rate})
-
-    def buymarket(self, market, quantity):
-        return self.query('buymarket',
-                          {'market': market, 'quantity': quantity})
-
-    def selllimit(self, market, quantity, rate):
-        return self.query('selllimit', {'market': market, 'quantity': quantity,
-                                        'rate': rate})
-
-    def sellmarket(self, market, quantity):
-        return self.query('sellmarket',
-                          {'market': market, 'quantity': quantity})
-
-    def cancel(self, uuid):
-        return self.query('cancel', {'uuid': uuid})
-
-    def getopenorders(self, market):
-        return self.query('getopenorders', {'market': market})
-
-    def getbalances(self):
-        return self.query('getbalances')
-
-    def getbalance(self, currency):
-        return self.query('getbalance', {'currency': currency})
-
-    def getdepositaddress(self, currency):
-        return self.query('getdepositaddress', {'currency': currency})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'quantity': quantity,
-                           'address': address})
-
-    def getorder(self, uuid):
-        return self.query('getorder', {'uuid': uuid})
-
-    def getorderhistory(self, market, count):
-        return self.query('getorderhistory',
-                          {'market': market, 'count': count})
-
-    def getwithdrawalhistory(self, currency, count):
-        return self.query('getwithdrawalhistory',
-                          {'currency': currency, 'count': count})
-
-    def getdeposithistory(self, currency, count):
-        return self.query('getdeposithistory',
-                          {'currency': currency, 'count': count})

catalyst/exchange/bittrex/extensions-example.py

@@ -1,7 +0,0 @@
-from catalyst.data.bundles import register
-from catalyst.exchange.exchange_bundle import exchange_bundle
-
-symbols = (
-    'neo_btc',
-)
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/exchange.py

@@ -9,15 +9,18 @@ from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
-from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError
-from catalyst.exchange.exchange_utils import get_exchange_symbols, \
-    get_frequency, resample_history_df
+    NoDataAvailableOnExchange, NoValueForField, \
+    NoCandlesReceivedFromExchange, \
+    TickerNotFoundError, NotEnoughCashError
+from catalyst.exchange.utils.datetime_utils import get_delta, \
+    get_periods_range, \
+    get_periods, get_start_dt, get_frequency
+from catalyst.exchange.utils.exchange_utils import \
+    resample_history_df, has_bundle
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -38,6 +41,8 @@ class Exchange:
         self.request_cpt = None
         self.bundle = ExchangeBundle(self.name)
 
+        self.low_balance_threshold = None
+
     @abstractproperty
     def account(self):
         pass
@@ -46,6 +51,9 @@ class Exchange:
     def time_skew(self):
         pass
 
+    def has_bundle(self, data_frequency):
+        return has_bundle(self.name, data_frequency)
+
     def is_open(self, dt):
         """
         Is the exchange open
@@ -148,7 +156,7 @@ class Exchange:
 
     def get_assets(self, symbols=None, data_frequency=None,
                    is_exchange_symbol=False,
-                   is_local=None):
+                   is_local=None, quote_currency=None):
         """
         The list of markets for the specified symbols.
 
@@ -172,6 +180,15 @@ class Exchange:
         if symbols is None:
             # Make a distinct list of all symbols
             symbols = list(set([asset.symbol for asset in self.assets]))
+            symbols.sort()
+
+            if quote_currency is not None:
+                for symbol in symbols[:]:
+                    suffix = '_{}'.format(quote_currency.lower())
+
+                    if not symbol.endswith(suffix):
+                        symbols.remove(symbol)
+
             is_exchange_symbol = False
 
         assets = []
@@ -220,11 +237,15 @@ class Exchange:
         """
         asset = None
 
+        # TODO: temp mapping, fix to use a single symbol convention
+        og_symbol = symbol
+        symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol
         log.debug(
             'searching assets for: {} {}'.format(
                 self.name, symbol
             )
         )
+        # TODO: simplify and loose the loop
         for a in self.assets:
             if asset is not None:
                 break
@@ -235,10 +256,11 @@ class Exchange:
 
             elif data_frequency is not None:
                 applies = (
-                        (
-                                data_frequency == 'minute' and a.end_minute is not None)
-                        or (
-                                data_frequency == 'daily' and a.end_daily is not None)
+                    (
+                        data_frequency == 'minute' and a.end_minute is not None
+                    ) or (
+                        data_frequency == 'daily' and a.end_daily is not None
+                    )
                 )
 
             else:
@@ -246,15 +268,24 @@ class Exchange:
 
             # The symbol provided may use the Catalyst or the exchange
             # convention
-            key = a.exchange_symbol if is_exchange_symbol else a.symbol
-            if not asset and key.lower() == symbol.lower() and applies:
-                asset = a
+            key = a.exchange_symbol if \
+                is_exchange_symbol else self.get_symbol(a)
+            if not asset and key.lower() == symbol.lower():
+                if applies:
+                    asset = a
+
+                else:
+                    raise NoDataAvailableOnExchange(
+                        symbol=key,
+                        exchange=self.name,
+                        data_frequency=data_frequency,
+                    )
 
         if asset is None:
             supported_symbols = sorted([a.symbol for a in self.assets])
 
             raise SymbolNotFoundOnExchange(
-                symbol=symbol,
+                symbol=og_symbol,
                 exchange=self.name.title(),
                 supported_symbols=supported_symbols
             )
@@ -262,35 +293,24 @@ class Exchange:
         log.debug('found asset: {}'.format(asset))
         return asset
 
-    def fetch_symbol_map(self, is_local=False):
-        index = 1 if is_local else 0
-        if self._symbol_maps[index] is not None:
-            return self._symbol_maps[index]
+    @abstractmethod
+    def init(self):
+        """
+        Load the asset list from the network.
 
-        else:
-            symbol_map = get_exchange_symbols(self.name, is_local)
-            self._symbol_maps[index] = symbol_map
-            return symbol_map
+        Returns
+        -------
+
+        """
 
     @abstractmethod
-    def load_assets(self, is_local=False):
+    def create_exchange_config(self):
         """
-        Populate the 'assets' attribute with a dictionary of Assets.
-        The key of the resulting dictionary is the exchange specific
-        currency pair symbol. The universal symbol is contained in the
-        'symbol' attribute of each asset.
-
-        Notes
-        -----
-        The sid of each asset is calculated based on a numeric hash of the
-        universal symbol. This simple approach avoids maintaining a mapping
-        of sids.
-
-        This method can be omerridden if an exchange offers equivalent data
-        via its api.
+        Fetch the exchange market data and generate a config object
+        Returns
+        -------
 
         """
-        pass
 
     def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
         """
@@ -377,6 +397,7 @@ class Exchange:
 
         return value
 
+    # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 data_frequency, field, previous_value=None):
         """
@@ -401,7 +422,7 @@ class Exchange:
         series = pd.Series(values, index=dates)
 
         periods = get_periods_range(
-            start_dt, end_dt, data_frequency
+            start_dt=start_dt, end_dt=end_dt, freq=data_frequency
         )
         # TODO: ensure that this working as expected, if not use fillna
         series = series.reindex(
@@ -463,47 +484,54 @@ class Exchange:
 
         """
         freq, candle_size, unit, data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
-        adj_bar_count = candle_size * bar_count
-
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
 
+        # we want to avoid receiving empty candles
+        # so we request more than needed
+        # TODO: consider defining a const per asset
+        # and/or some retry mechanism (in each iteration request more data)
+        requested_bar_count = bar_count + 30
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
-            bar_count=bar_count,
-            start_dt=start_dt if not is_current else None,
+            bar_count=requested_bar_count,
             end_dt=end_dt if not is_current else None,
         )
 
-        series = dict()
+        # candles sanity check - verify no empty candles were received:
         for asset in candles:
-            asset_series = self.get_series_from_candles(
-                candles=candles[asset],
-                start_dt=start_dt,
-                end_dt=end_dt,
-                data_frequency=frequency,
-                field=field,
-            )
-            if end_dt is not None:
-                delta = get_delta(candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_traded = asset_series.index[-1]
+            if not candles[asset]:
+                raise NoCandlesReceivedFromExchange(
+                    bar_count=requested_bar_count,
+                    end_dt=end_dt,
+                    asset=asset,
+                    exchange=self.name)
 
-                if last_traded < adj_end_dt:
-                    raise LastCandleTooEarlyError(
-                        last_traded=last_traded,
-                        end_dt=adj_end_dt,
-                        exchange=self.name,
-                    )
-            series[asset] = asset_series
+        series = get_candles_df(candles=candles,
+                                field=field,
+                                freq=frequency,
+                                bar_count=requested_bar_count,
+                                end_dt=end_dt)
+
+        # TODO: consider how to approach this edge case
+        # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
+        # Checking to make sure that the dates match
+        # delta = get_delta(delta_candle_size, data_frequency)
+        # adj_end_dt = end_dt - delta
+        # last_traded = asset_series.index[-1]
+        # if last_traded < adj_end_dt:
+        #    raise LastCandleTooEarlyError(
+        #        last_traded=last_traded,
+        #        end_dt=adj_end_dt,
+        #        exchange=self.name,
+        #    )
 
         df = pd.DataFrame(series)
         df.dropna(inplace=True)
 
-        return df
+        return df.tail(bar_count)
 
     def get_history_window_with_bundle(self,
                                        assets,
@@ -551,11 +579,12 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
+        # TODO: this function needs some work,
+        # we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
-
         try:
             series = self.bundle.get_history_window_series_and_load(
                 assets=assets,
@@ -582,15 +611,14 @@ class Exchange:
                 # The get_history method supports multiple asset
                 # Use the original frequency to let each api optimize
                 # the size of result sets
-                trailing_bar_count = get_periods(
+                trailing_bars = get_periods(
                     trailing_dt, end_dt, freq
                 )
                 candles = self.get_candles(
                     freq=freq,
                     assets=asset,
-                    bar_count=trailing_bar_count,
-                    start_dt=start_dt,
-                    end_dt=end_dt
+                    end_dt=end_dt,
+                    bar_count=trailing_bars if trailing_bars < 500 else 500,
                 )
 
                 last_value = series[asset].iloc(0) if asset in series \
@@ -619,46 +647,103 @@ class Exchange:
 
         return df
 
-    def calculate_totals(self, check_cash=False, positions=None):
+    def _check_low_balance(self, currency, balances, amount, open_orders=None):
+        free = balances[currency]['free'] if currency in balances else 0.0
+
+        if open_orders:
+            # TODO: make sure that this works
+            free += sum([order.amount for order in open_orders])
+
+        if free < amount:
+            return free, True
+
+        else:
+            return free, False
+
+    def sync_positions(self, positions, open_orders=None, cash=None,
+                       check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
+        Parameters
+        ----------
+        positions:
+            The positions to synchronize.
+
+        check_balances:
+            Check balances amounts against the exchange.
+
         """
-        log.debug('synchronizing portfolio with exchange {}'.format(self.name))
-
-        cash = None
-        if check_cash:
+        free_cash = 0.0
+        if check_balances:
+            log.debug('fetching {} balances'.format(self.name))
             balances = self.get_balances()
-
-            cash = balances[self.base_currency]['free'] \
-                if self.base_currency in balances else None
-
-            if cash is None:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=self.base_currency,
-                    exchange=self.name
+            log.debug(
+                'got free balances for {} currencies'.format(
+                    len(balances)
                 )
-            log.debug('found base currency balance: {}'.format(cash))
+            )
+            if cash is not None:
+                free_cash, is_lower = self._check_low_balance(
+                    currency=self.base_currency,
+                    balances=balances,
+                    amount=cash,
+                )
+                if is_lower and not open_orders:
+                    raise NotEnoughCashError(
+                        currency=self.base_currency,
+                        exchange=self.name,
+                        free=free_cash,
+                        cash=cash,
+                    )
 
         positions_value = 0.0
         if positions:
-            assets = set([position.asset for position in positions])
+            assets = list(set([position.asset for position in positions]))
             tickers = self.tickers(assets)
-            log.debug('got tickers for positions: {}'.format(tickers))
 
-            for asset in tickers:
+            for position in positions:
+                asset = position.asset
+                if asset not in tickers:
+                    raise TickerNotFoundError(
+                        symbol=asset.symbol,
+                        exchange=self.name,
+                    )
+
                 ticker = tickers[asset]
-                positions = [p for p in positions if p.asset == asset]
+                log.debug(
+                    'updating {symbol} position, last traded on {dt} for '
+                    '{price}{currency}'.format(
+                        symbol=asset.symbol,
+                        dt=ticker['last_traded'],
+                        price=ticker['last_price'],
+                        currency=asset.quote_currency,
+                    )
+                )
+                position.last_sale_price = ticker['last_price']
+                position.last_sale_date = ticker['last_traded']
 
-                for position in positions:
-                    position.last_sale_price = ticker['last_price']
-                    position.last_sale_date = ticker['last_traded']
+                positions_value += \
+                    position.amount * position.last_sale_price
 
-                    positions_value += \
-                        position.amount * position.last_sale_price
+                if check_balances:
+                    free, is_lower = self._check_low_balance(
+                        currency=asset.base_currency,
+                        balances=balances,
+                        amount=position.amount,
+                    )
 
-        return cash, positions_value
+                    if is_lower:
+                        log.warn(
+                            'detected lower balance for {} on {}: {} < {}, '
+                            'updating position amount'.format(
+                                asset.symbol, self.name, free, position.amount
+                            )
+                        )
+                        position.amount = free
+
+        return free_cash, positions_value
 
     def order(self, asset, amount, style):
         """Place an order.
@@ -816,7 +901,24 @@ class Exchange:
         pass
 
     @abstractmethod
-    def cancel_order(self, order_param, symbol_or_asset=None):
+    def process_order(self, order):
+        """
+        Similar to get_order but looks only for executed orders.
+
+        Parameters
+        ----------
+        order: Order
+
+        Returns
+        -------
+        float
+            Avg execution price
+
+        """
+
+    @abstractmethod
+    def cancel_order(self, order_param,
+                     symbol_or_asset=None, params={}):
         """Cancel an open order.
 
         Parameters
@@ -825,12 +927,12 @@ class Exchange:
             The order_id or order object to cancel.
         symbol_or_asset: str|TradingPair
             The catalyst symbol, some exchanges need this
+        params:
         """
         pass
 
     @abstractmethod
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
+    def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None):
         """
         Retrieve OHLCV candles for the given assets
 
@@ -870,13 +972,15 @@ class Exchange:
         pass
 
     @abc.abstractmethod
-    def tickers(self, assets):
+    def tickers(self, assets, on_ticker_error='raise'):
         """
         Retrieve current tick data for the given assets
 
         Parameters
         ----------
         assets: list[TradingPair]
+        on_ticker_error: str [raise|warn]
+            How to handle an error when retrieving a single ticker.
 
         Returns
         -------
@@ -895,7 +999,7 @@ class Exchange:
     @abc.abstractmethod
     def get_orderbook(self, asset, order_type, limit):
         """
-        Retrieve the the orderbook for the given trading pair.
+        Retrieve the orderbook for the given trading pair.
 
         Parameters
         ----------
@@ -909,3 +1013,20 @@ class Exchange:
         list[dict[str, float]
         """
         pass
+
+    @abc.abstractmethod
+    def get_trades(self, asset, my_trades, start_dt, limit):
+        """
+        Retrieve a list of trades.
+
+        Parameters
+        ----------
+        my_trades: bool
+            List only my trades.
+        start_dt
+        limit
+
+        Returns
+        -------
+
+        """

catalyst/exchange/exchange_algorithm.py

@@ -10,16 +10,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import pickle
 import signal
 import sys
 from datetime import timedelta
 from os import listdir
-from os.path import isfile, join
-from time import sleep
+from os.path import isfile, join, exists
 
 import logbook
 import pandas as pd
+from redo import retry
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
@@ -27,23 +28,26 @@ from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangePortfolioDataError,
-    OrderTypeNotSupported, )
+    OrderTypeNotSupported)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-from catalyst.exchange.exchange_utils import (
+from catalyst.exchange.live_graph_clock import LiveGraphClock
+from catalyst.exchange.simple_clock import SimpleClock
+from catalyst.exchange.utils.exchange_utils import (
     save_algo_object,
     get_algo_object,
     get_algo_folder,
     get_algo_df,
     save_algo_df,
+    clear_frame_stats_directory,
+    remove_old_files,
     group_assets_by_exchange, )
-from catalyst.exchange.live_graph_clock import LiveGraphClock
-from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
-    stats_to_algo_folder
+from catalyst.exchange.utils.stats_utils import \
+    get_pretty_stats, stats_to_s3, stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
+from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
+from catalyst.marketplace.marketplace import Marketplace
 from catalyst.utils.api_support import api_method
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
@@ -66,18 +70,34 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         self.current_day = None
 
-        if self.simulate_orders is None \
-                and self.sim_params.arena == 'backtest':
+        if self.simulate_orders is None and \
+                self.sim_params.arena == 'backtest':
             self.simulate_orders = True
 
+        # Operations with retry features
+        self.attempts = dict(
+            get_transactions_attempts=5,
+            order_attempts=5,
+            synchronize_portfolio_attempts=5,
+            get_order_attempts=5,
+            get_open_orders_attempts=5,
+            cancel_order_attempts=5,
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
+
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
             cancel_policy=self.cancel_policy,
             simulate_orders=self.simulate_orders,
-            exchanges=self.exchanges
+            exchanges=self.exchanges,
+            attempts=self.attempts,
         )
 
+        self._marketplace = None
+
     @staticmethod
     def __convert_order_params_for_blotter(limit_price, stop_price, style):
         """
@@ -115,7 +135,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_commission(self, maker=None, taker=None):
-        key = self.blotter.commission_models.keys()[0]
+        key = list(self.blotter.commission_models.keys())[0]
         if maker is not None:
             self.blotter.commission_models[key].maker = maker
 
@@ -124,7 +144,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
     @api_method
     def set_slippage(self, spread=None):
-        key = self.blotter.slippage_models.keys()[0]
+        key = list(self.blotter.slippage_models.keys())[0]
         if spread is not None:
             self.blotter.slippage_models[key].spread = spread
 
@@ -144,6 +164,25 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                                                         style)
         return amount, style
 
+    def _calculate_order_target_amount(self, asset, target):
+        """
+        removes order amounts so we won't run into issues
+        when two orders are placed one after the other.
+        it then proceeds to removing positions amount at TradingAlgorithm
+        :param asset:
+        :param target:
+        :return: target
+        """
+        if asset in self.blotter.open_orders:
+            for open_order in self.blotter.open_orders[asset]:
+                current_amount = open_order.amount
+                target -= current_amount
+
+        target = super(ExchangeTradingAlgorithmBase, self). \
+            _calculate_order_target_amount(asset, target)
+
+        return target
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies
@@ -153,6 +192,15 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         """
         return round_nearest(amount, asset.min_trade_size)
 
+    @api_method
+    def get_dataset(self, data_source_name, start=None, end=None):
+        if self._marketplace is None:
+            self._marketplace = Marketplace()
+
+        return self._marketplace.get_dataset(
+            data_source_name, start, end,
+        )
+
     @api_method
     @preprocess(symbol_str=ensure_upper_case)
     def symbol(self, symbol_str, exchange_name=None):
@@ -218,28 +266,28 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         """
         tracker = self.perf_tracker
-        period = tracker.todays_performance
+        cum = tracker.cumulative_performance
 
-        pos_stats = period.position_tracker.stats()
-        period_stats = calc_period_stats(pos_stats, period.ending_cash)
+        pos_stats = cum.position_tracker.stats()
+        period_stats = calc_period_stats(pos_stats, cum.ending_cash)
 
         stats = dict(
             period_start=tracker.period_start,
             period_end=tracker.period_end,
             capital_base=tracker.capital_base,
             progress=tracker.progress,
-            ending_value=period.ending_value,
-            ending_exposure=period.ending_exposure,
-            capital_used=period.cash_flow,
-            starting_value=period.starting_value,
-            starting_exposure=period.starting_exposure,
-            starting_cash=period.starting_cash,
-            ending_cash=period.ending_cash,
-            portfolio_value=period.ending_cash + period.ending_value,
-            pnl=period.pnl,
-            returns=period.returns,
-            period_open=period.period_open,
-            period_close=period.period_close,
+            ending_value=cum.ending_value,
+            ending_exposure=cum.ending_exposure,
+            capital_used=cum.cash_flow,
+            starting_value=cum.starting_value,
+            starting_exposure=cum.starting_exposure,
+            starting_cash=cum.starting_cash,
+            ending_cash=cum.ending_cash,
+            portfolio_value=cum.ending_cash + cum.ending_value,
+            pnl=cum.pnl,
+            returns=cum.returns,
+            period_open=start_dt,
+            period_close=end_dt,
             gross_leverage=period_stats.gross_leverage,
             net_leverage=period_stats.net_leverage,
             short_exposure=pos_stats.short_exposure,
@@ -256,6 +304,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
+        period = tracker.todays_performance
         stats['positions'] = period.position_tracker.get_positions_list()
 
         # we want the key to be absent, not just empty
@@ -276,12 +325,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         return stats
 
+    def run(self, data=None, overwrite_sim_params=True):
+        data.attempts = self.attempts
+        return super(ExchangeTradingAlgorithmBase, self).run(
+            data, overwrite_sim_params
+        )
+
 
 class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
         self.frame_stats = list()
+        self.state = {}
         log.info('initialized trading algorithm in backtest mode')
 
     def is_last_frame_of_day(self, data):
@@ -328,33 +384,103 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
         self.stats_output = kwargs.pop('stats_output', None)
+        self._analyze_live = kwargs.pop('analyze_live', None)
+        self.end = kwargs.pop('end', None)
 
         self._clock = None
         self.frame_stats = list()
 
-        self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
+        # erase the frame_stats folder to avoid overloading the disk
+        error = clear_frame_stats_directory(self.algo_namespace)
+        if error:
+            log.warning(error)
 
-        self.custom_signals_stats = \
-            get_algo_df(self.algo_namespace, 'custom_signals_stats')
+        # in order to save paper & live files separately
+        self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live'
 
-        self.exposure_stats = \
-            get_algo_df(self.algo_namespace, 'exposure_stats')
+        self.pnl_stats = get_algo_df(
+                self.algo_namespace,
+                'pnl_stats_{}'.format(self.mode_name),
+            )
+
+        self.custom_signals_stats = get_algo_df(
+                self.algo_namespace,
+                'custom_signals_stats_{}'.format(self.mode_name)
+            )
+
+        self.exposure_stats = get_algo_df(
+                self.algo_namespace,
+                'exposure_stats_{}'.format(self.mode_name)
+            )
 
         self.is_running = True
 
-        self.retry_check_open_orders = 5
-        self.retry_synchronize_portfolio = 5
-        self.retry_get_open_orders = 5
-        self.retry_order = 2
-        self.retry_delay = 5
+        self.stats_minutes = 1
 
-        self.stats_minutes = 10
+        self._last_orders = []
+        self._last_open_orders = []
+        self.trading_client = None
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
-        signal.signal(signal.SIGINT, self.signal_handler)
+        try:
+            signal.signal(signal.SIGINT, self.signal_handler)
+        except ValueError:
+            log.warn("Can't initialize signal handler inside another thread."
+                     "Exit should be handled by the user.")
 
-        log.info('initialized trading algorithm in live mode')
+    def interrupt_algorithm(self):
+        """
+
+        when algorithm comes to an end this function is called.
+        extracts the stats and calls analyze.
+        after finishing, it exits the run.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
+        self.is_running = False
+
+        if self._analyze is None:
+            log.info('Exiting the algorithm.')
+
+        else:
+            log.info('Exiting the algorithm. Calling `analyze()` '
+                     'before exiting the algorithm.')
+
+            # add the last day stats which is not saved in the directory
+            current_stats = pd.DataFrame(self.frame_stats)
+            current_stats.set_index('period_close', drop=False, inplace=True)
+
+            # get the location of the directory
+            algo_folder = get_algo_folder(self.algo_namespace)
+            folder = join(algo_folder, 'frame_stats')
+
+            if exists(folder):
+                files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+                period_stats_list = []
+                for item in files:
+                    filename = join(folder, item)
+
+                    with open(filename, 'rb') as handle:
+                        perf_period = pickle.load(handle)
+                        period_stats_list.extend(perf_period)
+
+                stats = pd.DataFrame(period_stats_list)
+                stats.set_index('period_close', drop=False, inplace=True)
+
+                stats = pd.concat([stats, current_stats])
+            else:
+                stats = current_stats
+
+            self.analyze(stats)
+
+        sys.exit(0)
 
     def signal_handler(self, signal, frame):
         """
@@ -369,31 +495,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         -------
 
         """
-        self.is_running = False
-
-        if self._analyze is None:
-            log.info('Interruption signal detected {}, exiting the '
-                     'algorithm'.format(signal))
-
-        else:
-            log.info('Interruption signal detected {}, calling `analyze()` '
-                     'before exiting the algorithm'.format(signal))
-
-            algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
-
-            daily_perf_list = []
-            for item in files:
-                filename = join(folder, item)
-                with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
-
-            stats = pd.DataFrame(daily_perf_list)
-
-            self.analyze(stats)
-
-        sys.exit(0)
+        log.info('Interruption signal detected {}, exiting the '
+                 'algorithm'.format(signal))
+        self.interrupt_algorithm()
 
     @property
     def clock(self):
@@ -419,10 +523,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
-        if self.live_graph:
+        if self.live_graph or self._analyze_live is not None:
             self._clock = LiveGraphClock(
                 self.sim_params.sessions,
-                context=self
+                context=self,
+                callback=self._analyze_live,
             )
         else:
             self._clock = SimpleClock(
@@ -431,25 +536,82 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def _create_generator(self, sim_params):
-        if self.perf_tracker is None:
-            self.perf_tracker = get_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker'
-            )
+    def _init_trading_client(self):
+        """
+        This replaces Ziplines `_create_generator` method. The main difference
+        is that we are restoring performance tracker objects if available.
+        This allows us to stop/start algos without loosing their state.
 
-        # Call the simulation trading algorithm for side-effects:
-        # it creates the perf tracker
-        TradingAlgorithm._create_generator(self, sim_params)
-        self.trading_client = ExchangeAlgorithmExecutor(
-            self,
-            sim_params,
-            self.data_portal,
-            self.clock,
-            self._create_benchmark_source(),
-            self.restrictions,
-            universe_func=self._calculate_universe
+        """
+        self.state = get_algo_object(
+            algo_name=self.algo_namespace,
+            key='context.state_{}'.format(self.mode_name),
         )
+        if self.state is None:
+            self.state = {}
+
+        if self.perf_tracker is None:
+            # Note from the Zipline dev:
+            # HACK: When running with the `run` method, we set perf_tracker to
+            # None so that it will be overwritten here.
+            tracker = self.perf_tracker = PerformanceTracker(
+                sim_params=self.sim_params,
+                trading_calendar=self.trading_calendar,
+                env=self.trading_environment,
+            )
+            # Set the dt initially to the period start by forcing it to change.
+            self.on_dt_changed(self.sim_params.start_session)
+
+            new_position_tracker = tracker.position_tracker
+            tracker.position_tracker = None
+
+            # Unpacking the perf_tracker and positions if available
+            cum_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key='cumulative_performance_{}'.format(self.mode_name),
+            )
+            if cum_perf is not None:
+                tracker.cumulative_performance = cum_perf
+                # Ensure single common position tracker
+                tracker.position_tracker = cum_perf.position_tracker
+
+            today = pd.Timestamp.utcnow().floor('1D')
+            todays_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                rel_path='daily_performance_{}'.format(self.mode_name),
+            )
+            if todays_perf is not None:
+                # Ensure single common position tracker
+                if tracker.position_tracker is not None:
+                    todays_perf.position_tracker = tracker.position_tracker
+                else:
+                    tracker.position_tracker = todays_perf.position_tracker
+
+                tracker.todays_performance = todays_perf
+
+            if tracker.position_tracker is None:
+                # Use a new position_tracker if not is found in the state
+                tracker.position_tracker = new_position_tracker
+
+        if not self.initialized:
+            # Calls the initialize function of the algorithm
+            self.initialize(*self.initialize_args, **self.initialize_kwargs)
+            self.initialized = True
+
+        self.trading_client = ExchangeAlgorithmExecutor(
+            algo=self,
+            sim_params=self.sim_params,
+            data_portal=self.data_portal,
+            clock=self.clock,
+            benchmark_source=self._create_benchmark_source(),
+            restrictions=self.restrictions,
+            universe_func=self._calculate_universe,
+        )
+
+    def get_generator(self):
+        if self.trading_client is None:
+            self._init_trading_client()
 
         return self.trading_client.transform()
 
@@ -459,7 +621,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self):
         """
         Synchronizes the portfolio tracked by the algorithm to refresh
         its current value.
@@ -468,10 +630,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         positions, returning the available cash, and raising error
         if the data goes out of sync.
 
-        Parameters
-        ----------
-        attempt_index: int
-
         Returns
         -------
         float
@@ -481,63 +639,59 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The total value of all tracked positions.
 
         """
+        check_balances = (not self.simulate_orders)
+        base_currency = None
         tracker = self.perf_tracker.position_tracker
         total_cash = 0.0
         total_positions_value = 0.0
 
-        try:
-            # Position keys correspond to assets
-            positions = self.portfolio.positions
-            assets = list(positions)
-            exchange_assets = group_assets_by_exchange(assets)
-            for exchange_name in self.exchanges:
-                assets = exchange_assets[exchange_name] \
-                    if exchange_name in exchange_assets else []
+        # Position keys correspond to assets
+        positions = self.portfolio.positions
+        assets = list(positions)
+        exchange_assets = group_assets_by_exchange(assets)
+        for exchange_name in self.exchanges:
+            assets = exchange_assets[exchange_name] \
+                if exchange_name in exchange_assets else []
 
-                exchange_positions = \
-                    [positions[asset] for asset in assets]
-
-                check_cash = (not self.simulate_orders)
-
-                exchange = self.exchanges[exchange_name]  # Type: Exchange
-                cash, positions_value = exchange.calculate_totals(
-                    positions=exchange_positions,
-                    check_cash=check_cash,
-                )
-                total_positions_value += positions_value
-
-                if cash is not None:
-                    total_cash += cash
-
-                for position in exchange_positions:
-                    tracker.update_position(
-                        asset=position.asset,
-                        last_sale_date=position.last_sale_date,
-                        last_sale_price=position.last_sale_price
-                    )
-
-            if cash is None:
-                total_cash = self.portfolio.cash
-
-            elif total_cash < self.portfolio.cash:
-                raise ValueError('Cash on exchanges is lower than the algo.')
-
-            return total_cash, total_positions_value
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'update portfolio attempt {}: {}'.format(attempt_index, e)
+            exchange_positions = copy.deepcopy(
+                [positions[asset] for asset in assets if asset in positions]
             )
-            if attempt_index < self.retry_synchronize_portfolio:
-                sleep(self.retry_delay)
-                return self.synchronize_portfolio(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='update-portfolio',
-                    attempts=attempt_index,
-                    error=e
+
+            exchange = self.exchanges[exchange_name]  # Type: Exchange
+
+            if base_currency is None:
+                base_currency = exchange.base_currency
+
+            orders = []
+            for asset in self.blotter.open_orders:
+                asset_orders = self.blotter.open_orders[asset]
+                if asset_orders:
+                    orders += asset_orders
+
+            required_cash = self.portfolio.cash if not orders else None
+            cash, positions_value = exchange.sync_positions(
+                positions=exchange_positions,
+                open_orders=orders,
+                check_balances=check_balances,
+                cash=required_cash,
+            )
+            total_cash += cash
+            total_positions_value += positions_value
+
+            # Applying modifications to the original positions
+            for position in exchange_positions:
+                tracker.update_position(
+                    asset=position.asset,
+                    amount=position.amount,
+                    last_sale_date=position.last_sale_date,
+                    last_sale_price=position.last_sale_price,
                 )
 
+        if not check_balances:
+            total_cash = self.portfolio.cash
+
+        return total_cash, total_positions_value
+
     def add_pnl_stats(self, period_stats):
         """
         Save p&l stats.
@@ -563,7 +717,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.pnl_stats = pd.concat([self.pnl_stats, df])
 
-        save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
+        save_algo_df(
+            self.algo_namespace,
+            'pnl_stats_{}'.format(self.mode_name),
+            self.pnl_stats,
+        )
 
     def add_custom_signals_stats(self, period_stats):
         """
@@ -584,8 +742,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.custom_signals_stats = pd.concat([self.custom_signals_stats, df])
 
-        save_algo_df(self.algo_namespace, 'custom_signals_stats',
-                     self.custom_signals_stats)
+        save_algo_df(
+            self.algo_namespace,
+            'custom_signals_stats_{}'.format(self.mode_name),
+            self.custom_signals_stats,
+        )
 
     def add_exposure_stats(self, period_stats):
         """
@@ -612,9 +773,43 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.exposure_stats = pd.concat([self.exposure_stats, df])
 
         save_algo_df(
-            self.algo_namespace, 'exposure_stats', self.exposure_stats
+            self.algo_namespace,
+            'exposure_stats_{}'.format(self.mode_name),
+            self.exposure_stats
         )
 
+    def nullify_frame_stats(self, now):
+        """
+
+        Save all period_stats to local directory
+        erase old files from the folder and nullify
+        self.frame_stats
+
+        Parameters
+        ----------
+        now: Timestamp
+
+        Returns
+        -------
+
+        """
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=now.floor('1D').strftime('%Y-%m-%d'),
+            obj=self.frame_stats,
+            rel_path='frame_stats'
+        )
+
+        error = remove_old_files(
+            algo_name=self.algo_namespace,
+            today=now,
+            rel_path='frame_stats'
+        )
+        if error:
+            log.warning(error)
+
+        self.frame_stats = list()
+
     def handle_data(self, data):
         """
         Wrapper around the handle_data method of each algo.
@@ -627,20 +822,44 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if not self.is_running:
             return
 
+        if self.end is not None and self.end < data.current_dt:
+            log.info('Algorithm has reached specified end time. Finishing...')
+            self.interrupt_algorithm()
+
         # Resetting the frame stats every day to minimize memory footprint
         today = data.current_dt.floor('1D')
         if self.current_day is not None and today > self.current_day:
-            self.frame_stats = list()
+            self.nullify_frame_stats(now=data.current_dt)
 
-        new_transactions, new_commissions, closed_orders = \
-            self.blotter.get_transactions(data)
+        self.performance_needs_update = False
+        last_orders_list = list(self.blotter.orders.keys())
+        open_orders_list = list(self.blotter.open_orders.keys())
 
-        if len(new_transactions) > 0:
+        if last_orders_list != self._last_orders or \
+                open_orders_list != self._last_open_orders:
+            self.performance_needs_update = True
+
+        # Saving current order positions
+        # to detect changes in the next frame
+        self._last_orders = copy.deepcopy(last_orders_list)
+        self._last_open_orders = copy.deepcopy(open_orders_list)
+
+        if self.performance_needs_update:
             self.perf_tracker.update_performance()
+            self.performance_needs_update = False
+
+        if self.portfolio_needs_update:
+            cash, positions_value = retry(
+                action=self.synchronize_portfolio,
+                attempts=self.attempts['synchronize_portfolio_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.')
+            )
+            self.portfolio_needs_update = False
 
-        cash, positions_value = self.synchronize_portfolio()
         log.info(
-            'got totals from exchanges, cash: {} positions: {}'.format(
+            'portfolio balances, cash: {}, positions: {}'.format(
                 cash, positions_value
             )
         )
@@ -652,22 +871,34 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        self._save_algo_state(data)
+        self.current_day = data.current_dt.floor('1D')
+
+    def _save_algo_state(self, data):
+        today = data.current_dt.floor('1D')
         try:
             self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
-        # TODO: pickle does not seem to work in python 3
-        try:
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker',
-                obj=self.perf_tracker
-            )
-        except Exception as e:
-            log.warn('unable to save minute perfs to disk: {}'.format(e))
-
-        self.current_day = data.current_dt.floor('1D')
+        log.debug('saving cumulative performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='cumulative_performance_{}'.format(self.mode_name),
+            obj=self.perf_tracker.cumulative_performance,
+        )
+        log.debug('saving todays performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=self.perf_tracker.todays_performance,
+            rel_path='daily_performance_{}'.format(self.mode_name)
+        )
+        log.debug('saving context.state object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='context.state_{}'.format(self.mode_name),
+            obj=self.state)
 
     def _process_stats(self, data):
         today = data.current_dt.floor('1D')
@@ -677,11 +908,14 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.perf_tracker.update_performance()
 
         frame_stats = self.prepare_period_stats(
-            data.current_dt, data.current_dt + timedelta(minutes=1))
+            data.current_dt, data.current_dt + timedelta(minutes=1)
+        )
 
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
 
+        # creating and saving the pnl_stats into the local
+        # directory
         self.add_pnl_stats(frame_stats)
         if self.recorded_vars:
             self.add_custom_signals_stats(frame_stats)
@@ -699,7 +933,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 stats=get_pretty_stats(
                     stats=self.frame_stats,
                     recorded_cols=recorded_cols,
-                    num_rows=self.stats_minutes
+                    num_rows=self.stats_minutes,
                 )
             ))
 
@@ -709,12 +943,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             start_dt=today,
             end_dt=data.current_dt
         )
-        save_algo_object(
-            algo_name=self.algo_namespace,
-            key=today.strftime('%Y-%m-%d'),
-            obj=daily_stats,
-            rel_path='daily_perf'
-        )
 
         return recorded_cols
 
@@ -725,6 +953,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             csv_bytes = stats_to_algo_folder(
                 stats=self.frame_stats,
                 algo_namespace=self.algo_namespace,
+                folder_name='stats_{}'.format(self.mode_name),
                 recorded_cols=recorded_cols,
             )
         except Exception as e:
@@ -751,33 +980,26 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def batch_market_order(self, share_counts):
         raise NotImplementedError()
 
-    def _get_open_orders(self, asset=None, attempt_index=0):
-        try:
-            if asset:
-                exchange = self.exchanges[asset.exchange]
-                return exchange.get_open_orders(asset)
-
-            else:
-                open_orders = []
-                for exchange_name in self.exchanges:
-                    exchange = self.exchanges[exchange_name]
-                    exchange_orders = exchange.get_open_orders()
-                    open_orders.append(exchange_orders)
-
-                return open_orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'open orders attempt {}: {}'.format(attempt_index, e)
+    def _get_open_orders(self, asset=None):
+        if self.simulate_orders:
+            raise ValueError(
+                'The get_open_orders() method only works in live mode. '
+                'The purpose is to list open orders on the exchange '
+                'regardless who placed them. To list the open orders of '
+                'this algo, use `context.blotter.open_orders`.'
             )
-            if attempt_index < self.retry_get_open_orders:
-                sleep(self.retry_delay)
-                return self._get_open_orders(asset, attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='open-orders',
-                    attempts=attempt_index,
-                    error=e
-                )
+        if asset:
+            exchange = self.exchanges[asset.exchange]
+            return exchange.get_open_orders(asset)
+
+        else:
+            open_orders = []
+            for exchange_name in self.exchanges:
+                exchange = self.exchanges[exchange_name]
+                exchange_orders = exchange.get_open_orders()
+                open_orders.append(exchange_orders)
+
+            return open_orders
 
     @error_keywords(sid='Keyword argument `sid` is no longer supported for '
                         'get_open_orders. Use `asset` instead.')
@@ -799,7 +1021,15 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             If an asset is passed then this will return a list of the open
             orders for this asset.
         """
-        return self._get_open_orders(asset)
+        # TODO: should this be a shortcut to the open orders in the blotter?
+        return retry(
+            action=self._get_open_orders,
+            attempts=self.attempts['get_open_orders_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching open orders again.'),
+            args=(asset,)
+        )
 
     @api_method
     def get_order(self, order_id, exchange_name):
@@ -819,16 +1049,28 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The execution price per share of the order
         """
         exchange = self.exchanges[exchange_name]
-        return exchange.get_order(order_id)
+        return retry(
+            action=exchange.get_order,
+            attempts=self.attempts['get_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching orders again.'),
+            args=(order_id,))
 
     @api_method
-    def cancel_order(self, order_param, exchange_name):
+    def cancel_order(self, order_param, exchange_name,
+                     symbol=None, params={}):
         """Cancel an open order.
 
         Parameters
         ----------
         order_param : str or Order
             The order_id or order object to cancel.
+
+        exchange_name: name of exchange from
+                        which you want to cancel the order
+        symbol:
+        params:
         """
         exchange = self.exchanges[exchange_name]
 
@@ -836,4 +1078,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if isinstance(order_param, zp.Order):
             order_id = order_param.id
 
-        exchange.cancel_order(order_id)
+        retry(
+            action=exchange.cancel_order,
+            attempts=self.attempts['cancel_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('cancelling order again.'),
+            args=(order_id, symbol, params))

catalyst/exchange/exchange_asset_finder.py

@@ -0,0 +1,179 @@
+import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.factory import find_exchanges
+from logbook import Logger
+
+log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
+
+
+class ExchangeAssetFinder(object):
+    def __init__(self, exchanges):
+        self.exchanges = exchanges
+
+    @property
+    def sids(self):
+        """
+        This seems to be used to pre-fetch assets.
+        I don't think that we need this for live-trading.
+        Leaving the list empty.
+        """
+        all_sids = []
+        for exchange_name in self.exchanges:
+            # This is what initializes each exchanges at the beginning
+            # of an algo
+            exchange = self.exchanges[exchange_name]
+            exchange.init()
+
+            all_sids += [asset.sid for asset in exchange.assets]
+
+        sids = list(set(all_sids))
+        return sids
+
+    def retrieve_asset(self, sid, default_none=False):
+        """
+        Retrieve the first Asset found for a given sid.
+        """
+        asset = None
+        for exchange_name in self.exchanges:
+            if asset is not None:
+                break
+
+            exchange = self.exchanges[exchange_name]
+            assets = [asset for asset in exchange.assets if asset.sid == sid]
+            if assets:
+                asset = assets[0]
+
+        return asset
+
+    def retrieve_all(self, sids, default_none=False):
+        """
+        Retrieve all assets in `sids`.
+
+        Parameters
+        ----------
+        sids : iterable of int
+            Assets to retrieve.
+        default_none : bool
+            If True, return None for failed lookups.
+            If False, raise `SidsNotFound`.
+
+        Returns
+        -------
+        assets : list[Asset or None]
+            A list of the same length as `sids` containing Assets (or Nones)
+            corresponding to the requested sids.
+
+        Raises
+        ------
+        SidsNotFound
+            When a requested sid is not found and default_none=False.
+        """
+        assets = []
+        for exchange_name in self.exchanges:
+            exchange = self.exchanges[exchange_name]
+            xas = [asset for asset in exchange.assets if asset.sid in sids]
+            assets += xas
+
+        return assets
+
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
+        """Lookup an asset by symbol.
+
+        Parameters
+        ----------
+        symbol : str
+            The ticker symbol to resolve.
+        as_of_date : datetime or None
+            Look up the last owner of this symbol as of this datetime.
+            If ``as_of_date`` is None, then this can only resolve the equity
+            if exactly one equity has ever owned the ticker.
+        fuzzy : bool, optional
+            Should fuzzy symbol matching be used? Fuzzy symbol matching
+            attempts to resolve differences in representations for
+            shareclasses. For example, some people may represent the ``A``
+            shareclass of ``BRK`` as ``BRK.A``, where others could write
+            ``BRK_A``.
+
+        Returns
+        -------
+        equity : Asset
+            The equity that held ``symbol`` on the given ``as_of_date``, or the
+            only equity to hold ``symbol`` if ``as_of_date`` is None.
+
+        Raises
+        ------
+        SymbolNotFound
+            Raised when no equity has ever held the given symbol.
+        MultipleSymbolsFound
+            Raised when no ``as_of_date`` is given and more than one equity
+            has held ``symbol``. This is also raised when ``fuzzy=True`` and
+            there are multiple candidates for the given ``symbol`` on the
+            ``as_of_date``.
+        """
+        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
+
+        return exchange.get_asset(symbol, data_frequency)
+
+    def lifetimes(self, dates, include_start_date):
+        """
+        Compute a DataFrame representing asset lifetimes for the specified date
+        range.
+
+        Parameters
+        ----------
+        dates : pd.DatetimeIndex
+            The dates for which to compute lifetimes.
+        include_start_date : bool
+            Whether or not to count the asset as alive on its start_date.
+
+            This is useful in a backtesting context where `lifetimes` is being
+            used to signify "do I have data for this asset as of the morning of
+            this date?"  For many financial metrics, (e.g. daily close), data
+            isn't available for an asset until the end of the asset's first
+            day.
+
+        Returns
+        -------
+        lifetimes : pd.DataFrame
+            A frame of dtype bool with `dates` as index and an Int64Index of
+            assets as columns.  The value at `lifetimes.loc[date, asset]` will
+            be True iff `asset` existed on `date`.  If `include_start_date` is
+            False, then lifetimes.loc[date, asset] will be false when date ==
+            asset.start_date.
+
+        See Also
+        --------
+        numpy.putmask
+        catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
+        """
+        exchanges = find_exchanges(features=['minuteBundle'])
+        if not exchanges:
+            raise ValueError('exchange with minute bundles not found')
+
+        # TODO: find a way to support multiple exchanges
+        exchange = exchanges[0]
+        # Using a single exchange for now because are not unique for the
+        # same asset in different exchanges. I'd like to avoid binding
+        # pipeline to a single exchange.
+        exchange.init()
+
+        data = []
+        for dt in dates:
+            exists = []
+
+            for asset in exchange.assets:
+                if include_start_date:
+                    condition = (asset.start_date <= dt < asset.end_minute)
+
+                else:
+                    condition = (asset.start_date < dt < asset.end_minute)
+
+                exists.append(condition)
+
+            data.append(exists)
+
+        sids = [asset.sid for asset in exchange.assets]
+        df = pd.DataFrame(data, index=dates, columns=exchange.assets)
+
+        return df

catalyst/exchange/exchange_blotter.py

@@ -1,15 +1,14 @@
-from time import sleep
-
+import numpy as np
 import pandas as pd
-from catalyst.assets._assets import TradingPair
 from logbook import Logger
+from redo import retry
 
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import ExchangeRequestError, \
-    ExchangePortfolioDataError, ExchangeTransactionError
+from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
-from catalyst.finance.order import ORDER_STATUS, Order
+from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
@@ -44,6 +43,11 @@ class TradingPairFeeSchedule(CommissionModel):
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
@@ -57,15 +61,15 @@ class TradingPairFeeSchedule(CommissionModel):
         cost = abs(transaction.amount) * transaction.price
 
         asset = order.asset
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
+        maker, taker = self.get_maker_taker(asset)
 
-        multiplier = maker \
-            if ((order.amount > 0 and order.limit < transaction.price)
-                or (order.amount < 0 and order.limit > transaction.price)) \
-               and order.limit_reached else taker
+        multiplier = taker
+        if order.limit is not None:
+            multiplier = maker \
+                if ((order.amount > 0 and order.limit < transaction.price)
+                    or (order.amount < 0 and order.limit > transaction.price)) \
+                and order.limit_reached else taker
 
-        # Assuming just the taker fee for now
         fee = cost * multiplier
         return fee
 
@@ -91,7 +95,6 @@ class TradingPairFixedSlippage(SlippageModel):
 
     def simulate(self, data, asset, orders_for_asset):
         self._volume_for_bar = 0
-
         price = data.current(asset, 'close')
 
         dt = data.current_dt
@@ -101,18 +104,20 @@ class TradingPairFixedSlippage(SlippageModel):
 
             order.check_triggers(price, dt)
             if not order.triggered:
-                log.debug('order has not reached the trigger at current '
-                          'price {}'.format(price))
+                log.info(
+                    'order has not reached the trigger at current '
+                    'price {}'.format(price)
+                )
                 continue
 
             execution_price, execution_volume = self.process_order(data, order)
+            if execution_price is not None:
+                transaction = create_transaction(
+                    order, dt, execution_price, execution_volume
+                )
 
-            transaction = create_transaction(
-                order, dt, execution_price, execution_volume
-            )
-
-            self._volume_for_bar += abs(transaction.amount)
-            yield order, transaction
+                self._volume_for_bar += abs(transaction.amount)
+                yield order, transaction
 
     def process_order(self, data, order):
         price = data.current(order.asset, 'close')
@@ -132,6 +137,7 @@ class TradingPairFixedSlippage(SlippageModel):
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
         self.simulate_orders = kwargs.pop('simulate_orders', False)
+        self.attempts = kwargs.pop('attempts', False)
 
         self.exchanges = kwargs.pop('exchanges', None)
         if not self.exchanges:
@@ -151,31 +157,11 @@ class ExchangeBlotter(Blotter):
             TradingPair: TradingPairFeeSchedule()
         }
 
-        self.retry_delay = 5
-        self.retry_check_open_orders = 5
-
-    def exchange_order(self, asset, amount, style=None, attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(
-                asset, amount, style
-            )
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-
-                return self.exchange_order(
-                    asset, amount, style, attempt_index + 1
-                )
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
+    def exchange_order(self, asset, amount, style=None):
+        exchange = self.exchanges[asset.exchange]
+        return exchange.order(
+            asset, amount, style
+        )
 
     @expect_types(asset=TradingPair)
     def order(self, asset, amount, style, order_id=None):
@@ -190,8 +176,13 @@ class ExchangeBlotter(Blotter):
             )
 
         else:
-            order = self.exchange_order(
-                asset, amount, style
+            order = retry(
+                action=self.exchange_order,
+                attempts=self.attempts['order_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.'),
+                args=(asset, amount, style),
             )
 
             self.open_orders[order.asset].append(order)
@@ -217,34 +208,29 @@ class ExchangeBlotter(Blotter):
             for order in self.open_orders[asset]:
                 log.debug('found open order: {}'.format(order.id))
 
-                new_order, executed_price = exchange.get_order(order.id, asset)
-                log.debug(
-                    'got updated order {} {}'.format(
-                        new_order, executed_price
+                transactions = exchange.process_order(order)
+                # This is a temporary measure, we should really update all
+                # trades, not just when the order gets filled. I just think
+                # that this is safer until we have a robust way to track
+                # the trades already processed by the algo. We can't loose
+                # them if the algo shuts down.
+                if transactions and order.status == ORDER_STATUS.FILLED:
+                    avg_price = np.average(
+                        a=[t.price for t in transactions],
+                        weights=[t.amount for t in transactions],
                     )
-                )
-                order.status = new_order.status
-
-                if order.status == ORDER_STATUS.FILLED:
-                    order.commission = new_order.commission
-                    if order.amount != new_order.amount:
-                        log.warn(
-                            'executed order amount {} differs '
-                            'from original'.format(
-                                new_order.amount, order.amount
-                            )
+                    ostatus = 'filled' if order.open_amount == 0 else 'partial'
+                    log.info(
+                        '{} order {} / {}: {}, avg price: {}'.format(
+                            ostatus,
+                            order.id,
+                            asset.symbol,
+                            order.filled,
+                            avg_price,
                         )
-                        order.amount = new_order.amount
-
-                    transaction = Transaction(
-                        asset=order.asset,
-                        amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
-                        price=executed_price,
-                        order_id=order.id,
-                        commission=order.commission
                     )
-                    yield order, transaction
+                    for transaction in transactions:
+                        yield order, transaction
 
                 elif order.status == ORDER_STATUS.CANCELLED:
                     yield order, None
@@ -252,46 +238,40 @@ class ExchangeBlotter(Blotter):
                 else:
                     delta = pd.Timestamp.utcnow() - order.dt
                     log.info(
-                        'order {order_id} still open after {delta}'.format(
+                        '{exchange} order {order_id} for {symbol} still open '
+                        'after {delta}'.format(
+                            exchange=exchange.name,
                             order_id=order.id,
-                            delta=delta
+                            delta=delta,
+                            symbol=order.asset.symbol,
                         )
                     )
 
-    def get_exchange_transactions(self, attempt_index=0):
+    def get_exchange_transactions(self):
         closed_orders = []
         transactions = []
         commissions = []
 
-        try:
-            for order, txn in self.check_open_orders():
-                order.dt = txn.dt
+        for order, txn in self.check_open_orders():
+            order.dt = txn.dt
+            transactions.append(txn)
 
-                transactions.append(txn)
+            if not order.open:
+                closed_orders.append(order)
 
-                if not order.open:
-                    closed_orders.append(order)
-
-            return transactions, commissions, closed_orders
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self.get_exchange_transactions(attempt_index + 1)
-
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
+        return transactions, commissions, closed_orders
 
     def get_transactions(self, bar_data):
         if self.simulate_orders:
             return super(ExchangeBlotter, self).get_transactions(bar_data)
 
         else:
-            return self.get_exchange_transactions()
+            return retry(
+                action=self.get_exchange_transactions,
+                attempts=self.attempts['get_transactions_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn(
+                    'Fetching exchange transactions again.'
+                )
+            )

catalyst/exchange/exchange_bundle.py

@@ -1,3 +1,4 @@
+import copy
 import os
 import shutil
 from datetime import timedelta
@@ -18,18 +19,18 @@ from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
-    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
-from catalyst.exchange.exchange_utils import get_exchange_folder, \
-    save_exchange_symbols, mixin_market_params
+from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
+    get_bcolz_chunk, get_df_from_arrays, get_assets
+from catalyst.exchange.utils.datetime_utils import get_start_dt, \
+    get_period_label, get_month_start_end, get_year_start_end, get_period, \
+    timestr_to_dt
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -233,12 +234,12 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                            name=asset.symbol,
-                            start_dt=asset.start_date.strftime(
-                                DATE_TIME_FORMAT),
-                            end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                            dates=[date.strftime(
-                                DATE_TIME_FORMAT) for date in dates])
+                    name=asset.symbol,
+                    start_dt=asset.start_date.strftime(
+                        DATE_TIME_FORMAT),
+                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                    dates=[date.strftime(
+                        DATE_TIME_FORMAT) for date in dates])
 
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
@@ -287,12 +288,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                        name=asset.symbol,
-                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                        threshold=threshold,
-                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                               for date in dates])
+                name=asset.symbol,
+                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                threshold=threshold,
+                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                       for date in dates])
 
             problems.append(problem)
 
@@ -459,10 +460,10 @@ class ExchangeBundle:
                 last_entry = None
 
         if start is None or \
-                (earliest_trade is not None and earliest_trade > start):
+            (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
-        if end is None or (last_entry is not None and end > last_entry):
+        if last_entry is not None and (end is None or end > last_entry):
             end = last_entry.replace(minute=59, hour=23) \
                 if data_frequency == 'minute' else last_entry
 
@@ -513,8 +514,8 @@ class ExchangeBundle:
                 continue
 
             dates = pd.date_range(
-                start=get_period_label(adj_start, data_frequency),
-                end=get_period_label(adj_end, data_frequency),
+                start=get_period(adj_start, data_frequency),
+                end=get_period(adj_end, data_frequency),
                 freq='MS' if data_frequency == 'minute' else 'AS',
                 tz=UTC
             )
@@ -553,7 +554,9 @@ class ExchangeBundle:
 
             # We sort the chunks by end date to ingest most recent data first
             chunks[asset].sort(
-                key=lambda chunk: pd.to_datetime(chunk['period'])
+                key=lambda chunk: timestr_to_dt(
+                    chunk['period'], data_frequency
+                )
             )
 
         return chunks
@@ -601,14 +604,15 @@ class ExchangeBundle:
         if show_breakdown:
             for asset in chunks:
                 with maybe_show_progress(
-                        chunks[asset],
-                        show_progress,
-                        label='Ingesting {frequency} price data for '
-                              '{symbol} on {exchange}'.format(
-                            exchange=self.exchange_name,
-                            frequency=data_frequency,
-                            symbol=asset.symbol
-                        )) as it:
+                    chunks[asset],
+                    show_progress,
+                    label='Ingesting {frequency} price data for '
+                          '{symbol} on {exchange}'.format(
+                        exchange=self.exchange_name,
+                        frequency=data_frequency,
+                        symbol=asset.symbol
+                    )
+                ) as it:
                     for chunk in it:
                         problems += self.ingest_ctable(
                             asset=chunk['asset'],
@@ -623,16 +627,19 @@ class ExchangeBundle:
 
             # We sort the chunks by end date to ingest most recent data first
             all_chunks.sort(
-                key=lambda chunk: pd.to_datetime(chunk['period'])
+                key=lambda chunk: timestr_to_dt(
+                    chunk['period'], data_frequency
+                )
             )
             with maybe_show_progress(
-                    all_chunks,
-                    show_progress,
-                    label='Ingesting {frequency} price data on '
-                          '{exchange}'.format(
-                            exchange=self.exchange_name,
-                            frequency=data_frequency,
-                    )) as it:
+                all_chunks,
+                show_progress,
+                label='Ingesting {frequency} price data on '
+                      '{exchange}'.format(
+                    exchange=self.exchange_name,
+                    frequency=data_frequency,
+                )
+            ) as it:
                 for chunk in it:
                     problems += self.ingest_ctable(
                         asset=chunk['asset'],
@@ -668,7 +675,7 @@ class ExchangeBundle:
 
         if self.exchange is None:
             # Avoid circular dependencies
-            from catalyst.exchange.factory import get_exchange
+            from catalyst.exchange.utils.factory import get_exchange
             self.exchange = get_exchange(self.exchange_name)
 
         problems = []
@@ -681,6 +688,7 @@ class ExchangeBundle:
                 last_traded=np.object_,
                 open=np.float64,
                 high=np.float64,
+                low=np.float64,
                 close=np.float64,
                 volume=np.float64
             ),
@@ -700,42 +708,36 @@ class ExchangeBundle:
         for symbol in symbols:
             start_dt = df.index.get_level_values(1).min()
             end_dt = df.index.get_level_values(1).max()
-            end_dt_key = 'end_{}'.format(data_frequency)
 
-            market = self.exchange.get_market(symbol)
-            if market is None:
-                raise ValueError('symbol not available in the exchange.')
+            try:
+                asset = self.exchange.get_asset(symbol, is_local=True)
+            except:
+                asset = copy.deepcopy(self.exchange.get_asset(symbol))
 
-            params = dict(
-                exchange=self.exchange.name,
-                data_source='local',
-                exchange_symbol=market['id'],
-            )
-            mixin_market_params(self.exchange_name, params, market)
+            if asset.data_source == 'local':
+                asset.start_date = asset.start_date \
+                    if asset.start_date < start_dt else start_dt
 
-            asset_def = self.exchange.get_asset_def(market, True)
-            if asset_def is not None:
-                params['symbol'] = asset_def['symbol']
+                if data_frequency == 'daily':
+                    asset.end_date = asset.end_daily = asset.end_daily \
+                        if asset.end_daily > end_dt else end_dt
 
-                params['start_date'] = asset_def['start_date'] \
-                    if asset_def['start_date'] < start_dt else start_dt
-
-                params['end_date'] = asset_def[end_dt_key] \
-                    if asset_def[end_dt_key] > end_dt else end_dt
-
-                params['end_daily'] = end_dt \
-                    if data_frequency == 'daily' else asset_def['end_daily']
-
-                params['end_minute'] = end_dt \
-                    if data_frequency == 'minute' else asset_def['end_minute']
+                else:
+                    asset.end_date = asset.end_minute = asset.end_minute \
+                        if asset.end_minute > end_dt else end_dt
 
             else:
-                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+                asset.data_source = 'local'
+                asset.start_date = start_dt
+                asset.end_dt = end_dt
 
-                params['end_daily'] = end_dt \
-                    if data_frequency == 'daily' else 'N/A'
-                params['end_minute'] = end_dt \
-                    if data_frequency == 'minute' else 'N/A'
+                if data_frequency == 'daily':
+                    asset.end_daily = end_dt
+                    asset.end_minute = None
+
+                else:
+                    asset.end_daily = None
+                    asset.end_minute = end_dt
 
             if min_start_dt is None or start_dt < min_start_dt:
                 min_start_dt = start_dt
@@ -743,11 +745,9 @@ class ExchangeBundle:
             if max_end_dt is None or end_dt > max_end_dt:
                 max_end_dt = end_dt
 
-            asset = TradingPair(**params)
-            assets[market['id']] = asset
-
-        save_exchange_symbols(self.exchange_name, assets, True)
+            assets[symbol] = asset
 
+        # TODO: update config.json
         writer = self.get_writer(
             start_dt=min_start_dt.replace(hour=00, minute=00),
             end_dt=max_end_dt.replace(hour=23, minute=59),
@@ -755,9 +755,10 @@ class ExchangeBundle:
         )
 
         for symbol in assets:
+            # here the symbol is the market['id']
             asset = assets[symbol]
             ohlcv_df = df.loc[
-                (df.index.get_level_values(0) == symbol)
+                (df.index.get_level_values(0) == asset.symbol)
             ]  # type: pd.DataFrame
             ohlcv_df.index = ohlcv_df.index.droplevel(0)
 
@@ -805,7 +806,7 @@ class ExchangeBundle:
         else:
             if self.exchange is None:
                 # Avoid circular dependencies
-                from catalyst.exchange.factory import get_exchange
+                from catalyst.exchange.utils.factory import get_exchange
                 self.exchange = get_exchange(self.exchange_name)
 
             assets = get_assets(
@@ -829,7 +830,6 @@ class ExchangeBundle:
                                            field,
                                            data_frequency,
                                            algo_end_dt=None,
-                                           trailing_bar_count=None,
                                            force_auto_ingest=False
                                            ):
         """
@@ -857,7 +857,6 @@ class ExchangeBundle:
                     bar_count=bar_count,
                     field=field,
                     data_frequency=data_frequency,
-                    trailing_bar_count=trailing_bar_count,
                 )
                 return pd.DataFrame(series)
 
@@ -886,7 +885,6 @@ class ExchangeBundle:
                     field=field,
                     data_frequency=data_frequency,
                     reset_reader=True,
-                    trailing_bar_count=trailing_bar_count,
                 )
                 return series
 
@@ -897,7 +895,6 @@ class ExchangeBundle:
                 bar_count=bar_count,
                 field=field,
                 data_frequency=data_frequency,
-                trailing_bar_count=trailing_bar_count,
             )
             return pd.DataFrame(series)
 
@@ -961,17 +958,12 @@ class ExchangeBundle:
                                   bar_count,
                                   field,
                                   data_frequency,
-                                  trailing_bar_count=None,
                                   reset_reader=False):
         start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
         start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
 
-        if trailing_bar_count:
-            delta = get_delta(trailing_bar_count, data_frequency)
-            end_dt += delta
-
         # This is an attempt to resolve some caching with the reader
         # when auto-ingesting data.
         # TODO: needs more work

catalyst/exchange/exchange_data_portal.py

@@ -1,31 +1,30 @@
 import abc
-from time import sleep
 
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangeBarDataError,
     PricingDataNotLoadedError)
-from catalyst.exchange.exchange_utils import get_frequency, \
-    resample_history_df, group_assets_by_exchange
+from catalyst.exchange.utils.exchange_utils import resample_history_df, \
+    group_assets_by_exchange
+from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
+from logbook import Logger
+from redo import retry
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
-
-        # TODO: put somewhere accessible by each algo
-        self.retry_get_history_window = 5
-        self.retry_get_spot_value = 5
-        self.retry_delay = 5
+        self.attempts = dict(
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
 
         super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
 
@@ -36,33 +35,14 @@ class DataPortalExchangeBase(DataPortal):
                             frequency,
                             field,
                             data_frequency,
-                            ffill=True,
-                            attempt_index=0):
-        try:
-            exchange_assets = group_assets_by_exchange(assets)
-            if len(exchange_assets) > 1:
-                df_list = []
-                for exchange_name in exchange_assets:
-                    assets = exchange_assets[exchange_name]
+                            ffill=True):
+        exchange_assets = group_assets_by_exchange(assets)
+        if len(exchange_assets) > 1:
+            df_list = []
+            for exchange_name in exchange_assets:
+                assets = exchange_assets[exchange_name]
 
-                    df_exchange = self.get_exchange_history_window(
-                        exchange_name,
-                        assets,
-                        end_dt,
-                        bar_count,
-                        frequency,
-                        field,
-                        data_frequency,
-                        ffill)
-
-                    df_list.append(df_exchange)
-
-                # Merging the values values of each exchange
-                return pd.concat(df_list)
-
-            else:
-                exchange_name = list(exchange_assets.keys())[0]
-                return self.get_exchange_history_window(
+                df_exchange = self.get_exchange_history_window(
                     exchange_name,
                     assets,
                     end_dt,
@@ -72,26 +52,22 @@ class DataPortalExchangeBase(DataPortal):
                     data_frequency,
                     ffill)
 
-        except ExchangeRequestError as e:
-            log.warn(
-                'get history attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_history_window:
-                sleep(self.retry_delay)
-                return self._get_history_window(assets,
-                                                end_dt,
-                                                bar_count,
-                                                frequency,
-                                                field,
-                                                data_frequency,
-                                                ffill,
-                                                attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='history',
-                    attempts=attempt_index,
-                    error=e
-                )
+                df_list.append(df_exchange)
+
+            # Merging the values values of each exchange
+            return pd.concat(df_list)
+
+        else:
+            exchange_name = list(exchange_assets.keys())[0]
+            return self.get_exchange_history_window(
+                exchange_name,
+                assets,
+                end_dt,
+                bar_count,
+                frequency,
+                field,
+                data_frequency,
+                ffill)
 
     def get_history_window(self,
                            assets,
@@ -105,13 +81,19 @@ class DataPortalExchangeBase(DataPortal):
         if field == 'price':
             field = 'close'
 
-        return self._get_history_window(assets,
-                                        end_dt,
-                                        bar_count,
-                                        frequency,
-                                        field,
-                                        data_frequency,
-                                        ffill)
+        return retry(
+            action=self._get_history_window,
+            attempts=self.attempts['get_history_window_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching history again.'),
+            args=(assets,
+                  end_dt,
+                  bar_count,
+                  frequency,
+                  field,
+                  data_frequency,
+                  ffill))
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
@@ -125,69 +107,58 @@ class DataPortalExchangeBase(DataPortal):
                                     ffill=True):
         pass
 
-    def _get_spot_value(self, assets, field, dt, data_frequency,
-                        attempt_index=0):
-        try:
-            if isinstance(assets, TradingPair):
-                spot_values = self.get_exchange_spot_value(
-                    assets.exchange, [assets], field, dt, data_frequency)
+    def _get_spot_value(self, assets, field, dt, data_frequency):
+        if isinstance(assets, TradingPair):
+            spot_values = self.get_exchange_spot_value(
+                assets.exchange, [assets], field, dt, data_frequency)
 
-                if not spot_values:
-                    return np.nan
+            if not spot_values:
+                return np.nan
 
-                return spot_values[0]
+            return spot_values[0]
+
+        else:
+            exchange_assets = dict()
+            for asset in assets:
+                if asset.exchange not in exchange_assets:
+                    exchange_assets[asset.exchange] = list()
+
+                exchange_assets[asset.exchange].append(asset)
+
+            if len(list(exchange_assets.keys())) == 1:
+                exchange_name = list(exchange_assets.keys())[0]
+                return self.get_exchange_spot_value(
+                    exchange_name, assets, field, dt, data_frequency)
 
             else:
-                exchange_assets = dict()
-                for asset in assets:
-                    if asset.exchange not in exchange_assets:
-                        exchange_assets[asset.exchange] = list()
+                spot_values = []
+                for exchange_name in exchange_assets:
+                    assets = exchange_assets[exchange_name]
+                    exchange_spot_values = self.get_exchange_spot_value(
+                        exchange_name,
+                        assets,
+                        field,
+                        dt,
+                        data_frequency
+                    )
+                    if len(assets) == 1:
+                        spot_values.append(exchange_spot_values)
+                    else:
+                        spot_values += exchange_spot_values
 
-                    exchange_assets[asset.exchange].append(asset)
-
-                if len(list(exchange_assets.keys())) == 1:
-                    exchange_name = list(exchange_assets.keys())[0]
-                    return self.get_exchange_spot_value(
-                        exchange_name, assets, field, dt, data_frequency)
-
-                else:
-                    spot_values = []
-                    for exchange_name in exchange_assets:
-                        assets = exchange_assets[exchange_name]
-                        exchange_spot_values = self.get_exchange_spot_value(
-                            exchange_name,
-                            assets,
-                            field,
-                            dt,
-                            data_frequency
-                        )
-                        if len(assets) == 1:
-                            spot_values.append(exchange_spot_values)
-                        else:
-                            spot_values += exchange_spot_values
-
-                    return spot_values
-
-        except ExchangeRequestError as e:
-            log.warn(
-                'get spot value attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_spot_value:
-                sleep(self.retry_delay)
-                return self._get_spot_value(assets, field, dt, data_frequency,
-                                            attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='spot',
-                    attempts=attempt_index,
-                    error=e
-                )
+                return spot_values
 
     def get_spot_value(self, assets, field, dt, data_frequency):
         if field == 'price':
             field = 'close'
 
-        return self._get_spot_value(assets, field, dt, data_frequency)
+        return retry(
+            action=self._get_spot_value,
+            attempts=self.attempts['get_spot_value_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching spot value again.'),
+            args=(assets, field, dt, data_frequency))
 
     @abc.abstractmethod
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -321,13 +292,13 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         DataFrame
 
         """
+        # TODO: verify that the exchange supports the timeframe
         bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
-        trailing_bar_count = candle_size - 1
 
         if data_frequency == 'minute' and adj_data_frequency == 'daily':
             end_dt = end_dt.floor('1D')
@@ -339,10 +310,10 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
-            trailing_bar_count=trailing_bar_count
         )
 
-        df = resample_history_df(pd.DataFrame(series), freq, field)
+        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
+        df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
         return df
 
     def get_exchange_spot_value(self,

catalyst/exchange/exchange_errors.py

@@ -100,6 +100,19 @@ class InvalidHistoryFrequencyError(ZiplineError):
     ).strip()
 
 
+class UnsupportedHistoryFrequencyError(ZiplineError):
+    msg = (
+        '{exchange} does not support candle frequency {freq}, please choose '
+        'from: {freqs}.'
+    ).strip()
+
+
+class InvalidHistoryTimeframeError(ZiplineError):
+    msg = (
+        'CCXT timeframe {timeframe} not supported by the exchange.'
+    ).strip()
+
+
 class MismatchingFrequencyError(ZiplineError):
     msg = (
         'Bar aggregate frequency {frequency} not compatible with '
@@ -162,8 +175,8 @@ class SidHashError(ZiplineError):
 
 class BaseCurrencyNotFoundError(ZiplineError):
     msg = (
-        'Algorithm base currency {base_currency} not found in exchange '
-        '{exchange}.'
+        'Algorithm base currency {base_currency} not found in account '
+        'balances on {exchange}: {balances}'
     ).strip()
 
 
@@ -226,16 +239,20 @@ class PricingDataValueError(ZiplineError):
 
 
 class DataCorruptionError(ZiplineError):
-    msg = ('Unable to validate data for {exchange} {symbols} in date range '
-           '[{start_dt} - {end_dt}]. The data is either corrupted or '
-           'unavailable. Please try deleting this bundle:'
-           '\n`catalyst clean-exchange -x {exchange}\n'
-           'Then, ingest the data again. Please contact the Catalyst team if '
-           'the issue persists.').strip()
+    msg = (
+        'Unable to validate data for {exchange} {symbols} in date range '
+        '[{start_dt} - {end_dt}]. The data is either corrupted or '
+        'unavailable. Please try deleting this bundle:'
+        '\n`catalyst clean-exchange -x {exchange}\n'
+        'Then, ingest the data again. Please contact the Catalyst team if '
+        'the issue persists.'
+    ).strip()
 
 
 class ApiCandlesError(ZiplineError):
-    msg = ('Unable to fetch candles from the remote API: {error}.').strip()
+    msg = (
+        'Unable to fetch candles from the remote API: {error}.'
+    ).strip()
 
 
 class NoDataAvailableOnExchange(ZiplineError):
@@ -248,13 +265,16 @@ class NoDataAvailableOnExchange(ZiplineError):
 
 
 class NoValueForField(ZiplineError):
-    msg = ('Value not found for field: {field}.').strip()
+    msg = (
+        'Value not found for field: {field}.'
+    ).strip()
 
 
 class OrderTypeNotSupported(ZiplineError):
     msg = (
-        'Order type `{order_type}` not currencly supported by Catalyst. '
-        'Please use `limit` or `market` orders only.').strip()
+        'Order type `{order_type}` not currency supported by Catalyst. '
+        'Please use `limit` or `market` orders only.'
+    ).strip()
 
 
 class NotEnoughCapitalError(ZiplineError):
@@ -262,10 +282,64 @@ class NotEnoughCapitalError(ZiplineError):
         'Not enough capital on exchange {exchange} for trading. Each '
         'exchange should contain at least as much {base_currency} '
         'as the specified `capital_base`. The current balance {balance} is '
-        'lower than the `capital_base`: {capital_base}').strip()
+        'lower than the `capital_base`: {capital_base}'
+    ).strip()
+
+
+class NotEnoughCashError(ZiplineError):
+    msg = (
+        'Total {currency} amount on {exchange} is lower than the cash '
+        'reserved for this algo: {free} < {cash}. While trades can be made on '
+        'the exchange accounts outside of the algo, exchange must have enough '
+        'free {currency} to cover the algo cash.'
+    ).strip()
+
 
 class LastCandleTooEarlyError(ZiplineError):
     msg = (
         'The trade date of the last candle {last_traded} is before the '
         'specified end date minus one candle {end_dt}. Please verify how '
-        '{exchange} calculates the start date of OHLCV candles.').strip()
+        '{exchange} calculates the start date of OHLCV candles.'
+    ).strip()
+
+
+class TickerNotFoundError(ZiplineError):
+    msg = (
+        'Unable to fetch ticker for {symbol} on {exchange}.'
+    ).strip()
+
+
+class BalanceNotFoundError(ZiplineError):
+    msg = (
+        '{currency} not found in account balance on {exchange}: {balances}.'
+    ).strip()
+
+
+class BalanceTooLowError(ZiplineError):
+    msg = (
+        'Balance for {currency} on {exchange} too low: {free} < {amount}. '
+        'Positions have likely been sold outside of this algorithm. Please '
+        'add positions to hold a free amount greater than {amount}, or clean '
+        'the state of this algo and restart.'
+    ).strip()
+
+
+class NoCandlesReceivedFromExchange(ZiplineError):
+    msg = (
+        'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
+        'an empty list of candles was received for {exchange}.'
+    ).strip()
+
+
+class MarketsNotFoundError(ZiplineError):
+    msg = (
+        'Exchange {exchange} contains no valid market so it is unusable in '
+        'Catalyst.'
+    ).strip()
+
+
+class InvalidMarketError(ZiplineError):
+    msg = (
+        'Exchange {exchange} contains at least one incorrectly structured '
+        'market: {market}, so it is unusable in Catalyst.'
+    ).strip()

catalyst/exchange/exchange_portfolio.py

@@ -1,8 +1,7 @@
 import numpy as np
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from logbook import Logger
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 

catalyst/exchange/exchange_pricing_loader.py

@@ -0,0 +1,177 @@
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from catalyst.constants import LOG_LEVEL
+from catalyst.data.us_equity_pricing import BcolzDailyBarReader
+from catalyst.errors import NoFurtherDataError
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.lib.adjusted_array import AdjustedArray
+from catalyst.pipeline.data import DataSet, Column
+from catalyst.pipeline.loaders.base import PipelineLoader
+from catalyst.utils.calendars import get_calendar
+from catalyst.utils.numpy_utils import float64_dtype
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
+
+UINT32_MAX = iinfo(uint32).max
+
+log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
+
+
+class TradingPairPricing(DataSet):
+    """
+    Dataset representing daily trading prices and volumes.
+    """
+    open = Column(float64_dtype)
+    high = Column(float64_dtype)
+    low = Column(float64_dtype)
+    close = Column(float64_dtype)
+    volume = Column(float64_dtype)
+
+
+class ExchangePricingLoader(PipelineLoader):
+    """
+    PipelineLoader for Crypto Pricing data
+
+    Delegates loading of baselines and adjustments.
+    """
+
+    def __init__(self, data_frequency):
+
+        cal = get_calendar('OPEN')
+
+        if data_frequency == 'daily':
+            reader = None
+            all_sessions = cal.all_sessions
+
+        elif data_frequency == 'minute':
+            reader = None
+            all_sessions = cal.all_minutes
+
+        else:
+            raise ValueError(
+                'Invalid data frequency: {}'.format(data_frequency)
+            )
+
+        self.data_frequency = data_frequency
+        self.raw_price_loader = reader
+        self._columns = TradingPairPricing.columns
+        self._all_sessions = all_sessions
+
+    @classmethod
+    def from_files(cls, pricing_path):
+        """
+        Create a loader from a bcolz equity pricing dir and a SQLite
+        adjustments path.
+
+        Parameters
+        ----------
+        pricing_path : str
+            Path to a bcolz directory written by a BcolzDailyBarWriter.
+        """
+        return cls(
+            BcolzDailyBarReader(pricing_path),
+        )
+
+    def load_adjusted_array(self, columns, dates, assets, mask):
+        # load_adjusted_array is called with dates on which the user's algo
+        # will be shown data, which means we need to return the data that would
+        # be known at the start of each date.  We assume that the latest data
+        # known on day N is the data from day (N - 1), so we shift all query
+        # dates back by a day.
+        start_date, end_date = _shift_dates(
+            self._all_sessions, dates[0], dates[-1], shift=1,
+        )
+        colnames = [c.name for c in columns]
+
+        if len(assets) == 0:
+            raise ValueError(
+                'Pipeline cannot load data with eligible assets.'
+            )
+
+        exchange_names = []
+        for asset in assets:
+            if asset.exchange not in exchange_names:
+                exchange_names.append(asset.exchange)
+
+        exchange = get_exchange(exchange_names[0])
+        reader = exchange.bundle.get_reader(self.data_frequency)
+
+        raw_arrays = reader.load_raw_arrays(
+            colnames,
+            start_date,
+            end_date,
+            assets,
+        )
+
+        out = {}
+        for c, c_raw in zip(columns, raw_arrays):
+            out[c] = AdjustedArray(
+                c_raw.astype(c.dtype),
+                mask,
+                {},
+                c.missing_value,
+            )
+        return out
+
+    @property
+    def columns(self):
+        return self._columns
+
+
+def _shift_dates(dates, start_date, end_date, shift):
+    try:
+        start = dates.get_loc(start_date)
+    except KeyError:
+        if start_date < dates[0]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requested data starting on {query_start}, "
+                    "but first known date is {calendar_start}"
+                ).format(
+                    query_start=str(start_date),
+                    calendar_start=str(dates[0]),
+                )
+            )
+        else:
+            raise ValueError("Query start %s not in calendar" % start_date)
+
+    # Make sure that shifting doesn't push us out of the calendar.
+    if start < shift:
+        raise NoFurtherDataError(
+            msg=(
+                "Pipeline Query requested data from {shift}"
+                " days before {query_start}, but first known date is only "
+                "{start} days earlier."
+            ).format(shift=shift, query_start=start_date, start=start),
+        )
+
+    try:
+        end = dates.get_loc(end_date)
+    except KeyError:
+        if end_date > dates[-1]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requesting data up to {query_end}, "
+                    "but last known date is {calendar_end}"
+                ).format(
+                    query_end=end_date,
+                    calendar_end=dates[-1],
+                )
+            )
+        else:
+            raise ValueError("Query end %s not in calendar" % end_date)
+    return dates[start - shift], dates[end - shift]

catalyst/exchange/factory.py

@@ -1,34 +0,0 @@
-import os
-
-from catalyst.exchange.ccxt.ccxt_exchange import CCXT
-from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_exchange_folder
-
-
-def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
-    exchange_auth = get_exchange_auth(exchange_name)
-
-    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
-    if must_authenticate and not has_auth:
-        raise ExchangeAuthEmpty(
-            exchange=exchange_name.title(),
-            filename=os.path.join(
-                get_exchange_folder(exchange_name), 'auth.json'
-            )
-        )
-
-    return CCXT(
-        exchange_name=exchange_name,
-        key=exchange_auth['key'],
-        secret=exchange_auth['secret'],
-        base_currency=base_currency,
-    )
-
-
-def get_exchanges(exchange_names):
-    exchanges = dict()
-    for exchange_name in exchange_names:
-        exchanges[exchange_name] = get_exchange(exchange_name)
-
-    return exchanges

catalyst/exchange/live_graph_clock.py

@@ -1,14 +1,12 @@
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.stats_utils import prepare_stats
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
@@ -38,177 +36,23 @@ class LiveGraphClock(object):
     the exchange and the live trading machine's clock. It's not used currently.
     """
 
-    def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
-
-        global mdates, plt  # TODO: Could be cleaner
-        import matplotlib.dates as mdates
-        from matplotlib import pyplot as plt
-        from matplotlib import style
+    def __init__(self, sessions, context, callback=None,
+                 time_skew=pd.Timedelta('0s')):
 
         self.sessions = sessions
         self.time_skew = time_skew
         self._last_emit = None
         self._before_trading_start_bar_yielded = True
         self.context = context
-        self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
-
-        style.use('dark_background')
-
-        fig = plt.figure()
-        fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
-            self.context.algo_namespace))
-
-        self.ax_pnl = fig.add_subplot(311)
-
-        self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
-
-        self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
-
-        if len(context.minute_stats) > 0:
-            self.draw_pnl()
-            self.draw_custom_signals()
-            self.draw_exposure()
-
-        # rotates and right aligns the x labels, and moves the bottom of the
-        # axes up to make room for them
-        fig.autofmt_xdate()
-        fig.subplots_adjust(hspace=0.5)
-
-        plt.tight_layout()
-        plt.ion()
-        plt.show()
-
-    def format_ax(self, ax):
-        """
-        Trying to assign reasonable parameters to the time axis.
-
-        Parameters
-        ----------
-        ax:
-
-        """
-        # TODO: room for improvement
-        ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-        ax.xaxis.set_major_formatter(self.fmt)
-
-        locator = mdates.HourLocator(interval=4)
-        locator.MAXTICKS = 5000
-        ax.xaxis.set_minor_locator(locator)
-
-        datemin = pd.Timestamp.utcnow()
-        ax.set_xlim(datemin)
-
-        ax.grid(True)
-
-    def set_legend(self, ax):
-        """
-        Set legend on the chart.
-
-        Parameters
-        ----------
-        ax
-
-        """
-        ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-    def draw_pnl(self):
-        """
-        Draw p&l line on the chart.
-
-        """
-        ax = self.ax_pnl
-        df = self.context.pnl_stats
-
-        ax.clear()
-        ax.set_title('Performance')
-        ax.plot(df.index, df['performance'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Performance'
-                )
-
-        def perc(val):
-            return '{:2f}'.format(val)
-
-        ax.format_ydata = perc
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_custom_signals(self):
-        """
-        Draw custom signals on the chart.
-
-        """
-        ax = self.ax_custom_signals
-        df = self.context.custom_signals_stats
-
-        colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-        ax.clear()
-        ax.set_title('Custom Signals')
-        for index, column in enumerate(df.columns.values.tolist()):
-            ax.plot(df.index, df[column], '-',
-                    color=colors[index],
-                    linewidth=1.0,
-                    label=column
-                    )
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_exposure(self):
-        """
-        Draw exposure line on the chart.
-
-        """
-        ax = self.ax_exposure
-        context = self.context
-        df = context.exposure_stats
-
-        # TODO: list exchanges in graph
-        base_currency = None
-        positions = []
-        for exchange_name in context.exchanges:
-            exchange = context.exchanges[exchange_name]
-
-            if not base_currency:
-                base_currency = exchange.base_currency
-            elif base_currency != exchange.base_currency:
-                raise MismatchingBaseCurrenciesExchanges(
-                    base_currency=base_currency,
-                    exchange_name=exchange.name,
-                    exchange_currency=exchange.base_currency
-                )
-
-            positions += exchange.portfolio.positions
-
-        ax.clear()
-        ax.set_title('Exposure')
-        ax.plot(df.index, df['base_currency'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Base Currency: {}'.format(base_currency.upper())
-                )
-
-        symbols = []
-        for position in positions:
-            symbols.append(position.symbol)
-
-        ax.plot(df.index, df['long_exposure'], '-',
-                color='blue',
-                linewidth=1.0,
-                label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-        self.set_legend(ax)
-        self.format_ax(ax)
+        self.callback = callback
 
     def __iter__(self):
+        from matplotlib import pyplot as plt
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:
             current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
+            current_minute = current_time.floor('1T')
 
             if self._last_emit is None or current_minute > self._last_emit:
                 log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -216,14 +60,11 @@ class LiveGraphClock(object):
                 self._last_emit = current_minute
                 yield current_minute, BAR
 
-                try:
-                    self.draw_pnl()
-                    self.draw_custom_signals()
-                    self.draw_exposure()
-
-                    plt.draw()
-                except Exception as e:
-                    log.warn('Unable to update the graph: {}'.format(e))
+                recorded_cols = list(self.context.recorded_vars.keys())
+                df, _ = prepare_stats(
+                    self.context.frame_stats, recorded_cols=recorded_cols
+                )
+                self.callback(self.context, df)
 
             else:
                 # I can't use the "animate" reactive approach here because

catalyst/exchange/poloniex/poloniex.py

@@ -1,661 +0,0 @@
-import json
-import time
-from collections import defaultdict
-
-import numpy as np
-import pandas as pd
-import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-# import six
-from six import iteritems
-
-from catalyst.constants import LOG_LEVEL
-# from websocket import create_connection
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle,
-    OrphanOrderError,
-    OrphanOrderReverseError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.protocol import Account
-from catalyst.utils.deprecate import deprecated
-
-log = Logger('Poloniex', level=LOG_LEVEL)
-
-
-@deprecated
-class Poloniex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret)
-        self.name = 'poloniex'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-        self.transactions = defaultdict(list)
-
-        self.num_candles_limit = 2000
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from the Exchange order dictionary
-        :param order_status:
-        :return: Order
-        """
-        # if order_status['is_cancelled']:
-        #    status = ORDER_STATUS.CANCELLED
-        # elif not order_status['is_live']:
-        #    log.info('found executed order {}'.format(order_status))
-        #    status = ORDER_STATUS.FILLED
-        # else:
-        status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['amount'])
-        # filled = float(order_status['executed_amount'])
-        filled = None
-
-        if order_status['type'] == 'sell':
-            amount = -amount
-            # filled = -filled
-
-        price = float(order_status['rate'])
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        # if order_type.endswith('limit'):
-        #    limit_price = price
-        # elif order_type.endswith('stop'):
-        #    stop_price = price
-
-        # executed_price = float(order_status['avg_execution_price'])
-        executed_price = price
-
-        # TODO: Set Poloniex comission
-        commission = None
-
-        # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date=pytz.utc.localize(date)
-        date = None
-
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            # No such field in Poloniex
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['orderNumber']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        balances = self.api.returnbalances()
-        try:
-            log.debug('retrieving wallets balances')
-        except Exception as e:
-            log.debug(e)
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['error'])
-            )
-
-        std_balances = dict()
-        for (key, value) in iteritems(balances):
-            currency = key.lower()
-            std_balances[currency] = float(value)
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Poloniex
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '5m', '15m', '30m', '2h', '4h', '1D'
-        """
-
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T' and (bar_count == 1 or bar_count is None):
-            # TODO: use the order book instead
-            # We use the 5m to fetch the last bar
-            frequency = 300
-        elif freq == '5T':
-            frequency = 300
-        elif freq == '15T':
-            frequency = 900
-        elif freq == '30T':
-            frequency = 1800
-        elif freq == '120T':
-            frequency = 7200
-        elif freq == '240T':
-            frequency = 14400
-        elif freq == '1D':
-            frequency = 86400
-        else:
-            # Poloniex does not offer 1m data candles
-            # It is likely to error out there frequently
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-
-        for asset in asset_list:
-            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
-            end = int(delta.total_seconds())
-
-            if bar_count is None:
-                start = end - 2 * frequency
-            else:
-                start = end - bar_count * frequency
-
-            try:
-                response = self.api.returnchartdata(
-                    self.get_symbol(asset), frequency, start, end
-                )
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-                ohlc = dict(
-                    open=np.float64(candle['open']),
-                    high=np.float64(candle['high']),
-                    low=np.float64(candle['low']),
-                    close=np.float64(candle['close']),
-                    volume=np.float64(candle['volume']),
-                    price=np.float64(candle['close']),
-                    last_traded=last_traded
-                )
-
-                return ohlc
-
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(response[0])
-            else:
-                ohlc_bars = []
-                for candle in response:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-
-        if (isinstance(style, ExchangeLimitOrder)
-                or isinstance(style, ExchangeStopLimitOrder)):
-            if isinstance(style, ExchangeStopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-
-            try:
-                if (is_buy):
-                    response = self.api.buy(exchange_symbol, amount, price)
-                else:
-                    response = self.api.sell(exchange_symbol, -amount, price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            date = pd.Timestamp.utcnow()
-
-            if ('orderNumber' in response):
-                order_id = str(response['orderNumber'])
-                order = Order(
-                    dt=date,
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                log.warn(
-                    '{} order failed: {}'.format('buy' if is_buy else 'sell',
-                                                 response['error']))
-                return None
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset='all'):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not 'all', return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If 'all' is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-
-        return self.portfolio.open_orders
-
-        """
-        TODO: Why going to the exchange if we already have this info locally?
-              And why creating all these Orders if we later discard them?
-        """
-
-        try:
-            if (asset == 'all'):
-                response = self.api.returnopenorders('all')
-            else:
-                response = self.api.returnopenorders(self.get_symbol(asset))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    response['message'])
-            )
-
-        print(self.portfolio.open_orders)
-
-        # TODO: Need to handle openOrders for 'all'
-        orders = list()
-        for order_status in response:
-            # will Throw error b/c Polo doesn't track order['symbol']
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-
-        try:
-            order = self._portfolio.open_orders[order_id]
-        except Exception as e:
-            raise OrphanOrderError(order_id=order_id, exchange=self.name)
-
-        return order
-
-        # TODO: Need to decide whether we fetch orders locally or from exchnage
-        # The code below is ignored
-
-        try:
-            response = self.api.returnopenorders(self.get_symbol(order.sid))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        for o in response:
-            if (int(o['orderNumber']) == int(order_id)):
-                return order
-
-        return None
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-
-        if (isinstance(order_param, Order)):
-            order = order_param
-        else:
-            order = self._portfolio.open_orders[order_param]
-
-        try:
-            response = self.api.cancelorder(order.id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} '
-                '{error}.'.format(
-                    order_id=order.id,
-                    exchange=self.name,
-                    error=response['error']
-                ))
-
-            # raise OrderCancelError(
-            #    order_id=order.id,
-            #    exchange=self.name,
-            #    error=response['error']
-            # )
-
-        self.portfolio.remove_order(order)
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self.get_symbols(assets)
-
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            response = self.api.returnticker()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response['error'])
-            )
-
-        ticks = dict()
-
-        for index, symbol in enumerate(symbols):
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=float(response[symbol]['highestBid']),
-                ask=float(response[symbol]['lowestAsk']),
-                last_price=float(response[symbol]['last']),
-                low=float(response[symbol]['lowestAsk']),
-                # TODO: Polo does not provide low
-                high=float(response[symbol]['highestBid']),
-                # TODO: Polo does not provide high
-                volume=float(response[symbol]['baseVolume']),
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self.api.returnticker()
-
-        for exchange_symbol in response:
-            base, market = self.sanitize_curency_symbol(exchange_symbol).split(
-                '_')
-            symbol = '{market}_{base}'.format(market=market, base=base)
-
-            if (source_dates):
-                start_date = self.get_symbol_start_date(exchange_symbol)
-            else:
-                try:
-                    start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-        try:
-            r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
-                '2010-1-1').value // 10 ** 9)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
-
-    def check_open_orders(self):
-        """
-        Need to override this function for Poloniex:
-
-        Loop through the list of open orders in the Portfolio object.
-        Check if any transactions have been executed:
-            If so, create a transaction and apply to the Portfolio.
-        Check if the order is still open:
-            If not, remove it from open orders
-
-        :return:
-        transactions: Transaction[]
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-
-                order = self._portfolio.open_orders[order_id]
-                log.debug('found open order: {}'.format(order_id))
-
-                try:
-                    order_open = self.get_order(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if (order_open):
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta)
-                    )
-
-                try:
-                    response = self.api.returnordertrades(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if ('error' in response):
-                    if (not order_open):
-                        raise OrphanOrderReverseError(order_id=order_id,
-                                                      exchange=self.name)
-                else:
-                    for tx in response:
-                        """
-                        We maintain a list of dictionaries of transactions that
-                        correspond to partially filled orders, indexed by
-                        order_id. Every time we query executed transactions
-                        from the exchange, we check if we had that transaction
-                        for that order already. If not, we process it.
-
-                        When an order if fully filled, we flush the dict of
-                        transactions associated with that order.
-                        """
-                        if (not filter(
-                                lambda item: item['order_id'] == tx['tradeID'],
-                                self.transactions[order_id])):
-                            log.debug(
-                                'Got new transaction for order {}: amount {}, '
-                                'price {}'.format(
-                                    order_id, tx['amount'], tx['rate']))
-                            tx['amount'] = float(tx['amount'])
-                            if (tx['type'] == 'sell'):
-                                tx['amount'] = -tx['amount']
-                            transaction = Transaction(
-                                asset=order.asset,
-                                amount=tx['amount'],
-                                dt=pd.to_datetime(tx['date'], utc=True),
-                                price=float(tx['rate']),
-                                order_id=tx['tradeID'],
-                                # it's a misnomer, but keep for compatibility
-                                commission=float(tx['fee'])
-                            )
-                            self.transactions[order_id].append(transaction)
-                            self.portfolio.execute_transaction(transaction)
-                            transactions.append(transaction)
-
-                    if (not order_open):
-                        """
-                            Since transactions have been executed individually
-                            the only thing left to do is remove them from list
-                            of open_orders
-                        """
-                        del self.portfolio.open_orders[order_id]
-                        del self.transactions[order_id]
-
-        return transactions
-
-    def get_orderbook(self, asset, order_type='all'):
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.returnOrderBook(market=exchange_symbol)
-
-        result = dict()
-        for order_type in data:
-            # TODO: filter by type
-            if order_type != 'asks' and order_type != 'bids':
-                continue
-
-            result[order_type] = []
-            for entry in data[order_type]:
-                if len(entry) == 2:
-                    result[order_type].append(
-                        dict(
-                            rate=float(entry[0]),
-                            quantity=float(entry[1])
-                        )
-                    )
-        return result

catalyst/exchange/poloniex/poloniex_api.py

@@ -1,212 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-from six.moves import urllib
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-urlopen = urllib.request.urlopen
-
-
-class Poloniex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-
-        self.max_requests_per_second = 6
-        self.request_cpt = dict()
-
-        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                       'returnTradeHistory', 'returnChartData',
-                       'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances', 'returnCompleteBalances',
-                        'returnDepositAddresses',
-                        'generateNewAddress', 'returnDepositsWithdrawals',
-                        'returnOpenOrders',
-                        'returnTradeHistory', 'returnOrderTrades',
-                        'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo',
-                        'returnAvailableAccountBalances',
-                        'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary', 'marginBuy',
-                        'marginSell',
-                        'getMarginPosition', 'closeMarginPosition',
-                        'createLoanOffer',
-                        'cancelLoanOffer', 'returnOpenLoanOffers',
-                        'returnActiveLoans',
-                        'returnLendingHistory', 'toggleAutoRenew']
-
-    def ask_request(self):
-        """
-        Asks permission to issue a request to the exchange.
-        The primary purpose is to avoid hitting rate limits.
-
-        The application will pause if the maximum requests per minute
-        permitted by the exchange is exceeded.
-
-        :return boolean:
-
-        """
-        now = time.time()
-        if not self.request_cpt:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        cpt_date = list(self.request_cpt.keys())[0]
-        cpt = self.request_cpt[cpt_date]
-
-        if now > cpt_date + 1:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        if cpt >= self.max_requests_per_second:
-
-            time.sleep(1)
-
-            now = time.time()
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-        else:
-            self.request_cpt[cpt_date] += 1
-
-    def query(self, method, req={}):
-
-        if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + \
-                  urllib.parse.urlencode(req)
-            headers = {}
-            post_data = None
-        elif method in self.trading:
-            url = 'https://poloniex.com/tradingApi'
-            req['command'] = method
-            req['nonce'] = int(time.time() * 1000)
-            post_data = urllib.parse.urlencode(req)
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 post_data.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'Sign': signature, 'Key': self.key}
-
-            post_data = post_data.encode('utf-8')
-        else:
-            raise ValueError(
-                'Method "' + method + '" not found in neither the Public API '
-                                      'or Trading API endpoints'
-            )
-
-        self.ask_request()
-        req = urllib.request.Request(
-            url,
-            data=post_data,
-            headers=headers,
-        )
-        resource = urlopen(req, context=ssl._create_unverified_context())
-        content = resource.read().decode('utf-8')
-        return json.loads(content)
-
-    def returnticker(self):
-        return self.query('returnTicker', {})
-
-    def return24volume(self):
-        return self.query('return24Volume', {})
-
-    def returnOrderBook(self, market='all'):
-        return self.query('returnOrderBook', {'currencyPair': market})
-
-    def returntradehistory(self, market, start=None, end=None):
-        if (start is not None and end is not None):
-            return self.query('returntradehistory',
-                              {'currencyPair': market, 'start': start,
-                               'end': end})
-        else:
-            return self.query('returntradehistory', {'currencyPair': market})
-
-    def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData',
-                          {'currencyPair': market, 'period': period,
-                           'start': start, 'end': end})
-
-    def returncurrencies(self):
-        return self.query('returnCurrencies', {})
-
-    def returnloadorders(self, market):
-        return self.query('returnLoanOrders', {'currency': market})
-
-    def returnbalances(self):
-        return self.query('returnBalances')
-
-    def returncompletebalances(self, account):
-        if (account):
-            return self.query('returnCompleteBalances', {'account': account})
-        else:
-            return self.query('returnCompleteBalances')
-
-    def returndepositaddresses(self):
-        return self.query('returnDepositAddresses')
-
-    def generatenewaddress(self, currency):
-        return self.query('generateNewAddress', {'currency': currency})
-
-    def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals',
-                          {'start': start, 'end': end})
-
-    def returnopenorders(self, market):
-        return self.query('returnOpenOrders', {'currencyPair': market})
-
-    def returnordertrades(self, ordernumber):
-        return self.query('returnOrderTrades', {'orderNumber': ordernumber})
-
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-            postonly=0):
-        if (fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'postOnly': postonly, })
-        else:
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount, })
-
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-             postonly=0):
-        if (fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'immediateOrCancel': immediateorcancel})
-        elif (postonly):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'postOnly': postonly, })
-        else:
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount, })
-
-    def cancelorder(self, ordernumber):
-        return self.query('cancelOrder', {'orderNumber': ordernumber})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'amount': quantity,
-                           'address': address})
-
-    def returnfeeinfo(self):
-        return self.query('returnFeeInfo')

catalyst/exchange/simple_clock.py

@@ -14,14 +14,13 @@
 from time import sleep
 
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 

catalyst/exchange/utils/bundle_utils.py

@@ -0,0 +1,160 @@
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.constants import BUNDLE_URL
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+
+EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
+API_URL = 'http://data.enigma.co/api/v1'
+
+
+def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
+    """
+    Download and extract a bcolz bundle.
+
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
+
+    Returns
+    -------
+    str
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+
+    """
+    root = get_exchange_bundles_folder(exchange_name)
+    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
+        exchange=exchange_name,
+        frequency=data_frequency,
+        symbol=symbol,
+        period=period
+    )
+    path = os.path.join(root, name)
+
+    if not os.path.isdir(path):
+        url = BUNDLE_URL.format(
+            exchange=exchange_name,
+            data_frequency=data_frequency,
+            name=name,
+        )
+
+        bytes = download_without_progress(url)
+        with tarfile.open('r', fileobj=bytes) as tar:
+            tar.extractall(path)
+
+    return path
+
+
+def get_df_from_arrays(arrays, periods):
+    """
+    A DataFrame from the specified OHCLV arrays.
+
+    Parameters
+    ----------
+    arrays: Object
+    periods: DateTimeIndex
+
+    Returns
+    -------
+    DataFrame
+
+    """
+    ohlcv = dict()
+    for index, field in enumerate(['open', 'high', 'low', 'close', 'volume']):
+        ohlcv[field] = arrays[index].flatten()
+
+    df = pd.DataFrame(
+        data=ohlcv,
+        index=periods
+    )
+    df.index.name = 'last_traded'
+    return df
+
+
+def range_in_bundle(asset, start_dt, end_dt, reader):
+    """
+    Evaluate whether price data of an asset is included has been ingested in
+    the exchange bundle for the given date range.
+
+    Parameters
+    ----------
+    asset: TradingPair
+    start_dt: datetime
+    end_dt: datetime
+    reader: BcolzBarMinuteReader
+
+    Returns
+    -------
+    bool
+
+    """
+    has_data = True
+    dates = [start_dt, end_dt]
+
+    while dates and has_data:
+        try:
+            dt = dates.pop(0)
+            close = reader.get_value(asset.sid, dt, 'close')
+
+            if np.isnan(close):
+                has_data = False
+
+        except Exception:
+            has_data = False
+
+    return has_data
+
+
+def get_assets(exchange, include_symbols, exclude_symbols):
+    """
+    Get assets from an exchange, including or excluding the specified
+    symbols.
+
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
+
+        return exchange.get_assets(include_symbols_list)
+
+    else:
+        all_assets = exchange.get_assets()
+
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/utils/ccxt_utils.py

@@ -0,0 +1,307 @@
+import json
+import os
+import pandas as pd
+from six.moves.urllib import request
+
+from catalyst.assets._assets import TradingPair
+from ccxt import NetworkError
+from catalyst.constants import LOG_LEVEL, EXCHANGE_CONFIG_URL
+from catalyst.exchange.exchange_errors import MarketsNotFoundError, \
+    InvalidMarketError
+from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol, \
+    get_exchange_folder, get_exchange_auth
+from catalyst.exchange.utils.serialization_utils import ExchangeJSONDecoder, \
+    ExchangeJSONEncoder
+from logbook import Logger
+from redo import retry
+from ccxt.base.exchange import Exchange
+from catalyst.utils.paths import last_modified_time, data_root, \
+    ensure_directory
+import ccxt
+
+log = Logger('ccxt_utils', level=LOG_LEVEL)
+
+
+def scan_exchange_configs(features=None, history=None, is_authenticated=False,
+                          path=None):
+    """
+    Finding exchanges from their config files
+
+    Parameters
+    ----------
+    features
+    is_authenticated
+
+    Returns
+    -------
+
+    """
+    for exchange_name in ccxt.exchanges:
+        config = get_exchange_config(exchange_name, path)
+        if not config or 'error' in config:
+            log.info(
+                'skipping invalid exchange {}'.format(exchange_name)
+            )
+
+        # Check if the exchange has an auth.json file
+        if is_authenticated:
+            exchange_auth = get_exchange_auth(exchange_name)
+            has_auth = (exchange_auth['key'] != ''
+                        and exchange_auth['secret'] != '')
+
+            if not has_auth:
+                continue
+
+        if features is None:
+            has_features = True
+
+        else:
+            try:
+                supported_features = [
+                    feature for feature in features if
+                    feature in config['features']
+                ]
+                has_features = len(supported_features) > 0
+            except Exception:
+                has_features = False
+
+        # TODO: filter by history
+        if has_features:
+            yield config
+
+
+def get_exchange_config(exchange_name, path=None, environ=None,
+                        expiry='1H'):
+    """
+    The de-serialized content of the exchange's config.json.
+    Parameters
+    ----------
+    exchange_name: str
+        The exchange name
+    filename: str
+        The target file
+    environ:
+
+    Returns
+    -------
+    config: dict[srt, Object]
+        The config dictionary.
+
+    """
+    try:
+        if path is None:
+            root = data_root(environ)
+            path = os.path.join(root, 'exchanges')
+
+        folder = os.path.join(path, exchange_name)
+        ensure_directory(folder)
+
+        filename = os.path.join(folder, 'config.json')
+        url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
+        if os.path.isfile(filename):
+            # If the file exists, only update periodically to avoid
+            # unnecessary calls
+            now = pd.Timestamp.utcnow()
+            limit = pd.Timedelta(expiry)
+            if pd.Timedelta(now - last_modified_time(filename)) > limit:
+                try:
+                    request.urlretrieve(url=url, filename=filename)
+                except Exception as e:
+                    log.warn(
+                        'unable to update config {} => {}: {}'.format(
+                            url, filename, e
+                        )
+                    )
+
+        else:
+            request.urlretrieve(url=url, filename=filename)
+
+        with open(filename) as data_file:
+            data = json.load(data_file, cls=ExchangeJSONDecoder)
+            return data
+
+    except Exception as e:
+        log.warn(
+            'unable to download {} config: {}'.format(
+                exchange_name, e
+            )
+        )
+        return dict(error=e)
+
+
+def save_exchange_config(config, filename=None, environ=None):
+    """
+    Save assets into an exchange_config file.
+    Parameters
+    ----------
+    exchange_name: str
+    config
+    environ
+    Returns
+    -------
+    """
+    if filename is None:
+        name = 'config.json'
+        exchange_folder = get_exchange_folder(config['id'], environ)
+        filename = os.path.join(exchange_folder, name)
+
+    with open(filename, 'w+') as handle:
+        json.dump(config, handle, indent=4, cls=ExchangeJSONEncoder)
+
+
+def fetch_markets(ccxt_exchange):
+    """
+    Fetches CCXT market objects.
+    
+    Parameters
+    ----------
+    ccxt_exchange: Exchange
+
+    Returns
+    -------
+
+    """
+    markets_symbols = ccxt_exchange.load_markets()
+    log.debug(
+        'fetching {} markets:\n{}'.format(
+            ccxt_exchange.name, markets_symbols
+        )
+    )
+    markets = ccxt_exchange.fetch_markets()
+
+    if not markets:
+        raise MarketsNotFoundError(
+            exchange=ccxt_exchange.name,
+        )
+
+    for market in markets:
+        if 'id' not in market:
+            raise InvalidMarketError(
+                exchange=ccxt_exchange.name,
+                market=market,
+            )
+    return markets
+
+
+def create_exchange_config(ccxt_exchange):
+    """
+    Creates an exchange config structure. 
+    
+    Parameters
+    ----------
+    ccxt_exchange: Exchange
+
+    Returns
+    -------
+
+    """
+    exchange_name = ccxt_exchange.__class__.__name__
+    config = dict(
+        id=exchange_name,
+        name=ccxt_exchange.name,
+        features=[
+            feature for feature in ccxt_exchange.has if
+            ccxt_exchange.has[feature]
+        ]
+    )
+    markets = retry(
+        action=fetch_markets,
+        attempts=5,
+        sleeptime=5,
+        retry_exceptions=(NetworkError,),
+        cleanup=lambda: log.warn(
+            'fetching markets again for {}'.format(exchange_name)
+        ),
+        args=(ccxt_exchange,)
+    )
+
+    config['assets'] = []
+    for market in markets:
+        asset = create_trading_pair(exchange_name, market)
+        config['assets'].append(asset)
+
+    return config
+
+
+def create_trading_pair(exchange_name, market, start_dt=None, end_dt=None,
+                        leverage=1, end_daily=None, end_minute=None):
+    """
+    Creating a TradingPair from market and asset data.
+
+    Parameters
+    ----------
+    market: dict[str, Object]
+    start_dt
+    end_dt
+    leverage
+    end_daily
+    end_minute
+
+    Returns
+    -------
+
+    """
+    params = dict(
+        exchange=exchange_name,
+        data_source='catalyst',
+        exchange_symbol=market['id'],
+        symbol=get_catalyst_symbol(market),
+        start_date=start_dt,
+        end_date=end_dt,
+        leverage=leverage,
+        asset_name=market['symbol'],
+        end_daily=end_daily,
+        end_minute=end_minute,
+    )
+    apply_conditional_market_params(exchange_name, params, market)
+
+    return TradingPair(**params)
+
+
+def apply_conditional_market_params(exchange_name, params, market):
+    """
+    Applies a CCXT market dict to parameters of TradingPair init.
+
+    Parameters
+    ----------
+    params: dict[Object]
+    market: dict[Object]
+
+    Returns
+    -------
+
+    """
+    # TODO: make this more externalized / configurable
+    # Consider representing in some type of JSON structure
+    if 'active' in market:
+        params['trading_state'] = 1 if market['active'] else 0
+
+    else:
+        params['trading_state'] = 1
+
+    if 'lot' in market:
+        params['min_trade_size'] = market['lot']
+        params['lot'] = market['lot']
+
+    if exchange_name == 'bitfinex':
+        params['maker'] = 0.001
+        params['taker'] = 0.002
+
+    elif 'maker' in market and 'taker' in market \
+        and market['maker'] is not None \
+        and market['taker'] is not None:
+        params['maker'] = market['maker']
+        params['taker'] = market['taker']
+
+    else:
+        # TODO: default commission, make configurable
+        params['maker'] = 0.0015
+        params['taker'] = 0.0025
+
+    info = market['info'] if 'info' in market else None
+    if info:
+        if 'minimum_order_size' in info:
+            params['min_trade_size'] = float(info['minimum_order_size'])
+
+            if 'lot' not in params:
+                params['lot'] = params['min_trade_size']

catalyst/exchange/utils/datetime_utils.py

@@ -1,17 +1,12 @@
 import calendar
-import os
-import tarfile
-from datetime import timedelta, datetime, date
+import re
+from datetime import datetime, timedelta, date
 
-import numpy as np
 import pandas as pd
 import pytz
 
-from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
-
-EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
-API_URL = 'http://data.enigma.co/api/v1'
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
+    InvalidHistoryFrequencyAlias
 
 
 def get_date_from_ms(ms):
@@ -49,45 +44,6 @@ def get_seconds_from_date(date):
     return int((date - epoch).total_seconds())
 
 
-def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
-    """
-    Download and extract a bcolz bundle.
-
-    Parameters
-    ----------
-    exchange_name: str
-    symbol: str
-    data_frequency: str
-    period: str
-
-    Returns
-    -------
-    str
-        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-
-    """
-    root = get_exchange_bundles_folder(exchange_name)
-    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
-        exchange=exchange_name,
-        frequency=data_frequency,
-        symbol=symbol,
-        period=period
-    )
-    path = os.path.join(root, name)
-
-    if not os.path.isdir(path):
-        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
-              'exchange-{exchange}/{name}.tar.gz'.format(
-                exchange=exchange_name,
-                name=name)
-
-        bytes = download_without_progress(url)
-        with tarfile.open('r', fileobj=bytes) as tar:
-            tar.extractall(path)
-
-    return path
-
-
 def get_delta(periods, data_frequency):
     """
     Get a time delta based on the specified data frequency.
@@ -106,7 +62,7 @@ def get_delta(periods, data_frequency):
         if data_frequency == 'minute' else timedelta(days=periods)
 
 
-def get_periods_range(start_dt, end_dt, freq):
+def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
     """
     Get a date range for the specified parameters.
 
@@ -127,7 +83,38 @@ def get_periods_range(start_dt, end_dt, freq):
     elif freq == 'daily':
         freq = 'D'
 
-    return pd.date_range(start_dt, end_dt, freq=freq)
+    if start_dt is not None and end_dt is not None and periods is None:
+
+        return pd.date_range(start_dt, end_dt, freq=freq)
+
+    elif periods is not None and (start_dt is not None or end_dt is not None):
+        _, unit_periods, unit, _ = get_frequency(freq)
+        adj_periods = periods * unit_periods
+
+        # TODO: standardize time aliases to avoid any mapping
+        unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
+        delta = pd.Timedelta(adj_periods, unit)
+
+        if start_dt is not None:
+            return pd.date_range(
+                start=start_dt,
+                end=start_dt + delta,
+                freq=freq,
+                closed='left',
+            )
+
+        else:
+            return pd.date_range(
+                start=end_dt - delta,
+                end=end_dt,
+                freq=freq,
+            )
+
+    else:
+        raise ValueError(
+            'Choose only two parameters between start_dt, end_dt '
+            'and periods.'
+        )
 
 
 def get_periods(start_dt, end_dt, freq):
@@ -145,7 +132,7 @@ def get_periods(start_dt, end_dt, freq):
     int
 
     """
-    return len(get_periods_range(start_dt, end_dt, freq))
+    return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
 
 
 def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
@@ -157,6 +144,7 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     end_dt: datetime
     bar_count: int
     data_frequency: str
+    include_first
 
     Returns
     -------
@@ -176,6 +164,12 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     return start_dt
 
 
+def timestr_to_dt(timestr, data_frequency):
+    dt_format = '%Y' if data_frequency == 'daily' else '%Y%m'
+    dt = pd.to_datetime(timestr, format=dt_format, utc=True)
+    return dt
+
+
 def get_period_label(dt, data_frequency):
     """
     The period label for the specified date and frequency.
@@ -189,6 +183,26 @@ def get_period_label(dt, data_frequency):
     -------
     str
 
+    """
+    if data_frequency == 'minute':
+        return '{}{:02d}'.format(dt.year, dt.month)
+    else:
+        return '{}'.format(dt.year)
+
+
+def get_period(dt, data_frequency):
+    """
+    The period label for the specified date and frequency.
+
+    Parameters
+    ----------
+    dt: datetime
+    data_frequency: str
+
+    Returns
+    -------
+    str
+
     """
     if data_frequency == 'minute':
         return '{}-{:02d}'.format(dt.year, dt.month)
@@ -260,99 +274,81 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_df_from_arrays(arrays, periods):
+def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
     """
-    A DataFrame from the specified OHCLV arrays.
+    Get the frequency parameters.
+
+    Notes
+    -----
+    We're trying to use Pandas convention for frequency aliases.
 
     Parameters
     ----------
-    arrays: Object
-    periods: DateTimeIndex
+    freq: str
+    data_frequency: str
 
     Returns
     -------
-    DataFrame
+    str, int, str, str
 
     """
-    ohlcv = dict()
-    for index, field in enumerate(
-            ['open', 'high', 'low', 'close', 'volume']):
-        ohlcv[field] = arrays[index].flatten()
+    if data_frequency is None:
+        data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
 
-    df = pd.DataFrame(
-        data=ohlcv,
-        index=periods
-    )
-    return df
+    if freq == 'minute':
+        unit = 'T'
+        candle_size = 1
 
-
-def range_in_bundle(asset, start_dt, end_dt, reader):
-    """
-    Evaluate whether price data of an asset is included has been ingested in
-    the exchange bundle for the given date range.
-
-    Parameters
-    ----------
-    asset: TradingPair
-    start_dt: datetime
-    end_dt: datetime
-    reader: BcolzBarMinuteReader
-
-    Returns
-    -------
-    bool
-
-    """
-    has_data = True
-    dates = [start_dt, end_dt]
-
-    while dates and has_data:
-        try:
-            dt = dates.pop(0)
-            close = reader.get_value(asset.sid, dt, 'close')
-
-            if np.isnan(close):
-                has_data = False
-
-        except Exception:
-            has_data = False
-
-    return has_data
-
-
-def get_assets(exchange, include_symbols, exclude_symbols):
-    """
-    Get assets from an exchange, including or excluding the specified
-    symbols.
-
-    Parameters
-    ----------
-    exchange: Exchange
-    include_symbols: str
-    exclude_symbols: str
-
-    Returns
-    -------
-    list[TradingPair]
-
-    """
-    if include_symbols is not None:
-        include_symbols_list = include_symbols.split(',')
-
-        return exchange.get_assets(include_symbols_list)
+    elif freq == 'daily':
+        unit = 'D'
+        candle_size = 1
 
     else:
-        all_assets = exchange.get_assets()
-
-        if exclude_symbols is not None:
-            exclude_symbols_list = exclude_symbols.split(',')
-
-            assets = []
-            for asset in all_assets:
-                if asset.symbol not in exclude_symbols_list:
-                    assets.append(asset)
-
-            return assets
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
+                else 1
+            unit = freq_match.group(2)
 
         else:
-            return all_assets
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+    # TODO: some exchanges support H and W frequencies but not bundles
+    # Find a way to pass-through these parameters to exchanges
+    # but resample from minute or daily in backtest mode
+    # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
+    # Pandas offet aliases (used by Catalyst) and the CCXT timeframes
+    if unit.lower() == 'd':
+        unit = 'D'
+        alias = '{}D'.format(candle_size)
+
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm' or unit == 'T':
+        unit = 'T'
+        alias = '{}T'.format(candle_size)
+        data_frequency = 'minute'
+
+    elif unit.lower() == 'h':
+        if 'H' in supported_freqs:
+            unit = 'H'
+            alias = '{}H'.format(candle_size)
+
+        else:
+            candle_size = candle_size * 60
+            alias = '{}T'.format(candle_size)
+            data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyAlias(freq=freq)
+
+    return alias, candle_size, unit, data_frequency
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
+
+
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)

catalyst/exchange/utils/exchange_utils.py

@@ -1,19 +1,19 @@
 import hashlib
-import json
 import os
-import pickle
-import re
 import shutil
-from datetime import date, datetime
 
+import json
 import pandas as pd
+import pickle
 from catalyst.assets._assets import TradingPair
+from datetime import date, datetime
 from six import string_types
 from six.moves.urllib import request
 
-from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
-    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
+from catalyst.constants import EXCHANGE_CONFIG_URL
+from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
+    ExchangeJSONDecoder, ConfigJSONEncoder
+from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
@@ -62,7 +62,14 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
-def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
+def is_blacklist(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'blacklist.txt')
+
+    return os.path.exists(filename)
+
+
+def get_exchange_config_filename(exchange_name, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
 
@@ -76,12 +83,12 @@ def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     str
 
     """
-    name = 'symbols.json' if not is_local else 'symbols_local.json'
+    name = 'config.json'
     exchange_folder = get_exchange_folder(exchange_name, environ)
     return os.path.join(exchange_folder, name)
 
 
-def download_exchange_symbols(exchange_name, environ=None):
+def download_exchange_config(exchange_name, filename, environ=None):
     """
     Downloads the exchange's symbols.json from the repository.
 
@@ -95,29 +102,14 @@ def download_exchange_symbols(exchange_name, environ=None):
     str
 
     """
-    filename = get_exchange_symbols_filename(exchange_name)
-    url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = request.urlretrieve(url=url, filename=filename)
-    return response
+    url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
+    request.urlretrieve(url=url, filename=filename)
 
 
-def symbols_parser(asset_def):
-    for key, value in asset_def.items():
-        match = isinstance(value, string_types) \
-                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
-
-        if match:
-            try:
-                asset_def[key] = pd.to_datetime(value, utc=True)
-            except ValueError:
-                pass
-
-    return asset_def
-
-
-def get_exchange_symbols(exchange_name, is_local=False, environ=None):
+@deprecated
+def get_exchange_config(exchange_name, filename=None, environ=None):
     """
-    The de-serialized content of the exchange's symbols.json.
+    The de-serialized content of the exchange's config.json.
 
     Parameters
     ----------
@@ -130,52 +122,48 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     Object
 
     """
-    filename = get_exchange_symbols_filename(exchange_name, is_local)
-
-    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
-                pd.Timestamp('now', tz='UTC') - last_modified_time(
-                    filename)).days > 1):
-        download_exchange_symbols(exchange_name, environ)
+    if filename is None:
+        filename = get_exchange_config_filename(exchange_name)
 
     if os.path.isfile(filename):
-        with open(filename) as data_file:
-            try:
-                data = json.load(data_file, object_hook=symbols_parser)
-                return data
+        now = pd.Timestamp.utcnow()
+        limit = pd.Timedelta('2H')
+        if pd.Timedelta(now - last_modified_time(filename)) > limit:
+            download_exchange_config(exchange_name, filename, environ)
 
-            except ValueError:
-                return dict()
     else:
-        raise ExchangeSymbolsNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
+        download_exchange_config(exchange_name, filename, environ)
+
+    with open(filename) as data_file:
+        try:
+            data = json.load(data_file, cls=ExchangeJSONDecoder)
+            return data
+
+        except ValueError:
+            return dict()
 
 
-def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
+def save_exchange_config(exchange_name, config, filename=None, environ=None):
     """
-    Save assets into an exchange_symbols file.
+    Save assets into an exchange_config file.
 
     Parameters
     ----------
     exchange_name: str
-    assets: list[dict[str, object]]
-    is_local: bool
+    config
     environ
 
     Returns
     -------
 
     """
-    asset_dicts = dict()
-    for symbol in assets:
-        asset_dicts[symbol] = assets[symbol].to_dict()
+    if filename is None:
+        name = 'config.json'
+        exchange_folder = get_exchange_folder(exchange_name, environ)
+        filename = os.path.join(exchange_folder, name)
 
-    filename = get_exchange_symbols_filename(
-        exchange_name, is_local, environ
-    )
-    with open(filename, 'wt') as handle:
-        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
+    with open(filename, 'w+') as handle:
+        json.dump(config, handle, indent=4, cls=ConfigJSONEncoder)
 
 
 def get_symbols_string(assets):
@@ -195,7 +183,7 @@ def get_symbols_string(assets):
     return ', '.join([asset.symbol for asset in array])
 
 
-def get_exchange_auth(exchange_name, environ=None):
+def get_exchange_auth(exchange_name, alias=None, environ=None):
     """
     The de-serialized contend of the exchange's auth.json file.
 
@@ -210,7 +198,8 @@ def get_exchange_auth(exchange_name, environ=None):
 
     """
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'auth.json')
+    name = 'auth' if alias is None else alias
+    filename = os.path.join(exchange_folder, '{}.json'.format(name))
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
@@ -266,7 +255,7 @@ def get_algo_folder(algo_name, environ=None):
     return algo_folder
 
 
-def get_algo_object(algo_name, key, environ=None, rel_path=None):
+def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     """
     The de-serialized object of the algo name and key.
 
@@ -276,6 +265,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
     key: str
     environ:
     rel_path: str
+    how: str
 
     Returns
     -------
@@ -290,19 +280,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
 
-    filename = os.path.join(folder, key + '.p')
+    name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
+    filename = os.path.join(folder, name)
 
     if os.path.isfile(filename):
-        try:
+        if how == 'pickle':
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception:
-            return None
+
+        else:
+            with open(filename) as data_file:
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
+                return data
+
     else:
         return None
 
 
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
+                     how='pickle'):
     """
     Serialize and save an object by algo name and key.
 
@@ -313,6 +309,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
     obj: Object
     environ:
     rel_path: str
+    how: str
 
     """
     folder = get_algo_folder(algo_name, environ)
@@ -321,10 +318,15 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
-    filename = os.path.join(folder, key + '.p')
+    if how == 'json':
+        filename = os.path.join(folder, '{}.json'.format(key))
+        with open(filename, 'wt') as handle:
+            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
 
-    with open(filename, 'wb') as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
+    else:
+        filename = os.path.join(folder, '{}.p'.format(key))
+        with open(filename, 'wb') as handle:
+            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
@@ -384,6 +386,71 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
         df.to_csv(handle, encoding='UTF_8')
 
 
+def clear_frame_stats_directory(algo_name):
+    """
+    remove the outdated directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+
+    Returns
+    -------
+    error: str
+
+    """
+    error = None
+    algo_folder = get_algo_folder(algo_name)
+    folder = os.path.join(algo_folder, 'frame_stats')
+    if os.path.exists(folder):
+        try:
+            shutil.rmtree(folder)
+        except OSError:
+            error = 'unable to remove {}, the analyze ' \
+                    'data will be inconsistent'.format(folder)
+    return error
+
+
+def remove_old_files(algo_name, today, rel_path, environ=None):
+    """
+    remove old files from a directory
+    to avoid overloading the disk
+
+    Parameters
+    ----------
+    algo_name: str
+    today: Timestamp
+    rel_path: str
+    environ:
+
+    Returns
+    -------
+    error: str
+
+    """
+
+    error = None
+    algo_folder = get_algo_folder(algo_name, environ)
+    folder = os.path.join(algo_folder, rel_path)
+    ensure_directory(folder)
+
+    # run on all files in the folder
+    for f in os.listdir(folder):
+        try:
+            file_path = os.path.join(folder, f)
+            creation_unix = os.path.getctime(file_path)
+            creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
+
+            # if the file is older than 30 days erase it
+            if today - pd.DateOffset(30) > creation_time:
+                os.unlink(file_path)
+        except OSError:
+            error = 'unable to erase files in {}'.format(folder)
+
+    return error
+
+
 def get_exchange_minute_writer_root(exchange_name, environ=None):
     """
     The minute writer folder for the exchange.
@@ -428,23 +495,13 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
-def symbols_serial(obj):
-    """
-    JSON serializer for objects not serializable by default json code
+def has_bundle(exchange_name, data_frequency, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
 
-    Parameters
-    ----------
-    obj: Object
+    folder_name = '{}_bundle'.format(data_frequency.lower())
+    folder = os.path.join(exchange_folder, folder_name)
 
-    Returns
-    -------
-    str
-
-    """
-    if isinstance(obj, (datetime, date)):
-        return obj.floor('1D').strftime(DATE_FORMAT)
-
-    raise TypeError("Type %s not serializable" % type(obj))
+    return os.path.isdir(folder)
 
 
 def perf_serial(obj):
@@ -495,68 +552,7 @@ def get_common_assets(exchanges):
     return assets
 
 
-def get_frequency(freq, data_frequency):
-    """
-    Get the frequency parameters.
-
-    Notes
-    -----
-    We're trying to use Pandas convention for frequency aliases.
-
-    Parameters
-    ----------
-    freq: str
-    data_frequency: str
-
-    Returns
-    -------
-    str, int, str, str
-
-    """
-    if freq == 'minute':
-        unit = 'T'
-        candle_size = 1
-
-    elif freq == 'daily':
-        unit = 'D'
-        candle_size = 1
-
-    else:
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
-                else 1
-            unit = freq_match.group(2)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-    if unit.lower() == 'd':
-        alias = '{}D'.format(candle_size)
-
-        if data_frequency == 'minute':
-            data_frequency = 'daily'
-
-    elif unit.lower() == 'm' or unit == 'T':
-        alias = '{}T'.format(candle_size)
-
-        if data_frequency == 'daily':
-            data_frequency = 'minute'
-
-    # elif unit.lower() == 'h':
-    #     candle_size = candle_size * 60
-    #
-    #     alias = '{}T'.format(candle_size)
-    #     if data_frequency == 'daily':
-    #         data_frequency = 'minute'
-
-    else:
-        raise InvalidHistoryFrequencyAlias(freq=freq)
-
-    return alias, candle_size, unit, data_frequency
-
-
-def resample_history_df(df, freq, field):
+def resample_history_df(df, freq, field, start_dt=None):
     """
     Resample the OHCLV DataFrame using the specified frequency.
 
@@ -584,51 +580,19 @@ def resample_history_df(df, freq, field):
     else:
         raise ValueError('Invalid field.')
 
-    resampled_df = df.resample(freq).agg(agg)
+    resampled_df = df.resample(
+        freq, closed='left', label='left'
+    ).agg(agg)  # type: pd.DataFrame
+
+    # Because the samples are closed left, we get one more candle at
+    # the beginning then the requested number for bars. Removing this
+    # candle to avoid confusion.
+    if start_dt and not resampled_df.empty:
+        resampled_df = resampled_df[resampled_df.index >= start_dt]
+
     return resampled_df
 
 
-def mixin_market_params(exchange_name, params, market):
-    """
-    Applies a CCXT market dict to parameters of TradingPair init.
-
-    Parameters
-    ----------
-    params: dict[Object]
-    market: dict[Object]
-
-    Returns
-    -------
-
-    """
-    # TODO: make this more externalized / configurable
-    if 'lot' in market:
-        params['min_trade_size'] = market['lot']
-        params['lot'] = market['lot']
-
-    if exchange_name == 'bitfinex':
-        params['maker'] = 0.001
-        params['taker'] = 0.002
-
-    elif 'maker' in market and 'taker' in market \
-            and market['maker'] is not None and market['taker'] is not None:
-        params['maker'] = market['maker']
-        params['taker'] = market['taker']
-
-    else:
-        # TODO: default commission, make configurable
-        params['maker'] = 0.0015
-        params['taker'] = 0.0025
-
-    info = market['info'] if 'info' in market else None
-    if info:
-        if 'minimum_order_size' in info:
-            params['min_trade_size'] = float(info['minimum_order_size'])
-
-            if 'lot' not in params:
-                params['lot'] = params['min_trade_size']
-
-
 def from_ms_timestamp(ms):
     return pd.to_datetime(ms, unit='ms', utc=True)
 
@@ -646,3 +610,114 @@ def group_assets_by_exchange(assets):
         exchange_assets[asset.exchange].append(asset)
 
     return exchange_assets
+
+
+def get_catalyst_symbol(market_or_symbol):
+    """
+    The Catalyst symbol.
+
+    Parameters
+    ----------
+    market_or_symbol
+
+    Returns
+    -------
+
+    """
+    if isinstance(market_or_symbol, string_types):
+        parts = market_or_symbol.split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+    else:
+        return '{}_{}'.format(
+            market_or_symbol['base'].lower(),
+            market_or_symbol['quote'].lower(),
+        )
+
+
+def save_asset_data(folder, df, decimals=8):
+    symbols = df.index.get_level_values('symbol')
+    for symbol in symbols:
+        symbol_df = df.loc[(symbols == symbol)]  # Type: pd.DataFrame
+
+        filename = os.path.join(folder, '{}.csv'.format(symbol))
+        if os.path.exists(filename):
+            print_headers = False
+
+        else:
+            print_headers = True
+
+        with open(filename, 'a') as f:
+            symbol_df.to_csv(
+                path_or_buf=f,
+                header=print_headers,
+                float_format='%.{}f'.format(decimals),
+            )
+
+
+def forward_fill_df_if_needed(df, periods):
+    df = df.reindex(periods)
+    # volume should always be 0 (if there were no trades in this interval)
+    df['volume'] = df['volume'].fillna(0.0)
+    # ie pull the last close into this close
+    df['close'] = df.fillna(method='pad')
+    # now copy the close that was pulled down from the last timestep
+    # into this row, across into o/h/l
+    df['open'] = df['open'].fillna(df['close'])
+    df['low'] = df['low'].fillna(df['close'])
+    df['high'] = df['high'].fillna(df['close'])
+    return df
+
+
+def transform_candles_to_df(candles):
+    return pd.DataFrame(candles).set_index('last_traded')
+
+
+def get_candles_df(candles, field, freq, bar_count, end_dt):
+    all_series = dict()
+
+    for asset in candles:
+        asset_df = transform_candles_to_df(candles[asset])
+        rounded_end_dt = end_dt.floor(freq)
+        periods = pd.date_range(end=rounded_end_dt,
+                                periods=bar_count,
+                                freq=freq)
+        asset_df = forward_fill_df_if_needed(asset_df, periods)
+
+        all_series[asset] = pd.Series(asset_df[field])
+
+    df = pd.DataFrame(all_series)
+
+    df.dropna(inplace=True)
+
+    return df
+
+
+def get_trades_df(trades):
+    df = pd.DataFrame(trades)
+    df.index = pd.to_datetime(df.pop('datetime'))
+    df.index = df.index.tz_localize('UTC')
+
+    return df
+
+
+def candles_from_trades(trades_df, freq):
+    """
+    Calculate OHLCV from candles.
+
+    Parameters
+    ----------
+    trades_df
+    freq
+
+    Returns
+    -------
+
+    """
+    df = trades_df['price'].resample(freq).ohlc()  # type: pd.DataFrame
+    df['volume'] = trades_df['amount'].resample(freq).sum()
+
+    df.dropna(axis=0, how='all', inplace=True)
+    df.sort_index(inplace=True, ascending=False)
+
+    return df

catalyst/exchange/utils/factory.py

@@ -0,0 +1,109 @@
+import os
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.utils.ccxt_utils import scan_exchange_configs
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
+    get_exchange_folder
+from logbook import Logger
+
+log = Logger('factory', level=LOG_LEVEL)
+exchange_cache = dict()
+
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
+                 skip_init=False, auth_alias=None, config=None):
+    key = (exchange_name, base_currency)
+    if key in exchange_cache:
+        if not skip_init:
+            exchange_cache[key].init()
+
+        return exchange_cache[key]
+
+    exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
+        )
+
+    exchange = CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        password=exchange_auth['password'] if 'password'
+                                              in exchange_auth.keys() else '',
+        base_currency=base_currency,
+        config=config,
+    )
+    exchange_cache[key] = exchange
+
+    if not skip_init:
+        exchange.init()
+
+    return exchange
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges
+
+
+def find_exchanges(features=None, history=None, skip_blacklist=True, path=None,
+                   is_authenticated=False, base_currency=None):
+    """
+    Find exchanges filtered by a list of feature.
+
+    Parameters
+    ----------
+    features: str
+        The list of features.
+
+    skip_blacklist: bool
+    is_authenticated: bool
+    base_currency: bool
+
+    Returns
+    -------
+    list[Exchange]
+
+    """
+
+    return list(
+        scan_exchanges(
+            features,
+            history,
+            skip_blacklist,
+            path,
+            is_authenticated,
+            base_currency
+        )
+    )
+
+
+def scan_exchanges(features=None, history=None, skip_blacklist=True, path=None,
+                   is_authenticated=False, base_currency=None):
+    for config in scan_exchange_configs(
+        features=features,
+        history=history,
+        is_authenticated=is_authenticated,
+        path=path,
+    ):
+        if skip_blacklist and (config is None or 'error' in config):
+            continue
+
+        yield get_exchange(
+            exchange_name=config['id'],
+            skip_init=True,
+            base_currency=base_currency,
+            config=config,
+        )

catalyst/exchange/utils/live_chart_utils.py

@@ -0,0 +1,131 @@
+import matplotlib.dates as mdates
+import pandas as pd
+
+from catalyst.exchange.exchange_errors import \
+    MismatchingBaseCurrenciesExchanges
+
+fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+
+
+def format_ax(ax):
+    """
+    Trying to assign reasonable parameters to the time axis.
+
+    Parameters
+    ----------
+    ax:
+
+    """
+    # TODO: room for improvement
+    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
+    ax.xaxis.set_major_formatter(fmt)
+
+    locator = mdates.HourLocator(interval=4)
+    locator.MAXTICKS = 5000
+    ax.xaxis.set_minor_locator(locator)
+
+    datemin = pd.Timestamp.utcnow()
+    ax.set_xlim(datemin)
+
+    ax.grid(True)
+
+
+def set_legend(ax):
+    """
+    Set legend on the chart.
+
+    Parameters
+    ----------
+    ax
+
+    """
+    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
+
+
+def draw_pnl(ax, df):
+    """
+    Draw p&l line on the chart.
+
+    """
+    ax.clear()
+    ax.set_title('Performance')
+    index = df.index.unique()
+    dt = index.get_level_values(level=0)
+    pnl = index.get_level_values(level=4)
+    ax.plot(
+        dt, pnl, '-',
+        color='green',
+        linewidth=1.0,
+        label='Performance'
+    )
+
+    def perc(val):
+        return '{:2f}'.format(val)
+
+    ax.format_ydata = perc
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_custom_signals(ax, df):
+    """
+    Draw custom signals on the chart.
+
+    """
+    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
+
+    ax.clear()
+    ax.set_title('Custom Signals')
+    for index, column in enumerate(df.columns.values.tolist()):
+        ax.plot(df.index, df[column], '-',
+                color=colors[index],
+                linewidth=1.0,
+                label=column
+                )
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_exposure(ax, df, context):
+    """
+    Draw exposure line on the chart.
+
+    """
+    # TODO: list exchanges in graph
+    base_currency = None
+    positions = []
+    for exchange_name in context.exchanges:
+        exchange = context.exchanges[exchange_name]
+
+        if not base_currency:
+            base_currency = exchange.base_currency
+        elif base_currency != exchange.base_currency:
+            raise MismatchingBaseCurrenciesExchanges(
+                base_currency=base_currency,
+                exchange_name=exchange.name,
+                exchange_currency=exchange.base_currency
+            )
+
+        positions += exchange.portfolio.positions
+
+    ax.clear()
+    ax.set_title('Exposure')
+    ax.plot(df.index, df['base_currency'], '-',
+            color='green',
+            linewidth=1.0,
+            label='Base Currency: {}'.format(base_currency.upper())
+            )
+
+    symbols = []
+    for position in positions:
+        symbols.append(position.symbol)
+
+    ax.plot(df.index, df['long_exposure'], '-',
+            color='blue',
+            linewidth=1.0,
+            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
+
+    set_legend(ax)
+    format_ax(ax)

catalyst/exchange/utils/serialization_utils.py

@@ -0,0 +1,102 @@
+import json
+import re
+from json import JSONEncoder
+
+import pandas as pd
+from six import string_types
+
+from datetime import date, datetime
+from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT
+from catalyst.assets._assets import TradingPair
+
+
+class ConfigJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        """
+        JSON serializer for objects not serializable by default json code
+
+        Parameters
+        ----------
+        obj: Object
+
+        Returns
+        -------
+        str
+
+        """
+        if isinstance(obj, (datetime, date)):
+            return obj.floor('1D').strftime(DATE_FORMAT)
+
+        elif isinstance(obj, TradingPair):
+            return obj.to_dict()
+
+
+class ExchangeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pd.Timestamp):
+            return obj.strftime(DATE_TIME_FORMAT)
+
+        elif isinstance(obj, TradingPair):
+            asset = obj.to_dict()
+            asset['maker'] = round(asset['maker'], asset['decimals'])
+            asset['taker'] = round(asset['taker'], asset['decimals'])
+            asset['lot'] = round(asset['lot'], 4)
+            asset['min_trade_size'] = round(asset['min_trade_size'], 4)
+            asset['max_trade_size'] = round(asset['max_trade_size'], 4)
+            return asset
+
+        # Let the base class default method raise the TypeError
+        return JSONEncoder.default(self, obj)
+
+
+class ExchangeJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        json.JSONDecoder.__init__(
+            self, object_hook=self.object_hook, *args, **kwargs
+        )
+
+    def recursive_iter(self, obj):
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                match = isinstance(value, string_types) and re.search(
+                    r'(\d{4}-\d{2}-\d{2}).*', value
+                )
+                if match:
+                    try:
+                        obj[key] = pd.to_datetime(value, utc=True)
+                    except ValueError:
+                        pass
+
+        elif any(isinstance(obj, t) for t in (list, tuple)):
+            for item in obj:
+                self.recursive_iter(item)
+
+    def object_hook(self, obj):
+        self.recursive_iter(obj)
+        return obj
+
+
+def portfolio_to_dict(portfolio):
+    positions = []
+    for asset in portfolio.positions:
+        p = portfolio.positions[asset]  # Type: Position
+
+        position = dict(
+            symbol=asset.symbol,
+            exchange=asset.exchange,
+            amount=p.amount,
+            cost_basis=p.cost_basis,
+            last_sale_price=p.last_sale_price,
+            last_sale_date=p.last_sale_date,
+        )
+        positions.append(position)
+
+    portfolio_dict = vars(portfolio)
+    portfolio_dict['positions'] = positions
+
+    return portfolio_dict
+
+
+def portfolio_from_dict(self, portfolio_data):
+    from catalyst.protocol import Portfolio
+    return Portfolio()

catalyst/exchange/utils/stats_utils.py

@@ -1,18 +1,19 @@
-import csv
-import numbers
-
 import copy
-import numpy as np
+import csv
+import json
+import numbers
 import os
-import pandas as pd
-import boto3
 import time
 
+import numpy as np
+import pandas as pd
 from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_algo_folder
+from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
 
-from catalyst.exchange.exchange_utils import get_algo_folder
-
-s3 = boto3.resource('s3')
+s3_conn = []
+mailgun = []
 
 
 def trend_direction(series):
@@ -43,7 +44,7 @@ def crossover(source, target):
     """
     if isinstance(target, numbers.Number):
         if source[-1] is np.nan or source[-2] is np.nan \
-                or target is np.nan:
+            or target is np.nan:
             return False
 
         if source[-1] >= target > source[-2]:
@@ -53,7 +54,7 @@ def crossover(source, target):
 
     else:
         if source[-1] is np.nan or source[-2] is np.nan \
-                or target[-1] is np.nan or target[-2] is np.nan:
+            or target[-1] is np.nan or target[-2] is np.nan:
             return False
 
         if source[-1] > target[-1] and source[-2] < target[-2]:
@@ -80,7 +81,7 @@ def crossunder(source, target):
     """
     if isinstance(target, numbers.Number):
         if source[-1] is np.nan or source[-2] is np.nan \
-                or target is np.nan:
+            or target is np.nan:
             return False
 
         if source[-1] < target <= source[-2]:
@@ -89,7 +90,7 @@ def crossunder(source, target):
             return False
     else:
         if source[-1] is np.nan or source[-2] is np.nan \
-                or target[-1] is np.nan or target[-2] is np.nan:
+            or target[-1] is np.nan or target[-2] is np.nan:
             return False
 
         if source[-1] < target[-1] and source[-2] >= target[-2]:
@@ -195,6 +196,9 @@ def prepare_stats(stats, recorded_cols=list()):
         if recorded_cols is not None:
             for column in recorded_cols[:]:
                 value = row_data[column]
+                if isinstance(value, pd.Series):
+                    value = value.to_dict()
+
                 if type(value) is dict:
                     for asset in value:
                         if not isinstance(asset, TradingPair):
@@ -225,7 +229,10 @@ def prepare_stats(stats, recorded_cols=list()):
                                               asset_values)
 
     df = pd.DataFrame(stats)
-
+    df['orders'] = df['orders'].apply(lambda orders: len(orders))
+    df['transactions'] = df['transactions'].apply(
+        lambda transactions: len(transactions)
+    )
     index_cols = [
         'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
         'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
@@ -237,11 +244,6 @@ def prepare_stats(stats, recorded_cols=list()):
         for column in recorded_cols:
             index_cols.append(column)
 
-    df['orders'] = df['orders'].apply(lambda orders: len(orders))
-    df['transactions'] = df['transactions'].apply(
-        lambda transactions: len(transactions)
-    )
-
     if asset_cols:
         columns = asset_cols
         df.set_index(index_cols, drop=True, inplace=True)
@@ -257,7 +259,14 @@ def prepare_stats(stats, recorded_cols=list()):
     return df, columns
 
 
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
@@ -276,23 +285,19 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
 
     """
     if isinstance(stats, pd.DataFrame):
-        stats = stats.T.to_dict().values()
+        stats = list(stats.T.to_dict().values())
+        stats.sort(key=itemgetter('period_close'))
 
-    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
 
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-    formatters = {
-        'returns': lambda returns: "{0:.4f}".format(returns),
-    }
-
-    return df.tail(num_rows).to_string(
-        columns=columns,
-        formatters=formatters
+    df, columns = prepare_stats(
+        display_stats, recorded_cols=recorded_cols
     )
+    set_print_settings()
+    return df.to_string(columns=columns)
 
 
 def get_csv_stats(stats, recorded_cols=None):
@@ -338,6 +343,12 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     -------
 
     """
+    if not s3_conn:
+        import boto3
+        s3_conn.append(boto3.resource('s3'))
+
+    s3 = s3_conn[0]
+
     if bytes_to_write is None:
         bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
 
@@ -346,13 +357,47 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
     pid = os.getpid()
 
     parts = uri.split('//')
-    obj = s3.Object(parts[1], '{}/{}-{}-{}.csv'.format(
-        folder, timestr, algo_namespace, pid
-    ))
+    path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
+        folder=folder,
+        algo=algo_namespace,
+        time=timestr,
+        pid=pid,
+    )
+    obj = s3.Object(parts[1], path)
     obj.put(Body=bytes_to_write)
 
 
-def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
+def email_error(algo_name, dt, e, environ=None):
+    import requests
+    import traceback
+
+    if not mailgun:
+        root = data_root(environ)
+        filename = os.path.join(root, 'mailgun.json')
+        if not os.path.exists(filename):
+            raise ValueError(
+                'mailgun.json not found in the catalyst data folder'
+            )
+
+        with open(filename) as data_file:
+            mailgun.append(json.load(data_file))
+
+    mg = mailgun[0]
+
+    return requests.post(
+        mg['url'],
+        auth=("api", mg['api']),
+        data={
+            "from": mg['from'],
+            "to": mg['to'],
+            "subject": 'Error: {}'.format(algo_name),
+            "text": '{}\n\n{}\n{}'.format(
+                dt, e, traceback.format_exc()
+            )})
+
+
+def stats_to_algo_folder(stats, algo_namespace,
+                         folder_name, recorded_cols=None):
     """
     Saves the performance stats to the algo local folder.
 
@@ -360,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     ----------
     stats: list[Object]
     algo_namespace: str
+    folder_name: str
     recorded_cols: list[str]
 
     Returns
@@ -372,7 +418,10 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
     timestr = time.strftime('%Y%m%d')
     folder = get_algo_folder(algo_namespace)
 
-    filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
+    stats_folder = os.path.join(folder, folder_name)
+    ensure_directory(stats_folder)
+
+    filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
 
     with open(filename, 'wb') as handle:
         handle.write(bytes_to_write)
@@ -401,6 +450,17 @@ def df_to_string(df):
     return df.to_string()
 
 
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
 def extract_transactions(perf):
     """
     Compute indexes for buy and sell transactions

catalyst/exchange/utils/test_utils.py

@@ -0,0 +1,82 @@
+import os
+import random
+import tempfile
+
+from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import find_exchanges
+from catalyst.utils.paths import ensure_directory
+
+
+def handle_exchange_error(exchange, e):
+    try:
+        message = '{}: {}'.format(
+            e.__class__, e.message.decode('ascii', 'ignore')
+        )
+    except Exception:
+        message = 'unexpected error'
+
+    folder = get_exchange_folder(exchange.name)
+    filename = os.path.join(folder, 'blacklist.txt')
+    with open(filename, 'wt') as handle:
+        handle.write(message)
+
+
+def select_random_exchanges(population=3, features=None,
+                            is_authenticated=False, base_currency=None):
+    all_exchanges = find_exchanges(
+        features=features,
+        is_authenticated=is_authenticated,
+        base_currency=base_currency,
+    )
+
+    if population is not None:
+        if len(all_exchanges) < population:
+            population = len(all_exchanges)
+
+        exchanges = random.sample(all_exchanges, population)
+
+    else:
+        exchanges = all_exchanges
+
+    return exchanges
+
+
+def select_random_assets(all_assets, population=3):
+    assets = random.sample(all_assets, population)
+    return assets
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
+    else:
+        asset_folder = ','.join(
+            ['{}_{}'.format(a.exchange, a.symbol) for a in assets]
+        )
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path, folder

catalyst/exchange/validator.py

@@ -1,142 +0,0 @@
-import os
-import tempfile
-
-import pandas as pd
-import six
-from catalyst.assets._assets import TradingPair, get_calendar
-from logbook import Logger
-from pandas.util.testing import assert_frame_equal
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
-from catalyst.exchange.factory import get_exchanges
-from catalyst.utils.paths import ensure_directory
-
-log = Logger('Validator', level=LOG_LEVEL)
-
-
-def output_df(df, assets, name=None):
-    """
-    Outputs a price DataFrame to a temp folder.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-    assets
-    name
-
-    Returns
-    -------
-
-    """
-    if isinstance(assets, TradingPair):
-        exchange_folder = assets.exchange
-        asset_folder = assets.symbol
-    else:
-        exchange_folder = ','.join([asset.exchange for asset in assets])
-        asset_folder = ','.join([asset.symbol for asset in assets])
-
-    folder = os.path.join(
-        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
-    )
-    ensure_directory(folder)
-
-    if name is None:
-        name = 'output'
-
-    path = os.path.join(folder, '{}.csv'.format(name))
-    df.to_csv(path)
-
-    return path
-
-
-class Validator(object):
-    def __init__(self, data_portal):
-        self.data_portal = data_portal
-
-    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     sample_minutes):
-        """
-        Creates DataFrames from the bundle and exchange for the specified
-        data set.
-
-        Parameters
-        ----------
-        exchange: Exchange
-        assets
-        end_dt
-        bar_count
-        sample_minutes
-
-        Returns
-        -------
-
-        """
-        freq = '{}T'.format(sample_minutes)
-
-        log.info('creating data sample from bundle')
-        df1 = self.data_portal.get_history_window(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=bar_count,
-            frequency=freq,
-            field='close',
-            data_frequency='minute'
-        )
-        path = output_df(df1, assets, '{}_resampled'.format(freq))
-        log.info('saved resampled bundle candles: {}\n{}'.format(
-            path, df1.tail(10))
-        )
-
-        log.info('creating data sample from exchange api')
-        candles = exchange.get_candles(
-            end_dt=end_dt,
-            freq='{}T'.format(sample_minutes),
-            assets=assets,
-            bar_count=bar_count
-        )
-
-        series = dict()
-        for asset in assets:
-            series[asset] = pd.Series(
-                data=[candle['close'] for candle in candles[asset]],
-                index=[candle['last_traded'] for candle in candles[asset]]
-            )
-
-        df2 = pd.DataFrame(series)
-        path = output_df(df2, assets, '{}_api'.format(freq))
-        log.info('saved exchange api candles: {}\n{}'.format(
-            path, df2.tail(10))
-        )
-
-        try:
-            assert_frame_equal(df1, df2)
-            return True
-        except:
-            log.warn('differences found in dataframes')
-            return False
-
-
-if __name__ == '__main__':
-    exchanges = get_exchanges(['poloniex'])
-    exchange = six.next(six.itervalues(exchanges))
-    assets = exchange.get_assets(symbols=['eth_btc'])
-
-    open_calendar = get_calendar('OPEN')
-    asset_finder = AssetFinderExchange()
-    data_portal = DataPortalExchangeBacktest(
-        exchanges=exchanges,
-        asset_finder=asset_finder,
-        trading_calendar=open_calendar,
-        first_trading_day=None  # will set dynamically based on assets
-    )
-    validator = Validator(data_portal=data_portal)
-
-    validator.compare_bundle_with_exchange(
-        exchange=exchange,
-        assets=assets,
-        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
-        bar_count=200,
-        sample_minutes=30
-    )

catalyst/finance/risk/cumulative.py

@@ -27,15 +27,15 @@ from .risk import (
     choose_treasury
 )
 
-from empyrical import (
+from catalyst.patches.stats import (
     alpha_beta_aligned,
     annual_volatility,
-    cum_returns,
     downside_risk,
     information_ratio,
     max_drawdown,
     sharpe_ratio,
     sortino_ratio,
+    cum_returns,
 )
 import warnings
 from catalyst.constants import LOG_LEVEL
@@ -161,9 +161,13 @@ class RiskMetricsCumulative(object):
             if len(self.algorithm_returns) == 1:
                 self.algorithm_returns = np.append(0.0, self.algorithm_returns)
 
-        self.algorithm_cumulative_returns[dt_loc] = cum_returns(
-            self.algorithm_returns
-        )[-1]
+        try:
+            self.algorithm_cumulative_returns[dt_loc] = cum_returns(
+                self.algorithm_returns
+            )[-1]
+        except Exception as e:
+            log.debug('unable to calculate cum returns: {}'.format(e))
+            self.algorithm_cumulative_returns[dt_loc] = np.nan
 
         algo_cumulative_returns_to_date = \
             self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -196,8 +200,11 @@ class RiskMetricsCumulative(object):
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
-        except Exception:
-            self.benchmark_cumulative_returns[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.benchmark_cumulative_returns[dt_loc] = np.nan
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -269,9 +276,16 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.sharpe[dt_loc] = sharpe_ratio(
             self.algorithm_returns,
         )
-        self.downside_risk[dt_loc] = downside_risk(
-            self.algorithm_returns
-        )
+
+        try:
+            self.downside_risk[dt_loc] = downside_risk(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate downside risk returns: {}'.format(e)
+            )
+            self.downside_risk[dt_loc] = np.nan
 
         try:
             risk = self.downside_risk[dt_loc]
@@ -279,17 +293,26 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
                 self.algorithm_returns,
                 _downside_risk=risk
             )
-        except Exception:
-            # TODO: what causes it to error out?
-            self.sortino[dt_loc] = 0
+        except Exception as e:
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.sortino[dt_loc] = np.nan
 
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
         )
-        self.max_drawdown = max_drawdown(
-            self.algorithm_returns
-        )
+        try:
+            self.max_drawdown = max_drawdown(
+                self.algorithm_returns
+            )
+        except Exception as e:
+            log.debug(
+                'unable to calculate max drawdown: {}'.format(e)
+            )
+            self.max_drawdown = np.nan
+
         self.max_drawdowns[dt_loc] = self.max_drawdown
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage

catalyst/finance/risk/period.py

@@ -29,13 +29,15 @@ from .risk import check_entry
 from empyrical import (
     alpha_beta_aligned,
     annual_volatility,
-    cum_returns,
     downside_risk,
     information_ratio,
-    max_drawdown,
     sharpe_ratio,
     sortino_ratio
 )
+from catalyst.patches.stats import (
+    max_drawdown,
+    cum_returns,
+)
 
 from catalyst.constants import LOG_LEVEL
 

catalyst/marketplace/contract_enigma_abi.json

@@ -0,0 +1,302 @@
+[
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "name",
+      "outputs": [
+        {
+          "name": "",
+          "type": "string"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "approve",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "totalSupply",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_from",
+          "type": "address"
+        },
+        {
+          "name": "_to",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "transferFrom",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "INITIAL_SUPPLY",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "decimals",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint8"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_subtractedValue",
+          "type": "uint256"
+        }
+      ],
+      "name": "decreaseApproval",
+      "outputs": [
+        {
+          "name": "success",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [],
+      "name": "getAfterApproveTest",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [
+        {
+          "name": "_owner",
+          "type": "address"
+        }
+      ],
+      "name": "balanceOf",
+      "outputs": [
+        {
+          "name": "balance",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [],
+      "name": "symbol",
+      "outputs": [
+        {
+          "name": "",
+          "type": "string"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_to",
+          "type": "address"
+        },
+        {
+          "name": "_value",
+          "type": "uint256"
+        }
+      ],
+      "name": "transfer",
+      "outputs": [
+        {
+          "name": "",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": false,
+      "inputs": [
+        {
+          "name": "_spender",
+          "type": "address"
+        },
+        {
+          "name": "_addedValue",
+          "type": "uint256"
+        }
+      ],
+      "name": "increaseApproval",
+      "outputs": [
+        {
+          "name": "success",
+          "type": "bool"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "constant": true,
+      "inputs": [
+        {
+          "name": "_owner",
+          "type": "address"
+        },
+        {
+          "name": "_spender",
+          "type": "address"
+        }
+      ],
+      "name": "allowance",
+      "outputs": [
+        {
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "view",
+      "type": "function"
+    },
+    {
+      "inputs": [
+        {
+          "name": "testValue",
+          "type": "address"
+        }
+      ],
+      "payable": false,
+      "stateMutability": "nonpayable",
+      "type": "constructor"
+    },
+    {
+      "anonymous": false,
+      "inputs": [
+        {
+          "indexed": true,
+          "name": "owner",
+          "type": "address"
+        },
+        {
+          "indexed": true,
+          "name": "spender",
+          "type": "address"
+        },
+        {
+          "indexed": false,
+          "name": "value",
+          "type": "uint256"
+        }
+      ],
+      "name": "Approval",
+      "type": "event"
+    },
+    {
+      "anonymous": false,
+      "inputs": [
+        {
+          "indexed": true,
+          "name": "from",
+          "type": "address"
+        },
+        {
+          "indexed": true,
+          "name": "to",
+          "type": "address"
+        },
+        {
+          "indexed": false,
+          "name": "value",
+          "type": "uint256"
+        }
+      ],
+      "name": "Transfer",
+      "type": "event"
+    }
+  ]

catalyst/marketplace/contract_enigma_address.txt

@@ -0,0 +1 @@
+0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10

catalyst/marketplace/contract_marketplace_address.txt

@@ -0,0 +1 @@
+0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4

catalyst/marketplace/marketplace.py

@@ -0,0 +1,799 @@
+from __future__ import print_function
+
+import glob
+import json
+import os
+import re
+import shutil
+import sys
+import time
+import webbrowser
+
+import bcolz
+import logbook
+import pandas as pd
+import requests
+from requests_toolbelt import MultipartDecoder
+from requests_toolbelt.multipart.decoder import \
+    NonMultipartContentTypeException
+
+from catalyst.constants import (
+    LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
+    MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
+from catalyst.exchange.utils.stats_utils import set_print_settings
+from catalyst.marketplace.marketplace_errors import (
+    MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
+    MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
+    MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
+from catalyst.marketplace.utils.auth_utils import get_key_secret, \
+    get_signed_headers
+from catalyst.marketplace.utils.bundle_utils import merge_bundles
+from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
+    to_grains
+from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
+    get_data_source_folder, get_marketplace_folder, \
+    get_user_pubaddr, get_temp_bundles_folder, extract_bundle
+from catalyst.utils.paths import ensure_directory
+
+if sys.version_info.major < 3:
+    import urllib
+else:
+    import urllib.request as urllib
+
+log = logbook.Logger('Marketplace', level=LOG_LEVEL)
+
+
+class Marketplace:
+    def __init__(self):
+        global Web3
+        try:
+            from web3 import Web3, HTTPProvider
+        except ImportError:
+            raise MarketplaceRequiresPython3()
+
+        self.addresses = get_user_pubaddr()
+
+        if self.addresses[0]['pubAddr'] == '':
+            raise MarketplacePubAddressEmpty(
+                filename=os.path.join(
+                    get_marketplace_folder(), 'addresses.json')
+            )
+        self.default_account = self.addresses[0]['pubAddr']
+
+        self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
+
+        contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
+
+        self.mkt_contract_address = Web3.toChecksumAddress(
+            contract_url.readline().decode(
+                contract_url.info().get_content_charset()).strip())
+
+        abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
+        abi = json.load(abi_url)
+
+        self.mkt_contract = self.web3.eth.contract(
+            self.mkt_contract_address,
+            abi=abi,
+        )
+
+        contract_url = urllib.urlopen(ENIGMA_CONTRACT)
+
+        self.eng_contract_address = Web3.toChecksumAddress(
+            contract_url.readline().decode(
+                contract_url.info().get_content_charset()).strip())
+
+        abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
+        abi = json.load(abi_url)
+
+        self.eng_contract = self.web3.eth.contract(
+            self.eng_contract_address,
+            abi=abi,
+        )
+
+    # def get_data_sources_map(self):
+    #     return [
+    #         dict(
+    #             name='Marketcap',
+    #             desc='The marketcap value in USD.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily'],
+    #         ),
+    #         dict(
+    #             name='GitHub',
+    #             desc='The rate of development activity on GitHub.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily', 'hour'],
+    #         ),
+    #         dict(
+    #             name='Influencers',
+    #             desc='Tweets & related sentiments by selected influencers.',
+    #             start_date=pd.to_datetime('2017-01-01'),
+    #             end_date=pd.to_datetime('2018-01-15'),
+    #             data_frequencies=['daily', 'hour', 'minute'],
+    #         ),
+    #     ]
+
+    def to_text(self, hex):
+        return Web3.toText(hex).rstrip('\0')
+
+    def choose_pubaddr(self):
+        if len(self.addresses) == 1:
+            address = self.addresses[0]['pubAddr']
+            address_i = 0
+            print('Using {} for this transaction.'.format(address))
+        else:
+            while True:
+                for i in range(0, len(self.addresses)):
+                    print('{}\t{}\t{}'.format(
+                        i,
+                        self.addresses[i]['pubAddr'],
+                        self.addresses[i]['desc'])
+                    )
+                address_i = int(input('Choose your address associated with '
+                                      'this transaction: [default: 0] ') or 0)
+                if not (0 <= address_i < len(self.addresses)):
+                    print('Please choose a number between 0 and {}\n'.format(
+                        len(self.addresses) - 1))
+                else:
+                    address = Web3.toChecksumAddress(
+                        self.addresses[address_i]['pubAddr'])
+                    break
+
+        return address, address_i
+
+    def sign_transaction(self, tx):
+
+        url = 'https://www.myetherwallet.com/#offline-transaction'
+        print('\nVisit {url} and enter the following parameters:\n\n'
+              'From Address:\t\t{_from}\n'
+              '\n\tClick the "Generate Information" button\n\n'
+              'To Address:\t\t{to}\n'
+              'Value / Amount to Send:\t{value}\n'
+              'Gas Limit:\t\t{gas}\n'
+              'Gas Price:\t\t[Accept the default value]\n'
+              'Nonce:\t\t\t{nonce}\n'
+              'Data:\t\t\t{data}\n'.format(
+                url=url,
+                _from=tx['from'],
+                to=tx['to'],
+                value=tx['value'],
+                gas=tx['gas'],
+                nonce=tx['nonce'],
+                data=tx['data'], )
+              )
+
+        webbrowser.open_new(url)
+
+        signed_tx = input('Copy and Paste the "Signed Transaction" '
+                          'field here:\n')
+
+        if signed_tx.startswith('0x'):
+            signed_tx = signed_tx[2:]
+
+        return signed_tx
+
+    def check_transaction(self, tx_hash):
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
+                tx_hash)
+        else:
+            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
+
+        print('\nYou can check the outcome of your transaction here:\n'
+              '{}\n\n'.format(etherscan))
+
+    def _list(self):
+        data_sources = self.mkt_contract.functions.getAllProviders().call()
+
+        data = []
+        for index, data_source in enumerate(data_sources):
+            if index > 0:
+                if 'test' not in Web3.toText(data_source).lower():
+                    data.append(
+                        dict(
+                            dataset=self.to_text(data_source)
+                        )
+                    )
+        return pd.DataFrame(data)
+
+    def list(self):
+        df = self._list()
+
+        set_print_settings()
+        if df.empty:
+            print('There are no datasets available yet.')
+        else:
+            print(df)
+
+    def subscribe(self, dataset=None):
+
+        if dataset is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'subscribe to [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
+
+        dataset = dataset.lower()
+
+        address = self.choose_pubaddr()[0]
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset)
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(dataset))
+            return
+
+        grains = provider_info[1]
+        price = from_grains(grains)
+
+        subscribed = self.mkt_contract.functions.checkAddressSubscription(
+            address, Web3.toHex(dataset)
+        ).call()
+
+        if subscribed[5]:
+            print(
+                '\nYou are already subscribed to the "{}" dataset.\n'
+                'Your subscription started on {} UTC, and is valid until '
+                '{} UTC.'.format(
+                    dataset,
+                    pd.to_datetime(subscribed[3], unit='s', utc=True),
+                    pd.to_datetime(subscribed[4], unit='s', utc=True)
+                )
+            )
+            return
+
+        print('\nThe price for a monthly subscription to this dataset is'
+              ' {} ENG'.format(price))
+
+        print(
+            'Checking that the ENG balance in {} is greater than {} '
+            'ENG... '.format(address, price), end=''
+        )
+
+        wallet_address = address[2:]
+        balance = self.web3.eth.call({
+            'from': address,
+            'to': self.eng_contract_address,
+            'data': '0x70a08231000000000000000000000000{}'.format(
+                wallet_address
+            )
+        })
+
+        try:
+            balance = Web3.toInt(balance)  # web3 >= 4.0.0b7
+        except TypeError:
+            balance = Web3.toInt(hexstr=balance)  # web3 <= 4.0.0b6
+
+        if balance > grains:
+            print('OK.')
+        else:
+            print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
+                  'than the price of the dataset that you are trying to\n'
+                  'buy: {} ENG. Get enough ENG to cover the costs of the '
+                  'monthly\nsubscription for what you are trying to buy, '
+                  'and try again.'.format(
+                    address, from_grains(balance), price))
+            return
+
+        while True:
+            agree_pay = input('Please confirm that you agree to pay {} ENG '
+                              'for a monthly subscription to the dataset "{}" '
+                              'starting today. [default: Y] '.format(
+                                price, dataset)) or 'y'
+            if agree_pay.lower() not in ('y', 'n'):
+                print("Please answer Y or N.")
+            else:
+                if agree_pay.lower() == 'y':
+                    break
+                else:
+                    return
+
+        print('Ready to subscribe to dataset {}.\n'.format(dataset))
+        print('In order to execute the subscription, you will need to sign '
+              'two different transactions:\n'
+              '1. First transaction is to authorize the Marketplace contract '
+              'to spend {} ENG on your behalf.\n'
+              '2. Second transaction is the actual subscription for the '
+              'desired dataset'.format(price))
+
+        tx = self.eng_contract.functions.approve(
+            self.mkt_contract_address,
+            grains,
+        ).buildTransaction(
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
+        )
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(tx)
+        try:
+            tx_hash = '0x{}'.format(
+                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
+            )
+            print(
+                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
+            )
+
+        except Exception as e:
+            print('Unable to subscribe to data source: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the first transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nFirst transaction successful!\n'
+              'Now processing second transaction.')
+
+        tx = self.mkt_contract.functions.subscribe(
+            Web3.toHex(dataset),
+        ).buildTransaction({
+            'from': address,
+            'nonce': self.web3.eth.getTransactionCount(address)})
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(tx)
+
+        try:
+            tx_hash = '0x{}'.format(bin_hex(
+                self.web3.eth.sendRawTransaction(signed_tx)))
+            print('\nThis is the TxHash for this transaction: '
+                  '{}'.format(tx_hash))
+
+        except Exception as e:
+            print('Unable to subscribe to data source: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the second transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nSecond transaction successful!\n'
+              'You have successfully subscribed to dataset {} with'
+              'address {}.\n'
+              'You can now ingest this dataset anytime during the '
+              'next month by running the following command:\n'
+              'catalyst marketplace ingest --dataset={}'.format(
+                dataset, address, dataset))
+
+    def process_temp_bundle(self, ds_name, path):
+        """
+        Merge the temp bundle into the main bundle for the specified
+        data source.
+
+        Parameters
+        ----------
+        ds_name
+        path
+
+        Returns
+        -------
+
+        """
+        tmp_bundle = extract_bundle(path)
+        bundle_folder = get_data_source_folder(ds_name)
+        ensure_directory(bundle_folder)
+        if os.listdir(bundle_folder):
+            zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
+            ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
+            merge_bundles(zsource, ztarget)
+
+        else:
+            os.rename(tmp_bundle, bundle_folder)
+
+        pass
+
+    def ingest(self, ds_name=None, start=None, end=None, force_download=False):
+
+        if ds_name is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'ingest [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        ds_name = df_sets.iloc[dataset_num]['dataset']
+                        break
+
+        # ds_name = ds_name.lower()
+
+        # TODO: catch error conditions
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(ds_name)
+        ).call()
+
+        if not provider_info[4]:
+            print('The requested "{}" dataset is not registered in '
+                  'the Data Marketplace.'.format(ds_name))
+            return
+
+        address, address_i = self.choose_pubaddr()
+        fns = self.mkt_contract.functions
+        check_sub = fns.checkAddressSubscription(
+            address, Web3.toHex(ds_name)
+        ).call()
+
+        if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
+            print('You are not subscribed to dataset "{}" with address {}. '
+                  'Plese subscribe first.'.format(ds_name, address))
+            return
+
+        if not check_sub[5]:
+            print('Your subscription to dataset "{}" expired on {} UTC.'
+                  'Please renew your subscription by running:\n'
+                  'catalyst marketplace subscribe --dataset={}'.format(
+                    ds_name,
+                    pd.to_datetime(check_sub[4], unit='s', utc=True),
+                    ds_name)
+                  )
+
+        if 'key' in self.addresses[address_i]:
+            key = self.addresses[address_i]['key']
+            secret = self.addresses[address_i]['secret']
+        else:
+            key, secret = get_key_secret(address)
+
+        headers = get_signed_headers(ds_name, key, secret)
+        log.debug('Starting download of dataset for ingestion...')
+        r = requests.post(
+            '{}/marketplace/ingest'.format(AUTH_SERVER),
+            headers=headers,
+            stream=True,
+        )
+        if r.status_code == 200:
+            target_path = get_temp_bundles_folder()
+            try:
+                decoder = MultipartDecoder.from_response(r)
+                for part in decoder.parts:
+                    h = part.headers[b'Content-Disposition'].decode('utf-8')
+                    # Extracting the filename from the header
+                    name = re.search(r'filename="(.*)"', h).group(1)
+
+                    filename = os.path.join(target_path, name)
+                    with open(filename, 'wb') as f:
+                        # for chunk in part.content.iter_content(
+                        #         chunk_size=1024):
+                        #     if chunk: # filter out keep-alive new chunks
+                        #         f.write(chunk)
+                        f.write(part.content)
+
+                    self.process_temp_bundle(ds_name, filename)
+
+            except NonMultipartContentTypeException:
+                response = r.json()
+                raise MarketplaceHTTPRequest(
+                    request='ingest dataset',
+                    error=response,
+                )
+        else:
+            raise MarketplaceHTTPRequest(
+                request='ingest dataset',
+                error=r.status_code,
+            )
+
+        log.info('{} ingested successfully'.format(ds_name))
+
+    def get_dataset(self, ds_name, start=None, end=None):
+        ds_name = ds_name.lower()
+
+        # TODO: filter ctable by start and end date
+        bundle_folder = get_data_source_folder(ds_name)
+        z = bcolz.ctable(rootdir=bundle_folder, mode='r')
+
+        df = z.todataframe()  # type: pd.DataFrame
+        df.set_index(['date', 'symbol'], drop=True, inplace=True)
+
+        # TODO: implement the filter more carefully
+        # if start and end is None:
+        #     df = df.xs(start, level=0)
+
+        return df
+
+    def clean(self, ds_name=None, data_frequency=None):
+
+        if ds_name is None:
+            mktplace_root = get_marketplace_folder()
+            folders = [os.path.basename(f.rstrip('/'))
+                       for f in glob.glob('{}/*/'.format(mktplace_root))
+                       if 'temp_bundles' not in f]
+
+            while True:
+                for idx, f in enumerate(folders):
+                    print('{}\t{}'.format(idx, f))
+                dataset_num = input('Choose the dataset you want to '
+                                    'clean [0..{}]: '.format(
+                                        len(folders) - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        len(folders) - 1))
+                else:
+                    if dataset_num not in range(0, len(folders)):
+                        print('Enter a number between 0 and {}'.format(
+                            len(folders) - 1))
+                    else:
+                        ds_name = folders[dataset_num]
+                        break
+
+        ds_name = ds_name.lower()
+
+        if data_frequency is None:
+            folder = get_data_source_folder(ds_name)
+
+        else:
+            folder = get_bundle_folder(ds_name, data_frequency)
+
+        shutil.rmtree(folder)
+        pass
+
+    def create_metadata(self, key, secret, ds_name, data_frequency, desc,
+                        has_history=True, has_live=True):
+        """
+
+        Returns
+        -------
+
+        """
+        headers = get_signed_headers(ds_name, key, secret)
+        r = requests.post(
+            '{}/marketplace/register'.format(AUTH_SERVER),
+            json=dict(
+                ds_name=ds_name,
+                desc=desc,
+                data_frequency=data_frequency,
+                has_history=has_history,
+                has_live=has_live,
+            ),
+            headers=headers,
+        )
+
+        if r.status_code != 200:
+            raise MarketplaceHTTPRequest(
+                request='register', error=r.status_code
+            )
+
+        if 'error' in r.json():
+            raise MarketplaceHTTPRequest(
+                request='upload file', error=r.json()['error']
+            )
+
+    def register(self):
+        while True:
+            desc = input('Enter the name of the dataset to register: ')
+            dataset = desc.lower()
+            provider_info = self.mkt_contract.functions.getDataProviderInfo(
+                Web3.toHex(dataset)
+            ).call()
+
+            if provider_info[4]:
+                print('There is already a dataset registered under '
+                      'the name "{}". Please choose a different '
+                      'name.'.format(dataset))
+            else:
+                break
+
+        price = int(
+            input(
+                'Enter the price for a monthly subscription to '
+                'this dataset in ENG: '
+            )
+        )
+        while True:
+            freq = input('Enter the data frequency [daily, hourly, minute]: ')
+            if freq.lower() not in ('daily', 'hourly', 'minute'):
+                print('Not a valid frequency.')
+            else:
+                break
+
+        while True:
+            reg_pub = input(
+                'Does it include historical data? [default: Y]: '
+            ) or 'y'
+            if reg_pub.lower() not in ('y', 'n'):
+                print('Please answer Y or N.')
+            else:
+                if reg_pub.lower() == 'y':
+                    has_history = True
+                else:
+                    has_history = False
+                break
+
+        while True:
+            reg_pub = input(
+                'Doest it include live data? [default: Y]: '
+            ) or 'y'
+            if reg_pub.lower() not in ('y', 'n'):
+                print('Please answer Y or N.')
+            else:
+                if reg_pub.lower() == 'y':
+                    has_live = True
+                else:
+                    has_live = False
+                break
+
+        address, address_i = self.choose_pubaddr()
+        if 'key' in self.addresses[address_i]:
+            key = self.addresses[address_i]['key']
+            secret = self.addresses[address_i]['secret']
+        else:
+            key, secret = get_key_secret(address)
+
+        grains = to_grains(price)
+
+        tx = self.mkt_contract.functions.register(
+            Web3.toHex(dataset),
+            grains,
+            address,
+        ).buildTransaction(
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
+        )
+
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(tx)
+
+        try:
+            tx_hash = '0x{}'.format(
+                bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
+            )
+            print(
+                '\nThis is the TxHash for this transaction: {}'.format(tx_hash)
+            )
+
+        except Exception as e:
+            print('Unable to register the requested dataset: {}'.format(e))
+            return
+
+        self.check_transaction(tx_hash)
+
+        print('Waiting for the transaction to succeed...')
+
+        while True:
+            try:
+                if self.web3.eth.getTransactionReceipt(tx_hash).status:
+                    break
+                else:
+                    print('\nTransaction failed. Aborting...')
+                    return
+            except AttributeError:
+                pass
+            for i in range(0, 10):
+                print('.', end='', flush=True)
+                time.sleep(1)
+
+        print('\nWarming up the {} dataset'.format(dataset))
+        self.create_metadata(
+            key=key,
+            secret=secret,
+            ds_name=dataset,
+            data_frequency=freq,
+            desc=desc,
+            has_history=has_history,
+            has_live=has_live,
+        )
+        print('\n{} registered successfully'.format(dataset))
+
+    def publish(self, dataset, datadir, watch):
+        dataset = dataset.lower()
+        provider_info = self.mkt_contract.functions.getDataProviderInfo(
+            Web3.toHex(dataset)
+        ).call()
+
+        if not provider_info[4]:
+            raise MarketplaceDatasetNotFound(dataset=dataset)
+
+        match = next(
+            (l for l in self.addresses if l['pubAddr'] == provider_info[0]),
+            None
+        )
+        if not match:
+            raise MarketplaceNoAddressMatch(
+                dataset=dataset,
+                address=provider_info[0])
+
+        print('Using address: {} to publish this dataset.'.format(
+            provider_info[0]))
+
+        if 'key' in match:
+            key = match['key']
+            secret = match['secret']
+        else:
+            key, secret = get_key_secret(provider_info[0])
+
+        headers = get_signed_headers(dataset, key, secret)
+        filenames = glob.glob(os.path.join(datadir, '*.csv'))
+
+        if not filenames:
+            raise MarketplaceNoCSVFiles(datadir=datadir)
+
+        files = []
+        for file in filenames:
+            files.append(('file', open(file, 'rb')))
+
+        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                          files=files,
+                          headers=headers)
+
+        if r.status_code != 200:
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.status_code)
+
+        if 'error' in r.json():
+            raise MarketplaceHTTPRequest(request='upload file',
+                                         error=r.json()['error'])
+
+        print('Dataset {} uploaded successfully.'.format(dataset))

catalyst/marketplace/marketplace_errors.py

@@ -0,0 +1,97 @@
+import sys
+import traceback
+
+from catalyst.errors import ZiplineError
+
+
+def silent_except_hook(exctype, excvalue, exctraceback):
+    if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
+                   MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
+                   MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
+                   MarketplaceSubscriptionExpired, MarketplaceJSONError,
+                   MarketplaceWalletNotSupported, MarketplaceEmptySignature,
+                   MarketplaceRequiresPython3]:
+        fn = traceback.extract_tb(exctraceback)[-1][0]
+        ln = traceback.extract_tb(exctraceback)[-1][1]
+        print("Error traceback: {1} (line {2})\n"
+              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
+    else:
+        sys.__excepthook__(exctype, excvalue, exctraceback)
+
+
+sys.excepthook = silent_except_hook
+
+
+class MarketplacePubAddressEmpty(ZiplineError):
+    msg = (
+        'Please enter your public address to use in the Data Marketplace '
+        'in the following file: {filename}'
+    ).strip()
+
+
+class MarketplaceDatasetNotFound(ZiplineError):
+    msg = (
+        'The dataset "{dataset}" is not registered in the Data Marketplace.'
+    ).strip()
+
+
+class MarketplaceNoAddressMatch(ZiplineError):
+    msg = (
+        'The address registered with the dataset {dataset}: {address} '
+        'does not match any of your addresses.'
+    ).strip()
+
+
+class MarketplaceHTTPRequest(ZiplineError):
+    msg = (
+        'Request to remote server to {request} failed: {error}'
+    ).strip()
+
+
+class MarketplaceNoCSVFiles(ZiplineError):
+    msg = (
+        'No CSV files found on {datadir} to upload.'
+    )
+
+
+class MarketplaceContractDataNoMatch(ZiplineError):
+    msg = (
+        'The information found on the contract does not match the '
+        'requested data:\n{params}.'
+    )
+
+
+class MarketplaceSubscriptionExpired(ZiplineError):
+    msg = (
+        'Your subscription to dataset "{dataset}" expired on {date} '
+        'and is no longer active. You have to subscribe again running the '
+        'following command:\n'
+        'catalyst marketplace subscribe --dataset={dataset}'
+    )
+
+
+class MarketplaceWalletNotSupported(ZiplineError):
+    msg = (
+        'Wallet {wallet} is not supported.'
+    )
+
+
+class MarketplaceEmptySignature(ZiplineError):
+    msg = (
+        'Signature cannot be empty.'
+    )
+
+
+class MarketplaceJSONError(ZiplineError):
+    msg = (
+        'The configuration file {file} is malformed. Please correct '
+        'the following error:\n{error}'
+    )
+
+
+class MarketplaceRequiresPython3(ZiplineError):
+    msg = (
+        '\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n'
+        'If you want to use the Data Marketplace, you need to reinstall '
+        'Catalyst\nwith Python3. See the documentation website for additional '
+        'information.')

catalyst/marketplace/utils/auth_utils.py

@@ -0,0 +1,131 @@
+import hashlib
+import hmac
+
+import requests
+import time
+
+from catalyst.marketplace.marketplace_errors import (
+    MarketplaceHTTPRequest, MarketplaceWalletNotSupported,
+    MarketplaceEmptySignature)
+from catalyst.marketplace.utils.path_utils import (
+    get_user_pubaddr, save_user_pubaddr)
+from catalyst.constants import AUTH_SERVER
+
+
+def get_key_secret(pubAddr, wallet='mew'):
+    """
+    Obtain a new key/secret pair from authentication server
+
+    Parameters
+    ----------
+    pubAddr: str
+    dataset: str
+
+    Returns
+    -------
+    key: str
+    secret: str
+
+    """
+    session = requests.Session()
+    response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
+                           headers={
+                            'Authorization': 'Digest username="{0}"'.format(
+                                pubAddr)})
+
+    if response.status_code != 401:
+        raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                     error='Unexpected response code: '
+                                           '{}'.format(response.status_code))
+
+    header = response.headers.get('WWW-Authenticate')
+    auth_type, auth_info = header.split(None, 1)
+    d = requests.utils.parse_dict_header(auth_info)
+
+    nonce = '0x{}'.format(d['nonce'])
+
+    if wallet == 'mew':
+        print('\nObtaining a key/secret pair to streamline all future '
+              'requests with the authentication server.\n'
+              'Visit https://www.myetherwallet.com/signmsg.html and sign the '
+              'following message:\n{}'.format(nonce))
+        signature = input('Copy and Paste the "sig" field from '
+                          'the signature here (without the double quotes, '
+                          'only the HEX value):\n')
+    else:
+        raise MarketplaceWalletNotSupported(wallet=wallet)
+
+    if signature is None:
+        raise MarketplaceEmptySignature()
+
+    signature = signature[2:]
+    r = int(signature[0:64], base=16)
+    s = int(signature[64:128], base=16)
+    v = int(signature[128:130], base=16)
+    vrs = [v, r, s]
+
+    response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
+                           headers={
+                'Authorization': 'Digest username="{0}",realm="{1}",'
+                'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",'
+                'opaque="{4}"'.format(pubAddr,
+                                      d['realm'],
+                                      d['nonce'],
+                                      ','.join(str(e) for e in vrs+[wallet]),
+                                      d['opaque'])})
+
+    if response.status_code == 200:
+
+        if 'error' in response.json():
+            raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                         error=str(response.json()['error']))
+        else:
+            addresses = get_user_pubaddr()
+
+            match = next((l for l in addresses if
+                          l['pubAddr'] == pubAddr), None)
+            match['key'] = response.json()['key']
+            match['secret'] = response.json()['secret']
+
+            addresses[addresses.index(match)] = match
+
+            save_user_pubaddr(addresses)
+            print('Key/secret pair retrieved successfully from server.')
+
+            return match['key'], match['secret']
+
+    else:
+        raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
+                                     error=response.status_code)
+
+
+def get_signed_headers(ds_name, key, secret):
+    """
+    Return a new request header including the key / secret signature
+
+    Parameters
+    ----------
+    ds_name
+    key
+    secret
+
+    Returns
+    -------
+
+    """
+    nonce = str(int(time.time()))
+
+    signature = hmac.new(
+        secret.encode('utf-8'),
+        '{}{}'.format(ds_name, nonce).encode('utf-8'),
+        hashlib.sha512
+    ).hexdigest()
+
+    headers = {
+        'Sign': signature,
+        'Key': key,
+        'Nonce': nonce,
+        'Dataset': ds_name,
+    }
+
+    return headers

catalyst/marketplace/utils/bundle_utils.py

@@ -0,0 +1,94 @@
+import os
+import random
+import re
+import shutil
+
+import bcolz
+import numpy as np
+import pandas as pd
+from six import string_types
+
+
+def merge_bundles(zsource, ztarget):
+    """
+    Merge
+    Parameters
+    ----------
+    zsource
+    ztarget
+
+    Returns
+    -------
+
+    """
+    # TODO: find a way to do this iteratively instead of in-memory
+    df_source = zsource.todataframe()
+    df_target = ztarget.todataframe()
+
+    df = pd.concat(
+        [df_source, df_target], ignore_index=True
+    )  # type: pd.DataFrame
+    df.drop_duplicates(inplace=True)
+    df.set_index(['date', 'symbol'], drop=False, inplace=True)
+
+    sanitize_df(df)
+
+    dirname = os.path.basename(ztarget.rootdir)
+    bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
+    shutil.move(ztarget.rootdir, bak_dir)
+
+    z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
+    shutil.rmtree(bak_dir)
+    return z
+
+
+def sanitize_df(df):
+    # Using a sampling method to identify dates for efficiency with
+    # large datasets
+    if len(df) > 100:
+        indexes = random.sample(range(0, len(df) - 1), 100)
+    elif len(df) > 1:
+        indexes = range(0, len(df) - 1)
+    else:
+        indexes = [0, ]
+
+    for column in df.columns:
+        is_date = False
+        for index in indexes:
+            value = df[column].iloc[index]
+            if not isinstance(value, string_types):
+                continue
+
+            # TODO: assuming that the date is at least daily
+            exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
+            matches = exp.findall(value)
+
+            if matches:
+                is_date = True
+                break
+
+        if is_date:
+            df[column] = pd.to_datetime(df[column])
+
+        else:
+            try:
+                ser = safely_reduce_dtype(df[column])
+                df[column] = ser
+            except Exception:
+                pass
+
+    return df
+
+
+def safely_reduce_dtype(ser):  # pandas.Series or numpy.array
+    orig_dtype = "".join(
+        [x for x in ser.dtype.name if x.isalpha()])  # float/int
+    mx = 1
+    for val in ser.values:
+        new_itemsize = np.min_scalar_type(val).itemsize
+        if mx < new_itemsize:
+            mx = new_itemsize
+        if orig_dtype == 'int':
+            mx = max(mx, 4)
+    new_dtype = orig_dtype + str(mx * 8)
+    return ser.astype(new_dtype)

catalyst/marketplace/utils/eth_utils.py

@@ -0,0 +1,82 @@
+import binascii
+
+
+# def bytes32(string):
+#     """
+#     Convert string to bytes32 data type for smart contract
+
+#     Parameters
+#     ----------
+#     string: str
+
+#     Returns
+#     -------
+#     list
+
+#     """
+#     return binascii.hexlify(string.encode('utf-8'))
+
+
+# def b32_str(bytes32):
+#     """
+#     Convert bytes32 to string
+
+#     Parameters
+#     ----------
+#     input: bytes object
+
+#     Returns
+#     -------
+#     str
+
+#     """
+#     return binascii.unhexlify(
+#             bytes32.decode('utf-8').rstrip('\0')).decode('ascii')
+
+
+def bin_hex(binary):
+    """
+    Convert bytes32 to string
+
+    Parameters
+    ----------
+    input: bytes object
+
+    Returns
+    -------
+    str
+
+    """
+    return binascii.hexlify(binary).decode('utf-8')
+
+
+def from_grains(amount):
+    """
+    Convert from grains to cryptocurrency
+
+    Parameters
+    ----------
+    input: amount
+
+    Returns
+    -------
+    int
+
+    """
+    return amount // 10 ** 8
+
+
+def to_grains(amount):
+    """
+    Convert from cryptocurrency to grains
+
+    Parameters
+    ----------
+    input: amount
+
+    Returns
+    -------
+    int
+
+    """
+    return amount * 10 ** 8

catalyst/marketplace/utils/path_utils.py

@@ -0,0 +1,166 @@
+import os
+import json
+import tarfile
+
+from catalyst.utils.deprecate import deprecated
+from catalyst.utils.paths import data_root, ensure_directory
+from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
+
+
+def get_marketplace_folder(environ=None):
+    """
+    The root path of the marketplace folder.
+
+    Parameters
+    ----------
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    marketplace_folder = os.path.join(root, 'marketplace')
+    ensure_directory(marketplace_folder)
+
+    return marketplace_folder
+
+
+def get_data_source_folder(data_source_name, environ=None):
+    """
+    The root path of an data_source folder.
+
+    Parameters
+    ----------
+    data_source_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    if not environ:
+        environ = os.environ
+
+    root = data_root(environ)
+    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
+    ensure_directory(data_source_folder)
+
+    return data_source_folder
+
+
+@deprecated
+def get_bundle_folder(data_source_name, data_frequency, environ=None):
+    data_source_folder = get_data_source_folder(data_source_name, environ)
+
+    bundle_folder = os.path.join(data_source_folder, data_frequency)
+
+    ensure_directory(bundle_folder)
+
+    return bundle_folder
+
+
+def get_temp_bundles_folder(environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    ds_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    root = data_root(environ)
+    folder = os.path.join(root, 'marketplace', 'temp_bundles')
+    ensure_directory(folder)
+
+    return folder
+
+
+def extract_bundle(tar_filename):
+    """
+    Extract a bcolz bundle.
+
+    Parameters
+    ----------
+    ds_name
+
+    Returns
+    -------
+    str
+
+    """
+    target_path = tar_filename.replace('.tar.gz', '')
+    with tarfile.open(tar_filename, 'r') as tar:
+        tar.extractall(target_path)
+
+    return target_path
+
+
+def get_user_pubaddr(environ=None):
+    """
+    The de-serialized contend of the user's addresses.json file.
+
+    Parameters
+    ----------
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
+    marketplace_folder = get_marketplace_folder(environ)
+    filename = os.path.join(marketplace_folder, 'addresses.json')
+
+    if os.path.isfile(filename):
+        with open(filename) as data_file:
+            try:
+                data = json.load(data_file)
+            except json.decoder.JSONDecodeError as e:
+                raise MarketplaceJSONError(file=filename, error=e)
+            try:
+                d = data[0]['pubAddr']
+            except Exception as e:
+                return [data, ]
+            return data
+    else:
+        data = []
+        data.append(dict(pubAddr='', desc=''))
+        with open(filename, 'w') as f:
+            json.dump(data, f, sort_keys=False, indent=2,
+                      separators=(',', ':'))
+            return data
+
+
+def save_user_pubaddr(data, environ=None):
+    """
+    Saves the user's public addresses and their related metadata in
+    the corresponding addresses.json file.
+
+    Parameters
+    ----------
+    data: dict
+
+    Returns
+    -------
+    True
+
+    """
+    marketplace_folder = get_marketplace_folder(environ)
+    filename = os.path.join(marketplace_folder, 'addresses.json')
+
+    with open(filename, 'w') as f:
+        json.dump(data, f, sort_keys=False, indent=2,
+                  separators=(',', ':'))
+
+    return True

catalyst/pipeline/engine.py

@@ -7,6 +7,7 @@ from abc import (
 )
 from uuid import uuid4
 
+import six
 from six import (
     iteritems,
     with_metaclass,
@@ -33,7 +34,6 @@ from catalyst.utils.sharedoc import copydoc
 
 
 class PipelineEngine(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def run_pipeline(self, pipeline, start_date, end_date):
         """
@@ -118,6 +118,7 @@ class ExplodingPipelineEngine(PipelineEngine):
     """
     A PipelineEngine that doesn't do anything.
     """
+
     def run_pipeline(self, pipeline, start_date, end_date):
         raise NoEngineRegistered(
             "Attempted to run a pipeline but no pipeline "
@@ -484,8 +485,10 @@ class SimplePipelineEngine(PipelineEngine):
             )
 
             if isinstance(term, LoadableTerm):
+                term_key = loader_group_key(term)
+                # TODO: temp workaround
                 to_load = sorted(
-                    loader_groups[loader_group_key(term)],
+                    six.next(six.itervalues(loader_groups)),
                     key=lambda t: t.dataset
                 )
                 loader = get_loader(term)
@@ -565,9 +568,10 @@ class SimplePipelineEngine(PipelineEngine):
                 index=MultiIndex.from_arrays([empty_dates, empty_assets]),
             )
 
-        resolved_assets = array(self._finder.retrieve_all(assets))
+        # TODO: not sure what's wrong with the resolved_assets
+        # resolved_assets = array(self._finder.retrieve_all(assets))
         dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
-        assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
+        assets_kept = repeat_first_axis(assets, len(dates))[mask]
 
         final_columns = {}
         for name in data:

catalyst/pipeline/graph.py

@@ -142,7 +142,7 @@ class TermGraph(object):
         at the end of execution.
         """
         refcounts = self.graph.out_degree()
-        for t in self.outputs.values():
+        for t in list(self.outputs.values()):
             refcounts[t] += 1
 
         for t in initial_terms:
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
                  min_extra_rows=0):
         super(ExecutionPlan, self).__init__(terms)
 
-        for term in terms.values():
+        for term in list(terms.values()):
             self.set_extra_rows(
                 term,
                 all_dates,

catalyst/sources/test_source.py

@@ -144,7 +144,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
             for event in self.event_list:
                 event.sid = assets_by_identifier[event.sid].sid
 
@@ -167,7 +167,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
 
         # Hash_value for downstream sorting.
         self.arg_string = hash_args(*args, **kwargs)

catalyst/support/binance_history.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from catalyst import run_algorithm
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = 'cryptopia'
+    context.base_currency = 'btc'
+    context.coins = context.exchanges[context.exchange].assets
+    context.coins = [c for c in context.coins if
+                     c.quote_currency == context.base_currency]
+
+
+def handle_data(context, data):
+    # current date formatted into a string
+    today = data.current_dt
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.coins = context.exchanges[context.exchange].assets
+        context.coins = [c for c in context.coins if
+                         c.quote_currency == context.base_currency]
+
+    # get data every 30 minutes
+    minutes = 1
+    if not context.i % minutes:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            price = data.current(coin, 'price')
+            print(today, pair, price)
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2018-01-17', utc=True)
+    end_date = pd.to_datetime('2018-01-18', utc=True)
+
+    performance = run_algorithm(
+        capital_base=1.0,
+        # amount of base_currency, not always in dollars unless usd
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='cryptopia',
+        data_frequency='minute',
+        base_currency='btc',
+        live=True,
+        live_graph=False,
+        simulate_orders=True,
+        algo_namespace='simple_universe'
+    )

catalyst/support/ccxt_issue_1358.py

@@ -0,0 +1,8 @@
+import ccxt
+
+bitfinex = ccxt.bitfinex()
+bitfinex.verbose = True
+ohlcvs = bitfinex.fetch_ohlcv('ETH/BTC', '30m', 1504224000000)
+
+dt = bitfinex.iso8601(ohlcvs[0][0])
+print(dt)  # should print '2017-09-01T00:00:00.000Z'

catalyst/support/history_multiple_assets.py

@@ -0,0 +1,50 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset1 = symbol('fct_btc')
+    context.asset2 = symbol('btc_usdt')
+    context.coins = [context.asset1, context.asset2]
+
+
+def handle_data(context, data):
+    df = data.history(context.coins,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+    print(df)
+    print(data.current(context.asset1, 'close'))
+    print(data.current(context.asset2, 'close'))
+    exit(0)
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_111.py

@@ -0,0 +1,44 @@
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import order_target_percent
+
+NAMESPACE = 'goose7'
+log = Logger(NAMESPACE)
+
+from catalyst.api import record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('trx_btc')
+
+
+def handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    record(btc=price)
+
+    # Only ordering if it does not have any position to avoid trying some
+    # tiny orders with the leftover btc
+    pos_amount = context.portfolio.positions[context.asset].amount
+    if pos_amount > 0:
+        return
+
+    # Adding a limit price to workaround an issue with performance
+    # calculations of market orders
+    order_target_percent(
+        context.asset, 1, limit_price=price * 1.01
+    )
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.003,
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=NAMESPACE,
+        base_currency='btc',
+        live_graph=False,
+        simulate_orders=False,
+    )

catalyst/support/issue_112.py

@@ -0,0 +1,44 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    df = data.history(context.asset,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_126.py

@@ -0,0 +1,52 @@
+from catalyst import run_algorithm
+from catalyst.api import order, record, symbol
+import pandas as pd
+
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+
+
+def initialize(context):
+    context.assets = [symbol('eth_btc'), symbol('eth_usdt')]
+
+
+def handle_data(context, data):
+    order(context.assets[0], 1)
+
+    prices = data.current(context.assets, 'price')
+    record(price=prices)
+    pass
+
+
+def analyze(context, perf):
+    stats = get_pretty_stats(perf)
+    print(stats)
+    pass
+
+
+if __name__ == '__main__':
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='btc',
+            analyze=analyze,
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='usd',
+            analyze=analyze,
+            start=pd.to_datetime('2017-01-01', utc=True),
+            end=pd.to_datetime('2017-12-25', utc=True),
+        )

catalyst/support/issue_169.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from catalyst.utils.run_algo import run_algorithm
+from catalyst.api import symbol
+from exchange.utils.stats_utils import set_print_settings
+
+
+def initialize(context):
+    context.i = 0
+    context.data = []
+
+
+def handle_data(context, data):
+    prices = data.history(
+        symbol('xlm_eth'),
+        fields=['open', 'high', 'low', 'close'],
+        bar_count=50,
+        frequency='1T'
+    )
+    set_print_settings()
+    print(prices.tail(10))
+    context.data.append(prices)
+
+    context.i = context.i + 1
+    if context.i == 3:
+        context.interrupt_algorithm()
+
+
+def analyze(context, prefs):
+    for dataset in context.data:
+        print(dataset[-2:])
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.1,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='binance',
+        algo_namespace='Test candles',
+        base_currency='eth',
+        data_frequency='minute',
+        live=True,
+        simulate_orders=True)

catalyst/support/issue_216.py

@@ -0,0 +1,376 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python2
+
+import sys
+import os
+import pandas as pd
+import signal
+# import talib
+
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import (
+    symbol,
+    record,
+    order,
+    order_target,
+    order_target_percent,
+    get_open_orders
+)
+from catalyst.finance import commission
+
+
+# from base.telegrambot import TelegramBot
+
+
+class GracefulKiller:
+    # Source: https://stackoverflow.com/a/31464349
+    def __init__(self, context):
+        self.kill_now = False
+        self.signal = 0
+        self.context = context
+        signal.signal(signal.SIGINT, self.exit_gracefully)
+
+    def exit_gracefully(self, signum, frame):
+        self.kill_now = True
+        self.signal = signum
+        if hasattr(self.context,
+                   'telegram_bot') and self.context.telegram_bot is not None:
+            self.context.telegram_bot.updater.stop()
+        sys.exit(0)
+
+    def exit(self):
+        return self.kill_now
+
+
+class SimulationParameters:
+    MODE = 'paper'
+    CAPITAL_BASE = 1000
+    """
+    Capital base used on this simulation
+    """
+
+    DATA_FREQUECY = 'minute'
+
+    EXCHANGE_NAME = 'bitfinex'
+    # EXCHANGE_NAME = 'binance'
+    """
+    Exchange used on this simulation
+    """
+
+    DATA_DIR = '/home/av/Dropbox/simulations/data'
+    ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0]
+    ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images',
+                                                 ALGO_NAMESPACE)
+    ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables',
+                                                         ALGO_NAMESPACE + '_results')
+    ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR,
+                                                              'tables',
+                                                              ALGO_NAMESPACE + '_transactions')
+    BASE_CURRENCY = 'usd'
+    # BASE_CURRENCY = 'usdt'
+
+    # SHORT PERIOD
+    START_DATE = '2017-09-07'
+    """
+    Start date used on this simulation
+    """
+    END_DATE = '2017-12-12'
+    """
+    End date used on this simulation
+    """
+
+    SKIP_FIRST_CANDLES = 0
+
+    # CANDLES_SAMPLE_RATE = 60
+    # CANDLES_SAMPLE_RATE = 30
+    CANDLES_SAMPLE_RATE = 1
+    """
+    Candle interval used on this simulation (in minutes)
+    """
+
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    # 30 minute interval ohlcv data (the standard data required for candlestick or
+    # indicators/signals)
+    # 30T means 30 minutes re-sampling of one minute data.
+    # CANDLES_FREQUENCY = '60T'
+    # CANDLES_FREQUENCY = '30T'
+    CANDLES_FREQUENCY = '1T'
+    CANDLES_BUFFER_SIZE = 48
+    COIN_PAIR = 'btc_usd'
+    # COIN_PAIR = 'btc_usdt'
+    """
+    Coin pair used on this simulation
+    """
+
+    # TRANSACTIONS
+    COMMISSION_FEE = 0.0030
+    BUY_MIN_AMOUNT = 5  # i.e: USD
+    SELL_MIN_AMOUNT = 0.001  # i.e: USD
+    BUY_SELL_PERCENTAGE = 1  # 0.50
+    BUY_PERCENTAGE = BUY_SELL_PERCENTAGE
+    SELL_PERCENTAGE = BUY_SELL_PERCENTAGE
+
+    BASE_PRICE = 'close'
+    """
+    Base price used (close / Heiken Ashi)
+    """
+
+
+log = None
+parameters = None
+
+
+def print_facts(context):
+    context.log.info("""
+Index: {}
+Date: {}
+Candle:
+O: {}
+H: {}
+L: {}
+C: {}
+V: {}
+Metrics:
+...
+Portfolio:
+Base price: {}
+Base coin (coin2/usd): {}
+Amount (coin1/btc): {}
+""".format(
+        # Facts
+        context.i,
+        context.curr_minute,
+        context.candles_open[-1],
+        context.candles_high[-1],
+        context.candles_low[-1],
+        context.candles_close[-1],
+        context.candles_volume[-1],
+        # Metrics
+        # ...
+        # Portfolio
+        context.curr_base_price,
+        context.portfolio.cash,
+        context.portfolio.positions[context.coin_pair].amount,
+    ))
+
+
+def print_facts_telegram(context):
+    price = context.curr_base_price
+    amount = context.portfolio.positions[context.coin_pair].amount
+    pnl = context.portfolio.pnl
+    capital_used = context.portfolio.capital_used
+    portfolio_value = context.portfolio.portfolio_value
+    portfolio_returns = context.portfolio.returns
+    starting_cash = context.portfolio.starting_cash
+    cash = context.portfolio.cash
+
+    msg = """
+Status...
+Price: {}
+Starting cash: {}
+Cash: {}
+Capital used: {}
+Amount: {}
+Portfolio value: {}
+Returns: {}
+PnL: {}
+    """.format(
+        price,
+        starting_cash,
+        cash,
+        capital_used,
+        amount,
+        portfolio_value,
+        portfolio_returns,
+        pnl,
+    )
+    if hasattr(context, 'telegram_bot') and context.telegram_bot is not None:
+        context.telegram_bot.msg(msg)
+
+
+def default_initialize(context):
+    # FIXME: set_benchmark
+    # set_benchmark(symbol(context.parameters.COIN_PAIR))
+
+    context.coin_pair = symbol(context.parameters.COIN_PAIR)
+    context.base_price = None
+    context.current_day = None
+    context.counter = -1
+    context.i = 0
+
+    context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE
+    context.candles_frequency = context.parameters.CANDLES_FREQUENCY
+    context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE
+    context.set_commission(
+        commission.PerShare(cost=context.parameters.COMMISSION_FEE))
+
+
+def default_handle_data(context, data):
+    context.curr_minute = data.current_dt
+    context.counter += 1
+
+    if context.candles_sample_rate == 1:
+        context.i += 1
+    elif context.counter % context.candles_sample_rate != 0:
+        context.i += 1
+        return
+
+    if context.i < context.parameters.SKIP_FIRST_CANDLES:
+        return
+
+    context.candles_open = data.history(
+        context.coin_pair,
+        'open',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_high = data.history(
+        context.coin_pair,
+        'high',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_low = data.history(
+        context.coin_pair,
+        'low',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_close = data.history(
+        context.coin_pair,
+        'price',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+    context.candles_volume = data.history(
+        context.coin_pair,
+        'volume',
+        bar_count=context.candles_buffer_size,
+        frequency=context.candles_frequency)
+
+    # FIXME: Here is the error!
+    # The candles_close frame shows more or less always a value of 94, while
+    # bitcoin price is very different from that
+    print(context.candles_close)
+
+    context.base_prices = context.candles_close
+    cash = context.portfolio.cash
+    amount = context.portfolio.positions[context.coin_pair].amount
+    price = data.current(context.coin_pair, 'price')
+    order_id = None
+    context.last_base_price = context.base_prices[-2]
+    context.curr_base_price = context.base_prices[-1]
+
+    # TA calculations
+    # ...
+
+    # Sanity checks
+    # assert cash >= 0
+    if cash < 0:
+        import ipdb;
+        ipdb.set_trace()  # BREAKPOINT
+
+    print_facts(context)
+    print_facts_telegram(context)
+
+    # Order management
+    net_shares = 0
+    if context.counter == 2:
+        brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE
+        share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
+        net_shares = brute_shares - share_commission_fee
+        buy_order_id = order(context.coin_pair, net_shares)
+
+    if context.counter == 3:
+        brute_shares = amount * context.parameters.SELL_PERCENTAGE
+        share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
+        net_shares = -(brute_shares - share_commission_fee)
+        sell_order_id = order(context.coin_pair, net_shares)
+
+    # Record
+    record(
+        price=price,
+        foo='bar',
+        # volume=current['volume'],
+        # price_change=price_change,
+        # Metrics
+        cash=cash,
+        # buy=context.buy,
+        # sell=context.sell
+    )
+
+
+def default_analyze(context=None, perf=None):
+    pass
+
+
+def initialize(context):
+    global log
+    context.parameters = parameters
+    context.log = Logger(context.parameters.ALGO_NAMESPACE)
+    log = context.log
+    default_initialize(context)
+    context.killer = GracefulKiller(context)
+    context.telegram_bot = None
+
+    # TELEGRAM_TOKEN='token'
+    # context.telegram_bot = TelegramBot()
+    # context.telegram_bot.initialize(TELEGRAM_TOKEN, context)
+
+
+if __name__ == '__main__':
+    # Parameters:
+    parameters = SimulationParameters()
+    start_date = pd.to_datetime(parameters.START_DATE, utc=True)
+    end_date = pd.to_datetime(parameters.END_DATE, utc=True)
+
+    if parameters.MODE == 'backtest':
+        results = run_algorithm(
+            capital_base=parameters.CAPITAL_BASE,
+            data_frequency=parameters.DATA_FREQUECY,
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            start=start_date,
+            end=end_date,
+            live=False,
+            live_graph=False
+        )
+
+        returns_daily = results
+        results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE))
+
+        # returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1)
+
+        # FIXME: pyfolio integration
+        # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results)
+        # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01'])
+        # pyfolio.create_full_tear_sheet(*pf_data)
+
+    elif parameters.MODE == 'paper':
+        results = run_algorithm(
+            capital_base=parameters.CAPITAL_BASE,
+            data_frequency=parameters.DATA_FREQUECY,
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            live=True,
+            simulate_orders=True,
+            live_graph=False
+        )
+
+    elif parameters.MODE == 'live':
+        results = run_algorithm(
+            initialize=initialize,
+            handle_data=default_handle_data,
+            analyze=default_analyze,
+            exchange_name=parameters.EXCHANGE_NAME,
+            algo_namespace=parameters.ALGO_NAMESPACE,
+            base_currency=parameters.BASE_CURRENCY,
+            live=True,
+            live_graph=True
+        )

catalyst/support/issue_236.py

@@ -0,0 +1,34 @@
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+coins = ['dash', 'btc', 'dash', 'etc', 'eth', 'ltc', 'nxt', 'rep', 'str',
+         'xmr', 'xrp', 'zec']
+symbols = None
+
+
+def initialize(context):
+    pass
+
+
+def _handle_data(context, data):
+    global symbols
+    if symbols is None: symbols = [symbol(c + '_usdt') for c in coins]
+
+    print('getting history for: %s' % [s.symbol for s in symbols])
+    history = data.history(symbols,
+                           ['close', 'volume'],
+                           bar_count=1,  # EXCEPTION, Change to 2
+                           frequency='5T')
+    # print 'history: %s' % history.shape
+
+
+run_algorithm(initialize=initialize,
+              handle_data=_handle_data,
+              analyze=lambda _, results: True,
+              exchange_name='poloniex',
+              base_currency='usdt',
+              algo_namespace='issue-236',
+              live=True,
+              data_frequency='minute',
+              capital_base=3000,
+              simulate_orders=True)

catalyst/support/issue_30.py

@@ -0,0 +1,28 @@
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+
+def initialize(context):
+    context.asset = symbol('bcc_usdt')
+
+
+def handle_data(context, data):
+    data.history(context.asset, ['close'], bar_count=100, frequency='5T')
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=100,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bittrex',
+        algo_namespace="bittrex_is_broken",
+        base_currency='usdt',
+        data_frequency='minute',
+        simulate_orders=True,
+        live=True)

catalyst/utils/run_algo.py

@@ -8,11 +8,15 @@ from time import sleep
 
 import click
 import pandas as pd
-from logbook import Logger
+from six import string_types
 
+import catalyst
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
+    TradingPairPricing
+from catalyst.exchange.utils.factory import get_exchange
+from logbook import Logger
 
 try:
     from pygments import highlight
@@ -20,7 +24,7 @@ try:
     from pygments.formatters import TerminalFormatter
 
     PYGMENTS = True
-except:
+except ImportError:
     PYGMENTS = False
 from toolz import valfilter, concatv
 from functools import partial
@@ -37,10 +41,7 @@ from catalyst.exchange.exchange_algorithm import (
 )
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, NotEnoughCapitalError)
+from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
 
 from catalyst.constants import LOG_LEVEL
 
@@ -55,6 +56,7 @@ class _RunAlgoError(click.ClickException, ValueError):
     ----------
     pyfunc_msg : str
         The message that will be shown when called as a python function.
+
     cmdline_msg : str
         The message that will be shown on the command line.
     """
@@ -91,12 +93,15 @@ def _run(handle_data,
          algo_namespace,
          base_currency,
          live_graph,
+         analyze_live,
          simulate_orders,
+         auth_aliases,
          stats_output):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
     """
+    # TODO: refactor for more granularity
     if algotext is not None:
         if local_namespace:
             ip = get_ipython()  # noqa
@@ -141,22 +146,54 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
-    mode = 'paper-trading' if simulate_orders else 'live-trading' \
-        if live else 'backtest'
+    log.warn(
+        'Catalyst is currently in ALPHA. It is going through rapid '
+        'development and it is subject to errors. Please use carefully. '
+        'We encourage you to report any issue on GitHub: '
+        'https://github.com/enigmampc/catalyst/issues'
+    )
+    log.info('Catalyst version {}'.format(catalyst.__version__))
+    sleep(3)
+
+    if live:
+        if simulate_orders:
+            mode = 'paper-trading'
+        else:
+            mode = 'live-trading'
+    else:
+        mode = 'backtest'
+
     log.info('running algo in {mode} mode'.format(mode=mode))
 
     exchange_name = exchange
     if exchange_name is None:
         raise ValueError('Please specify at least one exchange.')
 
-    exchange_list = [x.strip().lower() for x in exchange.split(',')]
+    if isinstance(auth_aliases, string_types):
+        aliases = auth_aliases.split(',')
+        if len(aliases) < 2 or len(aliases) % 2 != 0:
+            raise ValueError(
+                'the `auth_aliases` parameter must contain an even list '
+                'of comma-delimited values. For example, '
+                '"binance,auth2" or "binance,auth2,bittrex,auth2".'
+            )
 
+        auth_aliases = dict(zip(aliases[::2], aliases[1::2]))
+
+    exchange_list = [x.strip().lower() for x in exchange.split(',')]
     exchanges = dict()
-    for exchange_name in exchange_list:
-        exchanges[exchange_name] = get_exchange(
-            exchange_name=exchange_name,
+    for name in exchange_list:
+        if auth_aliases is not None and name in auth_aliases:
+            auth_alias = auth_aliases[name]
+        else:
+            auth_alias = None
+
+        exchanges[name] = get_exchange(
+            exchange_name=name,
             base_currency=base_currency,
             must_authenticate=(live and not simulate_orders),
+            skip_init=True,
+            auth_alias=auth_alias,
         )
 
     open_calendar = get_calendar('OPEN')
@@ -172,14 +209,22 @@ def _run(handle_data,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
     )
-    env.asset_finder = AssetFinderExchange()
-    choose_loader = None  # TODO: use the DataPortal in the algo class for this
+    env.asset_finder = ExchangeAssetFinder(exchanges=exchanges)
+
+    def choose_loader(column):
+        bound_cols = TradingPairPricing.columns
+        if column in bound_cols:
+            return ExchangePricingLoader(data_frequency)
+        raise ValueError(
+            "No PipelineLoader registered for column %s." % column
+        )
 
     if live:
         start = pd.Timestamp.utcnow()
 
         # TODO: fix the end data.
-        end = start + timedelta(hours=8760)
+        if end is None:
+            end = start + timedelta(hours=8760)
 
         data = DataPortalExchangeLive(
             exchanges=exchanges,
@@ -188,66 +233,6 @@ def _run(handle_data,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
 
-        def fetch_capital_base(exchange, attempt_index=0):
-            """
-            Fetch the base currency amount required to bootstrap
-            the algorithm against the exchange.
-
-            The algorithm cannot continue without this value.
-
-            :param exchange: the targeted exchange
-            :param attempt_index:
-            :return capital_base: the amount of base currency available for
-            trading
-            """
-            try:
-                log.debug('retrieving capital base in {} to bootstrap '
-                          'exchange {}'.format(base_currency, exchange_name))
-                balances = exchange.get_balances()
-            except ExchangeRequestError as e:
-                if attempt_index < 20:
-                    log.warn(
-                        'could not retrieve balances on {}: {}'.format(
-                            exchange.name, e
-                        )
-                    )
-                    sleep(5)
-                    return fetch_capital_base(exchange, attempt_index + 1)
-
-                else:
-                    raise ExchangeRequestErrorTooManyAttempts(
-                        attempts=attempt_index,
-                        error=e
-                    )
-
-            if base_currency in balances:
-                base_currency_available = balances[base_currency]['free']
-                log.info(
-                    'base currency available in the account: {} {}'.format(
-                        base_currency_available, base_currency
-                    )
-                )
-
-                return base_currency_available
-            else:
-                raise BaseCurrencyNotFoundError(
-                    base_currency=base_currency,
-                    exchange=exchange_name
-                )
-
-        if not simulate_orders:
-            for exchange_name in exchanges:
-                exchange = exchanges[exchange_name]
-                balance = fetch_capital_base(exchange)
-
-                if balance < capital_base:
-                    raise NotEnoughCapitalError(
-                        exchange=exchange_name,
-                        base_currency=base_currency,
-                        balance=balance,
-                        capital_base=capital_base,
-                    )
-
         sim_params = create_simulation_parameters(
             start=start,
             end=end,
@@ -266,6 +251,8 @@ def _run(handle_data,
             live_graph=live_graph,
             simulate_orders=simulate_orders,
             stats_output=stats_output,
+            analyze_live=analyze_live,
+            end=end,
         )
     elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
@@ -276,6 +263,15 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
+        if start != pd.tslib.normalize_date(start) or \
+                        end != pd.tslib.normalize_date(end):
+            # todo: add to Sim_Params the option to start & end at specific times 
+            log.warn(
+                "Catalyst currently starts and ends on the start and "
+                "end of the dates specified, respectively. We hope to "
+                "Modify this and support specific times in a future release."
+            )
+
         data = DataPortalExchangeBacktest(
             exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,
@@ -427,10 +423,13 @@ def run_algorithm(initialize,
                   base_currency=None,
                   algo_namespace=None,
                   live_graph=False,
+                  analyze_live=None,
                   simulate_orders=True,
+                  auth_aliases=None,
                   stats_output=None,
                   output=os.devnull):
-    """Run a trading algorithm.
+    """
+    Run a trading algorithm.
 
     Parameters
     ----------
@@ -472,7 +471,7 @@ def run_algorithm(initialize,
         This argument is mutually exclusive with ``data``.
     default_extension : bool, optional
         Should the default catalyst extension be loaded. This is found at
-        ``$ZIPLINE_ROOT/extension.py``
+        ``$CATALYST_ROOT/extension.py``
     extensions : iterable[str], optional
         The names of any other extensions to load. Each element may either be
         a dotted module path like ``a.b.c`` or a path to a python file ending
@@ -483,12 +482,8 @@ def run_algorithm(initialize,
     environ : mapping[str -> str], optional
         The os environment to use. Many extensions use this to get parameters.
         This defaults to ``os.environ``.
-    live: execute live trading
-    exchange_conn: The exchange connection parameters
-
-    Supported Exchanges
-    -------------------
-    bitfinex
+    live : bool, optional
+        Execute algorithm in live trading mode.
 
     Returns
     -------
@@ -559,6 +554,8 @@ def run_algorithm(initialize,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=analyze_live,
         simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
         stats_output=stats_output
     )

docs/Makefile

@@ -23,7 +23,7 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  build      to build the C and Cython extensions for zipline"
+	@echo "  build      to build the C and Cython extensions for catalyst"
 	@echo "  html       to make standalone HTML files"
 	@echo "  livehtml   to run a persistent process that rebuilds the docs"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
@@ -96,9 +96,9 @@ qthelp: build
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/zipline.qhcp"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/catalyst.qhcp"
 	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/zipline.qhc"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/catalyst.qhc"
 
 applehelp: build
 	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@@ -113,8 +113,8 @@ devhelp: build
 	@echo
 	@echo "Build finished."
 	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/zipline"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/zipline"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/catalyst"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/catalyst"
 	@echo "# devhelp"
 
 epub: build

docs/deploy.py

@@ -8,8 +8,8 @@ from shutil import move, rmtree
 from subprocess import check_call
 
 HERE = dirname(abspath(__file__))
-ZIPLINE_ROOT = dirname(HERE)
-TEMP_LOCATION = '/tmp/zipline-doc'
+CATALYST_ROOT = dirname(HERE)
+TEMP_LOCATION = '/tmp/catalyst-doc'
 TEMP_LOCATION_GLOB = TEMP_LOCATION + '/*'
 
 
@@ -46,8 +46,8 @@ def main():
             print("Copying built files to temp location.")
             move('build/html', TEMP_LOCATION)
 
-            print("Moving to '%s'" % ZIPLINE_ROOT)
-            os.chdir(ZIPLINE_ROOT)
+            print("Moving to '%s'" % CATALYST_ROOT)
+            os.chdir(CATALYST_ROOT)
 
             print("Checking out gh-pages branch.")
             check_call(
@@ -70,7 +70,7 @@ def main():
         os.chdir(old_dir)
 
     print()
-    print("Updated documentation branch in directory %s" % ZIPLINE_ROOT)
+    print("Updated documentation branch in directory %s" % CATALYST_ROOT)
     print("If you are happy with these changes, commit and push to gh-pages.")
 
 if __name__ == '__main__':

docs/make.bat

@@ -127,9 +127,9 @@ if "%1" == "qthelp" (
 	echo.
 	echo.Build finished; now you can run "qcollectiongenerator" with the ^
 .qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\zipline.qhcp
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\catalyst.qhcp
 	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\zipline.ghc
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\catalyst.ghc
 	goto end
 )
 

docs/source/appendix.rst

@@ -4,7 +4,7 @@ API Reference
 Running a Backtest
 ~~~~~~~~~~~~~~~~~~
 
-.. autofunction:: zipline.run_algorithm(...)
+.. autofunction:: catalyst.run_algorithm(...)
 
 Algorithm API
 ~~~~~~~~~~~~~
@@ -18,341 +18,335 @@ currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance.
 Data Object
 ```````````
 
-.. autoclass:: zipline.protocol.BarData
+.. autoclass:: catalyst.protocol.BarData
    :members:
 
 Scheduling Functions
 ````````````````````
 
-.. autofunction:: zipline.api.schedule_function
+.. autofunction:: catalyst.api.schedule_function
 
-.. autoclass:: zipline.api.date_rules
+.. autoclass:: catalyst.api.date_rules
    :members:
    :undoc-members:
 
-.. autoclass:: zipline.api.time_rules
+.. autoclass:: catalyst.api.time_rules
    :members:
 
 Orders
 ``````
 
-.. autofunction:: zipline.api.order
+.. autofunction:: catalyst.api.order
 
-.. autofunction:: zipline.api.order_value
+.. autofunction:: catalyst.api.order_value
 
-.. autofunction:: zipline.api.order_percent
+.. autofunction:: catalyst.api.order_percent
 
-.. autofunction:: zipline.api.order_target
+.. autofunction:: catalyst.api.order_target
 
-.. autofunction:: zipline.api.order_target_value
+.. autofunction:: catalyst.api.order_target_value
 
-.. autofunction:: zipline.api.order_target_percent
+.. autofunction:: catalyst.api.order_target_percent
 
-.. autoclass:: zipline.finance.execution.ExecutionStyle
+.. autoclass:: catalyst.finance.execution.ExecutionStyle
    :members:
 
-.. autoclass:: zipline.finance.execution.MarketOrder
+.. autoclass:: catalyst.finance.execution.MarketOrder
 
-.. autoclass:: zipline.finance.execution.LimitOrder
+.. autoclass:: catalyst.finance.execution.LimitOrder
 
-.. autoclass:: zipline.finance.execution.StopOrder
+.. autoclass:: catalyst.finance.execution.StopOrder
 
-.. autoclass:: zipline.finance.execution.StopLimitOrder
+.. autoclass:: catalyst.finance.execution.StopLimitOrder
 
-.. autofunction:: zipline.api.get_order
+.. autofunction:: catalyst.api.get_order
 
-.. autofunction:: zipline.api.get_open_orders
+.. autofunction:: catalyst.api.get_open_orders
 
-.. autofunction:: zipline.api.cancel_order
+.. autofunction:: catalyst.api.cancel_order
 
 Order Cancellation Policies
 '''''''''''''''''''''''''''
 
-.. autofunction:: zipline.api.set_cancel_policy
+.. autofunction:: catalyst.api.set_cancel_policy
 
-.. autoclass:: zipline.finance.cancel_policy.CancelPolicy
+.. autoclass:: catalyst.finance.cancel_policy.CancelPolicy
    :members:
 
-.. autofunction:: zipline.api.EODCancel
+.. autofunction:: catalyst.api.EODCancel
 
-.. autofunction:: zipline.api.NeverCancel
+.. autofunction:: catalyst.api.NeverCancel
 
 
 Assets
 ``````
 
-.. autofunction:: zipline.api.symbol
+.. autofunction:: catalyst.api.symbol
 
-.. autofunction:: zipline.api.symbols
+.. autofunction:: catalyst.api.symbols
 
-.. autofunction:: zipline.api.future_symbol
+.. autofunction:: catalyst.api.set_symbol_lookup_date
 
-.. autofunction:: zipline.api.set_symbol_lookup_date
-
-.. autofunction:: zipline.api.sid
+.. autofunction:: catalyst.api.sid
 
 
 Trading Controls
 ````````````````
 
-Zipline provides trading controls to help ensure that the algorithm is
+zipline provides trading controls to help ensure that the algorithm is
 performing as expected. The functions help protect the algorithm from certian
 bugs that could cause undesirable behavior when trading with real money.
 
-.. autofunction:: zipline.api.set_do_not_order_list
+.. autofunction:: catalyst.api.set_do_not_order_list
 
-.. autofunction:: zipline.api.set_long_only
+.. autofunction:: catalyst.api.set_long_only
 
-.. autofunction:: zipline.api.set_max_leverage
+.. autofunction:: catalyst.api.set_max_leverage
 
-.. autofunction:: zipline.api.set_max_order_count
+.. autofunction:: catalyst.api.set_max_order_count
 
-.. autofunction:: zipline.api.set_max_order_size
+.. autofunction:: catalyst.api.set_max_order_size
 
-.. autofunction:: zipline.api.set_max_position_size
+.. autofunction:: catalyst.api.set_max_position_size
 
 
 Simulation Parameters
 `````````````````````
 
-.. autofunction:: zipline.api.set_benchmark
+.. autofunction:: catalyst.api.set_benchmark
 
 Commission Models
 '''''''''''''''''
 
-.. autofunction:: zipline.api.set_commission
+.. autofunction:: catalyst.api.set_commission
 
-.. autoclass:: zipline.finance.commission.CommissionModel
+.. autoclass:: catalyst.finance.commission.CommissionModel
    :members:
 
-.. autoclass:: zipline.finance.commission.PerShare
+.. autoclass:: catalyst.finance.commission.PerShare
 
-.. autoclass:: zipline.finance.commission.PerTrade
+.. autoclass:: catalyst.finance.commission.PerTrade
 
-.. autoclass:: zipline.finance.commission.PerDollar
+.. autoclass:: catalyst.finance.commission.PerDollar
 
 Slippage Models
 '''''''''''''''
 
-.. autofunction:: zipline.api.set_slippage
+.. autofunction:: catalyst.api.set_slippage
 
-.. autoclass:: zipline.finance.slippage.SlippageModel
+.. autoclass:: catalyst.finance.slippage.SlippageModel
    :members:
 
-.. autoclass:: zipline.finance.slippage.FixedSlippage
+.. autoclass:: catalyst.finance.slippage.FixedSlippage
 
-.. autoclass:: zipline.finance.slippage.VolumeShareSlippage
+.. autoclass:: catalyst.finance.slippage.VolumeShareSlippage
 
 Pipeline
 ````````
 
-For more information, see :ref:`pipeline-api`
+Not supported yet.
 
-.. autofunction:: zipline.api.attach_pipeline
+.. For more information, see :ref:`pipeline-api`
 
-.. autofunction:: zipline.api.pipeline_output
+.. .. autofunction:: catalyst.api.attach_pipeline
+
+.. .. autofunction:: catalyst.api.pipeline_output
 
 
 Miscellaneous
 `````````````
 
-.. autofunction:: zipline.api.record
+.. autofunction:: catalyst.api.record
 
-.. autofunction:: zipline.api.get_environment
+.. autofunction:: catalyst.api.get_environment
 
-.. autofunction:: zipline.api.fetch_csv
+.. autofunction:: catalyst.api.fetch_csv
 
 
 .. _pipeline-api:
 
-Pipeline API
-~~~~~~~~~~~~
+.. Pipeline API
+.. ~~~~~~~~~~~~
 
-.. autoclass:: zipline.pipeline.Pipeline
-   :members:
-   :member-order: groupwise
+.. .. autoclass:: zipline.pipeline.Pipeline
+..    :members:
+..    :member-order: groupwise
 
-.. autoclass:: zipline.pipeline.CustomFactor
-   :members:
-   :member-order: groupwise
+.. .. autoclass:: zipline.pipeline.CustomFactor
+..    :members:
+..    :member-order: groupwise
 
-.. autoclass:: zipline.pipeline.filters.Filter
-   :members: __and__, __or__
-   :exclude-members: dtype
+.. .. autoclass:: zipline.pipeline.filters.Filter
+..    :members: __and__, __or__
+..    :exclude-members: dtype
 
-.. autoclass:: zipline.pipeline.factors.Factor
-   :members: bottom, deciles, demean, linear_regression, pearsonr,
-             percentile_between, quantiles, quartiles, quintiles, rank,
-             spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
-             __add__, __sub__, __mul__, __div__, __mod__, __pow__, __lt__,
-             __le__, __ne__, __ge__, __gt__
-   :exclude-members: dtype
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.factors.Factor
+..    :members: bottom, deciles, demean, linear_regression, pearsonr,
+..              percentile_between, quantiles, quartiles, quintiles, rank,
+..              spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
+..              \__add__, \__sub__, \__mul__, \__div__, \__mod__, \__pow__, 
+..              \__lt__, \__le__, \__ne__, \__ge__, \__gt__
+..    :exclude-members: dtype
+..    :member-order: bysource
 
-.. autoclass:: zipline.pipeline.term.Term
-   :members:
-   :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
+.. .. autoclass:: zipline.pipeline.term.Term
+..    :members:
+..    :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
 
-.. autoclass:: zipline.pipeline.data.USEquityPricing
-   :members: open, high, low, close, volume
-   :undoc-members:
+.. .. autoclass:: zipline.pipeline.data.USEquityPricing
+..    :members: open, high, low, close, volume
+..    :undoc-members:
 
-Built-in Factors
-````````````````
+.. Built-in Factors
+.. ````````````````
 
-.. autoclass:: zipline.pipeline.factors.AverageDollarVolume
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.AverageDollarVolume
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BollingerBands
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BollingerBands
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.Latest
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.Latest
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.MaxDrawdown
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.MaxDrawdown
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.Returns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.Returns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.RSI
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.RSI
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.VWAP
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.VWAP
+..    :members:
 
-.. autoclass:: zipline.pipeline.factors.WeightedAverageValue
-   :members:
+.. .. autoclass:: zipline.pipeline.factors.WeightedAverageValue
+..    :members:
 
-Pipeline Engine
-```````````````
+.. Pipeline Engine
+.. ```````````````
 
-.. autoclass:: zipline.pipeline.engine.PipelineEngine
-   :members: run_pipeline, run_chunked_pipeline
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.engine.PipelineEngine
+..    :members: run_pipeline, run_chunked_pipeline
+..    :member-order: bysource
 
-.. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
-   :members: __init__, run_pipeline, run_chunked_pipeline
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
+..    :members: __init__, run_pipeline, run_chunked_pipeline
+..    :member-order: bysource
 
-.. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
+.. .. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
 
-Data Loaders
-````````````
+.. Data Loaders
+.. ````````````
 
-.. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
-   :members: __init__, from_files, load_adjusted_array
-   :member-order: bysource
+.. .. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
+..    :members: __init__, from_files, load_adjusted_array
+..    :member-order: bysource
 
 Asset Metadata
 ~~~~~~~~~~~~~~
 
-.. autoclass:: zipline.assets.Asset
+.. autoclass:: catalyst.assets.Asset
    :members:
 
-.. autoclass:: zipline.assets.Equity
-   :members:
-
-.. autoclass:: zipline.assets.Future
-   :members:
-
-.. autoclass:: zipline.assets.AssetConvertible
+.. autoclass:: catalyst.assets.AssetConvertible
    :members:
 
 
 Trading Calendar API
 ~~~~~~~~~~~~~~~~~~~~
 
-.. autofunction:: zipline.utils.calendars.get_calendar
+.. autofunction:: catalyst.utils.calendars.get_calendar
 
-.. autoclass:: zipline.utils.calendars.TradingCalendar
+.. autoclass:: catalyst.utils.calendars.TradingCalendar
    :members:
 
-.. autofunction:: zipline.utils.calendars.register_calendar
+.. autofunction:: catalyst.utils.calendars.register_calendar
 
-.. autofunction:: zipline.utils.calendars.register_calendar_type
+.. autofunction:: catalyst.utils.calendars.register_calendar_type
 
-.. autofunction:: zipline.utils.calendars.deregister_calendar
+.. autofunction:: catalyst.utils.calendars.deregister_calendar
 
-.. autofunction:: zipline.utils.calendars.clear_calendars
+.. autofunction:: catalyst.utils.calendars.clear_calendars
 
 
 Data API
 ~~~~~~~~
 
-Writers
-```````
-.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
-   :members:
+.. Writers
+.. ```````
+.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
+..    :members:
 
-.. autoclass:: zipline.assets.AssetDBWriter
-   :members:
+.. .. autoclass:: zipline.assets.AssetDBWriter
+..    :members:
 
-Readers
-```````
-.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
-   :members:
+.. Readers
+.. ```````
+.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
+..    :members:
 
-.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
-   :members:
+.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
+..    :members:
 
-.. autoclass:: zipline.assets.AssetFinder
-   :members:
+.. .. autoclass:: zipline.assets.AssetFinder
+..    :members:
 
-.. autoclass:: zipline.data.data_portal.DataPortal
-   :members:
+.. .. autoclass:: zipline.data.data_portal.DataPortal
+..    :members:
 
-Bundles
-```````
-.. autofunction:: zipline.data.bundles.register
+.. Bundles
+.. ```````
+.. .. autofunction:: zipline.data.bundles.register
 
-.. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
+.. .. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
 
-.. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
+.. .. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
 
-.. autofunction:: zipline.data.bundles.unregister
+.. .. autofunction:: zipline.data.bundles.unregister
 
-.. data:: zipline.data.bundles.bundles
+.. .. data:: zipline.data.bundles.bundles
 
-   The bundles that have been registered as a mapping from bundle name to bundle
-   data. This mapping is immutable and should only be updated through
-   :func:`~zipline.data.bundles.register` or
-   :func:`~zipline.data.bundles.unregister`.
+..    The bundles that have been registered as a mapping from bundle name to bundle
+..    data. This mapping is immutable and should only be updated through
+..    :func:`~zipline.data.bundles.register` or
+..    :func:`~zipline.data.bundles.unregister`.
 
-.. autofunction:: zipline.data.bundles.yahoo_equities
+.. .. autofunction:: zipline.data.bundles.yahoo_equities
 
 
 
@@ -362,16 +356,16 @@ Utilities
 Caching
 ```````
 
-.. autoclass:: zipline.utils.cache.CachedObject
+.. autoclass:: catalyst.utils.cache.CachedObject
 
-.. autoclass:: zipline.utils.cache.ExpiringCache
+.. autoclass:: catalyst.utils.cache.ExpiringCache
 
-.. autoclass:: zipline.utils.cache.dataframe_cache
+.. autoclass:: catalyst.utils.cache.dataframe_cache
 
-.. autoclass:: zipline.utils.cache.working_file
+.. autoclass:: catalyst.utils.cache.working_file
 
-.. autoclass:: zipline.utils.cache.working_dir
+.. autoclass:: catalyst.utils.cache.working_dir
 
 Command Line
 ````````````
-.. autofunction:: zipline.utils.cli.maybe_show_progress
+.. autofunction:: catalyst.utils.cli.maybe_show_progress

docs/source/beginner-tutorial.rst

@@ -168,7 +168,7 @@ We'll start with the CLI, and introduce the ``run_algorithm()`` in the last
 example of this tutorial. Some of the :doc:`example algorithms <example-algos>` 
 provide instructions on how to run them both from the CLI, and using the 
 :func:`~catalyst.run_algorithm` function. For the third method, refer to the 
-corresponding section on :doc:`Catalyst & Jupyter Notebook <jupyter>` after you 
+corresponding section on :ref:`Catalyst & Jupyter Notebook <jupyter>` after you 
 have assimilated the contents of this tutorial.
 
 Command line interface
@@ -473,6 +473,7 @@ Which we execute by running:
     </div>
 
 |
+
 There is a row for each trading day, starting on the first day of our 
 simulation Jan 1st, 2016. In the columns you can find various
 information about the state of your algorithm. The column
@@ -483,7 +484,7 @@ bitcoin price.
 
 Now we will run the simulation again, but this time we extend our original 
 algorithm with the addition of the ``analyze()`` function. Somewhat analogously 
-as how ``initialize()`` gets called once before the start of the algorith, 
+as how ``initialize()`` gets called once before the start of the algorithm, 
 ``analyze()`` gets called once at the end of the algorithm, and receives two 
 variables: ``context``, which we discussed at the very beginning, and ``perf``, 
 which is the pandas dataframe containing the performance data for our algorithm 
@@ -518,7 +519,7 @@ alongside enigma-catalyst (with the exception of the ``Conda`` install, where it
 was included by default inside the conda environment we created). If for any 
 reason you don't have it installed, you can add it by running:
 
-.. code-block:: python
+.. code-block:: bash
 
   (catalyst)$ pip install matplotlib
 
@@ -579,161 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in
 the ``examples`` directory:
 `dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
 
-.. code-block:: python
-
-    import numpy as np
-    import pandas as pd
-    from logbook import Logger
-    import matplotlib.pyplot as plt 
-
-    from catalyst import run_algorithm
-    from catalyst.api import (order, record, symbol, order_target_percent, 
-            get_open_orders)
-    from catalyst.exchange.stats_utils import extract_transactions
-
-    NAMESPACE = 'dual_moving_average'
-    log = Logger(NAMESPACE)
-
-    def initialize(context):
-        context.i = 0
-        context.asset = symbol('ltc_usd')
-        context.base_price = None
-
-
-    def handle_data(context, data):
-        # define the windows for the moving averages 
-        short_window = 50
-        long_window = 200
-
-        # Skip as many bars as long_window to properly compute the average
-        context.i += 1
-        if context.i < long_window:
-           return
-
-        # Compute moving averages calling data.history() for each 
-        # moving average with the appropriate parameters. We choose to use
-        # minute bars for this simulation -> freq="1m"
-        # Returns a pandas dataframe.
-        short_mavg = data.history(context.asset, 'price', 
-                            bar_count=short_window, frequency="1m").mean()
-        long_mavg = data.history(context.asset, 'price',
-                            bar_count=long_window, frequency="1m").mean()
-
-        # Let's keep the price of our asset in a more handy variable
-        price = data.current(context.asset, 'price')
-
-        # If base_price is not set, we use the current value. This is the
-        # price at the first bar which we reference to calculate price_change.
-        if context.base_price is None:
-            context.base_price = price
-        price_change = (price - context.base_price) / context.base_price
-
-        # Save values for later inspection
-        record(price=price,
-               cash=context.portfolio.cash,
-               price_change=price_change,
-               short_mavg=short_mavg,
-               long_mavg=long_mavg)
-
-        # Since we are using limit orders, some orders may not execute immediately
-        # we wait until all orders are executed before considering more trades.
-        orders = get_open_orders(context.asset)
-        if len(orders) > 0:
-            return
-
-        # Exit if we cannot trade
-        if not data.can_trade(context.asset):
-            return
-
-        # We check what's our position on our portfolio and trade accordingly
-        pos_amount = context.portfolio.positions[context.asset].amount
-
-        # Trading logic
-        if short_mavg > long_mavg and pos_amount == 0:
-           # we buy 100% of our portfolio for this asset
-           order_target_percent(context.asset, 1)
-        elif short_mavg < long_mavg and pos_amount > 0:
-           # we sell all our positions for this asset
-           order_target_percent(context.asset, 0)
-
-
-    def analyze(context, perf):
-
-        # Get the base_currency that was passed as a parameter to the simulation
-        base_currency = context.exchanges.values()[0].base_currency.upper()
-
-        # First chart: Plot portfolio value using base_currency
-        ax1 = plt.subplot(411)
-        perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-        ax1.legend_.remove()
-        ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-        start, end = ax1.get_ylim()
-        ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Second chart: Plot asset price, moving averages and buys/sells
-        ax2 = plt.subplot(412, sharex=ax1)
-        perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-        ax2.legend_.remove()
-        ax2.set_ylabel('{asset}\n({base})'.format(
-            asset = context.asset.symbol,
-            base = base_currency
-            ))
-        start, end = ax2.get_ylim()
-        ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        transaction_df = extract_transactions(perf)
-        if not transaction_df.empty:
-            buy_df = transaction_df[transaction_df['amount'] > 0]
-            sell_df = transaction_df[transaction_df['amount'] < 0]
-            ax2.scatter(
-                buy_df.index.to_pydatetime(),
-                perf.loc[buy_df.index, 'price'],
-                marker='^',
-                s=100,
-                c='green',
-                label=''
-            )
-            ax2.scatter(
-                sell_df.index.to_pydatetime(),
-                perf.loc[sell_df.index, 'price'],
-                marker='v',
-                s=100,
-                c='red',
-                label=''
-            )
-
-        # Third chart: Compare percentage change between our portfolio
-        # and the price of the asset
-        ax3 = plt.subplot(413, sharex=ax1)
-        perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-        ax3.legend_.remove()
-        ax3.set_ylabel('Percent Change')
-        start, end = ax3.get_ylim()
-        ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-        # Fourth chart: Plot our cash
-        ax4 = plt.subplot(414, sharex=ax1)
-        perf.cash.plot(ax=ax4)
-        ax4.set_ylabel('Cash\n({})'.format(base_currency))
-        start, end = ax4.get_ylim()
-        ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-        plt.show()
-
-
-    if __name__ == '__main__':
-        run_algorithm(
-                capital_base=1000,
-                data_frequency='minute',
-                initialize=initialize,
-                handle_data=handle_data,
-                analyze=analyze,
-                exchange_name='bitfinex',
-                algo_namespace=NAMESPACE,
-                base_currency='usd',
-                start=pd.to_datetime('2017-9-22', utc=True),
-                end=pd.to_datetime('2017-9-23', utc=True),
-            )
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
+   :language: python
 
 In order to run the code above, you have to ingest the needed data first:
 
@@ -805,6 +653,7 @@ the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
 ``pandas.DataFrame``\ s, so you can simply pass the underlying
 ``ndarray`` of a ``DataFrame`` via ``.values``).
 
+.. _jupyter:
 
 Jupyter Notebook
 ~~~~~~~~~~~~~~~~
@@ -825,13 +674,13 @@ In order to use Jupyter Notebook, you first have to install it inside your
 environment. It's available as ``pip`` package, so regardless of how you 
 installed Catalyst, go inside your catalyst environemnt and run:
 
-.. code:: bash
+.. code-block:: bash
 
     (catalyst)$ pip install jupyter
 
 Once you have Jupyter Notebook installed, every time you want to use it run:
 
-.. code:: bash
+.. code-block:: bash
 
     (catalyst)$ jupyter notebook
 
@@ -845,7 +694,7 @@ Before running your algorithms inside the Jupyter Notebook, remember to ingest
 the data from the command line interface (CLI). In the example below, you would
 need to run first:
 
-.. code:: bash
+.. code-block:: bash
 
   catalyst ingest-exchange -x bitfinex -i btc_usd
 
@@ -928,7 +777,6 @@ functions.
                 context.asset,
                 target_hodl_value,
                 limit_price=price*1.1,
-                stop_price=price*0.9,
             )
 
         record(
@@ -16607,7 +16455,49 @@ NaN
 
    </div>
 
+PyCharm IDE
+~~~~~~~~~~~
 
+PyCharm is an Integrated Development Environment (IDE) used in computer 
+programming, specifically for the Python language. It streamlines the continuos
+development of Python code, and among other things includes a debugger that 
+comes in handy to see the inner workings of Catalyst, and your trading 
+algorithms.
+
+Install
+^^^^^^^
+Install PyCharm from their `Website <https://www.jetbrains.com/pycharm/download/>`__.
+There is a free and open-source **Community** version.
+
+Setup
+^^^^^
+
+1. When creating a new project in PyCharm, right under you specify the Location,
+   click on **Project Interpreter** to display a drop down menu
+
+2. Select **Existing interpreter**, click the gear box right next to it and 
+   select 'add local'. Depending on your installation, select either 
+   "*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to
+   navigate to your catalyst env and select the Python binary file: 
+   ``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows 
+   installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe'). 
+   Select OK. You may want to click on *Make available to all projects* for your
+   future reference. Click OK again, and create your new environment using the
+   set up of your virtual environment.
+
+Alternatively, if you already have your project created, in Windows do:
+
+1. File -> Default Settings -> Project Interpreter. Click the gear box next to 
+   the project interpreter and select ‘add local’, and follow the steps from the
+   second step above.
+
+On MacOS:
+
+1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ -> 
+   Project Interpreter. Click the gear box next to the project interpreter 
+   and select ‘add local’, and follow the steps from the second step above.
+
+You should now be able to run your project/scripts in PyCharm.
 
 Next steps
 ~~~~~~~~~~

docs/source/conf.py

@@ -27,8 +27,8 @@ extlinks = {
 
 # -- Docstrings ---------------------------------------------------------------
 
-#extensions += ['numpydoc']
-#numpydoc_show_class_members = False
+extensions += ['numpydoc']
+numpydoc_show_class_members = False
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['.templates']
@@ -41,11 +41,11 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC, Inc.'
+copyright = u'2018, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
-release = '0.3'
+release = '0.4'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -97,3 +97,6 @@ intersphinx_mapping = {
 doctest_global_setup = "import catalyst"
 
 todo_include_todos = True
+
+suppress_warnings = ['image.nonlocal_uri']
+

docs/source/development-guidelines.rst

@@ -36,25 +36,15 @@ Finally, you can build the C extensions by running:
 
    $ python setup.py build_ext --inplace
 
-.. To finish, make sure `tests`__ pass.
+Development with Docker
+-----------------------
 
-.. __ #style-guide-running-tests
+If you want to work with zipline using a `Docker`__ container, you'll need to 
+build the ``Dockerfile`` in the Zipline root directory, and then build 
+``Dockerfile-dev``. Instructions for building both containers can be found in 
+``Dockerfile`` and ``Dockerfile-dev``, respectively.
 
-.. If you get an error running nosetests after setting up a fresh virtualenv, please try running
-
-.. code-block
-
-..   # where zipline is the name of your virtualenv
-..   $ deactivate zipline
-..   $ workon zipline
-
-
-.. Development with Docker
-.. -----------------------
-
-..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
-
-.. __ https://docs.docker.com/get-started/
+__ https://docs.docker.com/get-started/
    
 Git Branching Structure
 -----------------------
@@ -84,6 +74,25 @@ To build and view the docs locally, run:
    $ {BROWSER} build/html/index.html
 
 
+There is a `documented issue <https://github.com/sphinx-doc/sphinx/issues/3212>`_ 
+with ``sphinx`` and ``docutils`` that causes the error below when trying to build 
+the docs.
+
+.. code-block:: text
+
+   Exception occurred:
+     File "(...)/env-c/lib/python2.7/site-packages/docutils/writers/_html_base.py", line 671, in depart_document
+       assert not self.context, 'len(context) = %s' % len(self.context)
+   AssertionError: len(context) = 3
+
+If you get this error, you need to downgrade your version of ``docutils`` as 
+follows, and build the docs again:
+
+.. code-block:: bash
+
+   $ pip install docutils==0.12
+
+
 Commit messages
 ---------------
 

docs/source/example-algos.rst

@@ -1,4 +1,5 @@
 |
+
 Example Algorithms
 ==================
 
@@ -51,35 +52,8 @@ Buy BTC Simple Algorithm
 
 Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
 
-.. code-block:: python
-
-  '''
-    Run this example, by executing the following from your terminal:
-      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
-      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    If you want to run this code using another exchange, make sure that 
-    the asset is available on that exchange. For example, if you were to run 
-    it for exchange Poloniex, you would need to edit the following line:
-
-      context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
-
-    and specify exchange poloniex as follows:
-    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
-    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-    To see which assets are available on each exchange, visit:
-    https://www.enigma.co/catalyst/status
-  '''
-
-  from catalyst.api import order, record, symbol
-
-  def initialize(context):
-      context.asset = symbol('btc_usd')
-
-  def handle_data(context, data):
-      order(context.asset, 1)
-      record(btc = data.current(context.asset, 'price'))
+.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
+   :language: python
 
 This simple algorithm does not produce any output nor displays any chart.
 
@@ -89,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart.
 Buy and Hodl Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
-
 First ingest the historical pricing data needed to run this algorithm:
 
 .. code-block:: bash
@@ -118,158 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
 earlier date, you'll get an error), and the most recent date you can choose is 
 one day prior to the current date. 
 
+Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
 
-.. code-block:: python
-
-  #!/usr/bin/env python
-  #
-  # Copyright 2017 Enigma MPC, Inc.
-  # Copyright 2015 Quantopian, Inc.
-  #
-  # Licensed under the Apache License, Version 2.0 (the "License");
-  # you may not use this file except in compliance with the License.
-  # You may obtain a copy of the License at
-  #
-  #     http://www.apache.org/licenses/LICENSE-2.0
-  #
-  # Unless required by applicable law or agreed to in writing, software
-  # distributed under the License is distributed on an "AS IS" BASIS,
-  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  # See the License for the specific language governing permissions and
-  # limitations under the License.
-  import pandas as pd
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order_target_value, symbol, record,
-      cancel_order, get_open_orders, )
-
-
-  def initialize(context):
-      context.ASSET_NAME = 'btc_usd'
-      context.TARGET_HODL_RATIO = 0.8
-      context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
-
-      context.is_buying = True
-      context.asset = symbol(context.ASSET_NAME)
-
-      context.i = 0
-
-
-  def handle_data(context, data):
-      context.i += 1
-
-      starting_cash = context.portfolio.starting_cash
-      target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
-      reserve_value = context.RESERVE_RATIO * starting_cash
-
-      # Cancel any outstanding orders
-      orders = get_open_orders(context.asset) or []
-      for order in orders:
-          cancel_order(order)
-
-      # Stop buying after passing the reserve threshold
-      cash = context.portfolio.cash
-      if cash <= reserve_value:
-          context.is_buying = False
-
-      # Retrieve current asset price from pricing data
-      price = data.current(context.asset, 'price')
-
-      # Check if still buying and could (approximately) afford another purchase
-      if context.is_buying and cash > price:
-          print('buying')
-          # Place order to make position in asset equal to target_hodl_value
-          order_target_value(
-              context.asset,
-              target_hodl_value,
-              limit_price=price * 1.1,
-              stop_price=price * 0.9,
-          )
-
-      record(
-          price=price,
-          volume=data.current(context.asset, 'volume'),
-          cash=cash,
-          starting_cash=context.portfolio.starting_cash,
-          leverage=context.account.leverage,
-      )
-
-
-  def analyze(context=None, results=None):
-
-      # Plot the portfolio and asset data.
-      ax1 = plt.subplot(611)
-      results[['portfolio_value']].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio Value (USD)')
-
-      ax2 = plt.subplot(612, sharex=ax1)
-      ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-      results[['price']].plot(ax=ax2)
-
-      trans = results.ix[[t != [] for t in results.transactions]]
-      buys = trans.ix[
-          [t[0]['amount'] > 0 for t in trans.transactions]
-      ]
-      ax2.scatter(
-          buys.index.to_pydatetime(),
-          results.price[buys.index],
-          marker='^',
-          s=100,
-          c='g',
-          label=''
-      )
-
-      ax3 = plt.subplot(613, sharex=ax1)
-      results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-      ax3.set_ylabel('Leverage ')
-
-      ax4 = plt.subplot(614, sharex=ax1)
-      results[['starting_cash', 'cash']].plot(ax=ax4)
-      ax4.set_ylabel('Cash (USD)')
-
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]] = results[[
-          'treasury_period_return',
-          'algorithm_period_return',
-          'benchmark_period_return',
-      ]]
-
-      ax5 = plt.subplot(615, sharex=ax1)
-      results[[
-          'treasury',
-          'algorithm',
-          'benchmark',
-      ]].plot(ax=ax5)
-      ax5.set_ylabel('Percent Change')
-
-      ax6 = plt.subplot(616, sharex=ax1)
-      results[['volume']].plot(ax=ax6)
-      ax6.set_ylabel('Volume (mCoins/5min)')
-
-      plt.legend(loc=3)
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-          capital_base=10000,
-          data_frequency='daily',
-          initialize=initialize,
-          handle_data=handle_data,
-          analyze=analyze,
-          exchange_name='bitfinex',
-          algo_namespace='buy_and_hodl',
-          base_currency='usd',
-          start=pd.to_datetime('2015-03-01', utc=True),
-          end=pd.to_datetime('2017-10-31', utc=True),
-      )
+.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
 
@@ -278,166 +102,13 @@ one day prior to the current date.
 Dual Moving Average Crossover
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
-
 This strategy is covered in detail in the last part of 
 `this tutorial <beginner-tutorial.html#history>`_.
 
-.. code-block:: python
+Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
 
-  import numpy as np
-  import pandas as pd
-  from logbook import Logger
-  import matplotlib.pyplot as plt
-
-  from catalyst import run_algorithm
-  from catalyst.api import (order, record, symbol, order_target_percent,
-          get_open_orders)
-  from catalyst.exchange.stats_utils import extract_transactions
-
-  NAMESPACE = 'dual_moving_average'
-  log = Logger(NAMESPACE)
-
-  def initialize(context):
-      context.i = 0
-      context.asset = symbol('ltc_usd')
-      context.base_price = None
-
-
-  def handle_data(context, data):
-      # define the windows for the moving averages
-      short_window = 50
-      long_window = 200
-
-      # Skip as many bars as long_window to properly compute the average
-      context.i += 1
-      if context.i < long_window:
-         return
-
-      # Compute moving averages calling data.history() for each
-      # moving average with the appropriate parameters. We choose to use
-      # minute bars for this simulation -> freq="1m"
-      # Returns a pandas dataframe.
-      short_mavg = data.history(context.asset, 'price',
-                          bar_count=short_window, frequency="1m").mean()
-      long_mavg = data.history(context.asset, 'price',
-                          bar_count=long_window, frequency="1m").mean()
-
-      # Let's keep the price of our asset in a more handy variable
-      price = data.current(context.asset, 'price')
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-      price_change = (price - context.base_price) / context.base_price
-
-      # Save values for later inspection
-      record(price=price,
-             cash=context.portfolio.cash,
-             price_change=price_change,
-             short_mavg=short_mavg,
-             long_mavg=long_mavg)
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.asset)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.asset):
-          return
-
-      # We check what's our position on our portfolio and trade accordingly
-      pos_amount = context.portfolio.positions[context.asset].amount
-
-      # Trading logic
-      if short_mavg > long_mavg and pos_amount == 0:
-         # we buy 100% of our portfolio for this asset
-         order_target_percent(context.asset, 1)
-      elif short_mavg < long_mavg and pos_amount > 0:
-         # we sell all our positions for this asset
-         order_target_percent(context.asset, 0)
-
-
-  def analyze(context, perf):
-
-      # Get the base_currency that was passed as a parameter to the simulation
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # First chart: Plot portfolio value using base_currency
-      ax1 = plt.subplot(411)
-      perf.loc[:, ['portfolio_value']].plot(ax=ax1)
-      ax1.legend_.remove()
-      ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
-      start, end = ax1.get_ylim()
-      ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Second chart: Plot asset price, moving averages and buys/sells
-      ax2 = plt.subplot(412, sharex=ax1)
-      perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
-      ax2.legend_.remove()
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset = context.asset.symbol,
-          base = base_currency
-          ))
-      start, end = ax2.get_ylim()
-      ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index, 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index, 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      # Third chart: Compare percentage change between our portfolio
-      # and the price of the asset
-      ax3 = plt.subplot(413, sharex=ax1)
-      perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
-      ax3.legend_.remove()
-      ax3.set_ylabel('Percent Change')
-      start, end = ax3.get_ylim()
-      ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
-
-      # Fourth chart: Plot our cash
-      ax4 = plt.subplot(414, sharex=ax1)
-      perf.cash.plot(ax=ax4)
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-      start, end = ax4.get_ylim()
-      ax4.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      plt.show()
-
-
-  if __name__ == '__main__':
-      run_algorithm(
-              capital_base=1000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-9-22', utc=True),
-              end=pd.to_datetime('2017-9-23', utc=True),
-          )
+.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
 
@@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of
 Mean Reversion Algorithm
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
-
 This algorithm is based on a simple momentum strategy. When the cryptoasset goes
 up quickly, we're going to buy; when it goes down quickly, we're going to sell. 
 Hopefully, we'll ride the waves.
@@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type:
 
    python mean_reversion_simple.py
 
-.. code-block:: python
+Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
 
-  import os
-  import tempfile
-  import time
-
-  import numpy as np
-  import pandas as pd
-  import talib
-  from logbook import Logger
-
-  from catalyst import run_algorithm
-  from catalyst.api import symbol, record, order_target_percent, get_open_orders
-  from catalyst.exchange.stats_utils import extract_transactions
-  # We give a name to the algorithm which Catalyst will use to persist its state.
-  # In this example, Catalyst will create the `.catalyst/data/live_algos`
-  # directory. If we stop and start the algorithm, Catalyst will resume its
-  # state using the files included in the folder.
-  from catalyst.utils.paths import ensure_directory
-
-  NAMESPACE = 'mean_reversion_simple'
-  log = Logger(NAMESPACE)
-
-
-  # To run an algorithm in Catalyst, you need two functions: initialize and
-  # handle_data.
-
-  def initialize(context):
-      # This initialize function sets any data or variables that you'll use in
-      # your algorithm.  For instance, you'll want to define the trading pair (or
-      # trading pairs) you want to backtest.  You'll also want to define any
-      # parameters or values you're going to use.
-
-      # In our example, we're looking at Neo in USD.
-      context.neo_eth = symbol('neo_usd')
-      context.base_price = None
-      context.current_day = None
-
-      context.RSI_OVERSOLD = 30
-      context.RSI_OVERBOUGHT = 80
-      context.CANDLE_SIZE = '15T'
-
-      context.start_time = time.time()
-
-
-  def handle_data(context, data):
-      # This handle_data function is where the real work is done.  Our data is
-      # minute-level tick data, and each minute is called a frame.  This function
-      # runs on each frame of the data.
-
-      # We flag the first period of each day.
-      # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-      # would only execute once. This method works with minute and daily
-      # frequencies.
-      today = data.current_dt.floor('1D')
-      if today != context.current_day:
-          context.traded_today = False
-          context.current_day = today
-
-      # We're computing the volume-weighted-average-price of the security
-      # defined above, in the context.neo_eth variable.  For this example, we're 
-      # using three bars on the 15 min bars.
-
-      # The frequency attribute determine the bar size. We use this convention
-      # for the frequency alias:
-      # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-      prices = data.history(
-          context.neo_eth,
-          fields='close',
-          bar_count=50,
-          frequency=context.CANDLE_SIZE
-      )
-
-      # Ta-lib calculates various technical indicator based on price and
-      # volume arrays.
-
-      # In this example, we are comp
-      rsi = talib.RSI(prices.values, timeperiod=14)
-
-      # We need a variable for the current price of the security to compare to
-      # the average. Since we are requesting two fields, data.current()
-      # returns a DataFrame with
-      current = data.current(context.neo_eth, fields=['close', 'volume'])
-      price = current['close']
-
-      # If base_price is not set, we use the current value. This is the
-      # price at the first bar which we reference to calculate price_change.
-      if context.base_price is None:
-          context.base_price = price
-
-      price_change = (price - context.base_price) / context.base_price
-      cash = context.portfolio.cash
-
-      # Now that we've collected all current data for this frame, we use
-      # the record() method to save it. This data will be available as
-      # a parameter of the analyze() function for further analysis.
-      record(
-          price=price,
-          volume=current['volume'],
-          price_change=price_change,
-          rsi=rsi[-1],
-          cash=cash
-      )
-
-      # We are trying to avoid over-trading by limiting our trades to
-      # one per day.
-      if context.traded_today:
-          return
-
-      # Since we are using limit orders, some orders may not execute immediately
-      # we wait until all orders are executed before considering more trades.
-      orders = get_open_orders(context.neo_eth)
-      if len(orders) > 0:
-          return
-
-      # Exit if we cannot trade
-      if not data.can_trade(context.neo_eth):
-          return
-
-      # Another powerful built-in feature of the Catalyst backtester is the
-      # portfolio object.  The portfolio object tracks your positions, cash,
-      # cost basis of specific holdings, and more.  In this line, we calculate
-      # how long or short our position is at this minute.   
-      pos_amount = context.portfolio.positions[context.neo_eth].amount
-
-      if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
-          log.info(
-              '{}: buying - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          # Set a style for limit orders,
-          limit_price = price * 1.005
-          order_target_percent(
-              context.neo_eth, 1, limit_price=limit_price
-          )
-          context.traded_today = True
-
-      elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
-          log.info(
-              '{}: selling - price: {}, rsi: {}'.format(
-                  data.current_dt, price, rsi[-1]
-              )
-          )
-          limit_price = price * 0.995
-          order_target_percent(
-              context.neo_eth, 0, limit_price=limit_price
-          )
-          context.traded_today = True
-
-
-  def analyze(context=None, perf=None):
-      end = time.time()
-      log.info('elapsed time: {}'.format(end - context.start_time))
-
-      import matplotlib.pyplot as plt
-      # The base currency of the algo exchange
-      base_currency = context.exchanges.values()[0].base_currency.upper()
-
-      # Plot the portfolio value over time.
-      ax1 = plt.subplot(611)
-      perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-      ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
-
-      # Plot the price increase or decrease over time.
-      ax2 = plt.subplot(612, sharex=ax1)
-      perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-
-      ax2.set_ylabel('{asset}\n({base})'.format(
-          asset=context.neo_eth.symbol, base=base_currency
-      ))
-
-      transaction_df = extract_transactions(perf)
-      if not transaction_df.empty:
-          buy_df = transaction_df[transaction_df['amount'] > 0]
-          sell_df = transaction_df[transaction_df['amount'] < 0]
-          ax2.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'price'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax2.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'price'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-
-      ax4 = plt.subplot(613, sharex=ax1)
-      perf.loc[:, 'cash'].plot(
-          ax=ax4, label='Base Currency ({})'.format(base_currency)
-      )
-      ax4.set_ylabel('Cash\n({})'.format(base_currency))
-
-      perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-      ax5 = plt.subplot(614, sharex=ax1)
-      perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-      ax5.set_ylabel('Percent\nChange')
-
-      ax6 = plt.subplot(615, sharex=ax1)
-      perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-      ax6.set_ylabel('RSI')
-      ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
-      ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
-
-      if not transaction_df.empty:
-          ax6.scatter(
-              buy_df.index.to_pydatetime(),
-              perf.loc[buy_df.index.floor('1 min'), 'rsi'],
-              marker='^',
-              s=100,
-              c='green',
-              label=''
-          )
-          ax6.scatter(
-              sell_df.index.to_pydatetime(),
-              perf.loc[sell_df.index.floor('1 min'), 'rsi'],
-              marker='v',
-              s=100,
-              c='red',
-              label=''
-          )
-      plt.legend(loc=3)
-      start, end = ax6.get_ylim()
-      ax6.yaxis.set_ticks(np.arange(0, end, end/5))
-
-      # Show the plot.
-      plt.gcf().set_size_inches(18, 8)
-      plt.show()
-      pass
-
-
-  if __name__ == '__main__':
-      # The execution mode: backtest or live
-      MODE = 'backtest'
-
-      if MODE == 'backtest':
-          folder = os.path.join(
-              tempfile.gettempdir(), 'catalyst', NAMESPACE
-          )
-          ensure_directory(folder)
-
-          timestr = time.strftime('%Y%m%d-%H%M%S')
-          out = os.path.join(folder, '{}.p'.format(timestr))
-          # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
-          run_algorithm(
-              capital_base=10000,
-              data_frequency='minute',
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bitfinex',
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              start=pd.to_datetime('2017-10-01', utc=True),
-              end=pd.to_datetime('2017-11-10', utc=True),
-              output=out
-          )
-          log.info('saved perf stats: {}'.format(out))
-
-      elif MODE == 'live':
-          run_algorithm(
-              capital_base=0.5,
-              initialize=initialize,
-              handle_data=handle_data,
-              analyze=analyze,
-              exchange_name='bittrex',
-              live=True,
-              algo_namespace=NAMESPACE,
-              base_currency='usd',
-              live_graph=False
-          )
+.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
+   :language: python
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
 
@@ -763,8 +158,6 @@ strategy.
 Simple Universe
 ~~~~~~~~~~~~~~~
 
-Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
-
 This example aims to provide an easy way for users to learn how to 
 collect data from any given exchange and select a subset of the available
 currency pairs for trading. You simply need to specify the exchange and 
@@ -791,142 +184,10 @@ of the file:
   
   catalyst ingest-exchange -x bitfinex -f minute
 
-.. code-block:: bash
-
-  python simple_universe.py
-
-Credits: This code was originally submitted by `Abner Ayala-Acevedo 
-<https://github.com/abnera>`_. Thank you!
-
-.. code-block:: python
-
-  from datetime import timedelta
-
-  import numpy as np
-  import pandas as pd
-
-  from catalyst import run_algorithm
-  from catalyst.exchange.exchange_utils import get_exchange_symbols
-  from catalyst.api import (symbols, )
-
-
-  def initialize(context):
-      context.i = -1  # minute counter
-      context.exchange = context.exchanges.values()[0].name.lower()  
-      context.base_currency = context.exchanges.values()[0].base_currency.lower()  
-
-
-  def handle_data(context, data):
-      context.i += 1
-      lookback_days = 7  # 7 days
-
-      # current date & time in each iteration formatted into a string
-      now = data.current_dt
-      date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-      lookback_date = now - timedelta(days=lookback_days) 
-      # keep only the date as a string, discard the time
-      lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  
-
-      one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
-      # update universe everyday at midnight
-      if not context.i % one_day_in_minutes:
-          context.universe = universe(context, lookback_date, date)
-
-      # get data every 30 minutes
-      minutes = 30
-      # get lookback_days of history data: that is 'lookback' number of bins
-      lookback = one_day_in_minutes / minutes * lookback_days  
-      if not context.i % minutes and context.universe:
-          # we iterate for every pair in the current universe
-          for coin in context.coins:
-              pair = str(coin.symbol)
-
-              # Get 30 minute interval OHLCV data. This is the standard data 
-              # required for candlestick or indicators/signals. Return Pandas
-              # DataFrames. 30T means 30-minute re-sampling of one minute data. 
-              # Adjust it to your desired time interval as needed.
-              opened = fill(data.history(coin, 'open', 
-                                    bar_count=lookback, frequency='30T')).values
-              high = fill(data.history(coin, 'high', 
-                                    bar_count=lookback, frequency='30T')).values
-              low = fill(data.history(coin, 'low', 
-                                    bar_count=lookback, frequency='30T')).values
-              close = fill(data.history(coin, 'price', 
-                                    bar_count=lookback, frequency='30T')).values
-              volume = fill(data.history(coin, 'volume', 
-                                    bar_count=lookback, frequency='30T')).values
-
-              # close[-1] is the last value in the set, which is the equivalent 
-              # to current price (as in the most recent value)
-              # displays the minute price for each pair every 30 minutes
-              print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
-                      now=now, 
-                      pair=pair, 
-                      o=opened[-1], 
-                      h=high[-1], 
-                      l=low[-1],
-                      c=close[-1],
-                      v=volume[-1],
-                   ))
-
-              # -------------------------------------------------------------
-              # --------------- Insert Your Strategy Here -------------------
-              # -------------------------------------------------------------
-
-
-  def analyze(context=None, results=None):
-      pass
-
-
-  # Get the universe for a given exchange and a given base_currency market
-  # Example: Poloniex BTC Market
-  def universe(context, lookback_date, current_date):
-      # get all the pairs for the given exchange
-      json_symbols = get_exchange_symbols(context.exchange)  
-      # convert into a DataFrame for easier processing
-      df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) 
-      df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
-      df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
-
-      # Filter all the pairs to get only the ones for a given base_currency
-      df = df[df['base_currency'] == context.base_currency]
-
-      # Filter all the pairs to ensure that pair existed in the current date range
-      df = df[df.start_date < lookback_date]
-      df = df[df.end_daily >= current_date]
-      context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
-
-      return df.symbol.tolist()
-
-
-  # Replace all NA, NAN or infinite values with its nearest value
-  def fill(series):
-      if isinstance(series, pd.Series):
-          return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-      elif isinstance(series, np.ndarray):
-          return pd.Series(series).replace(
-                       [np.inf, -np.inf], np.nan
-                      ).ffill().bfill().values
-      else:
-          return series
-
-
-  if __name__ == '__main__':
-      start_date = pd.to_datetime('2017-11-10', utc=True)
-      end_date = pd.to_datetime('2017-11-13', utc=True)
-
-      performance = run_algorithm(start=start_date, end=end_date,
-                                  capital_base=100.0,  # amount of base_currency
-                                  initialize=initialize,
-                                  handle_data=handle_data,
-                                  analyze=analyze,
-                                  exchange_name='bitfinex',
-                                  data_frequency='minute',
-                                  base_currency='btc',
-                                  live=False,
-                                  live_graph=False,
-                                  algo_namespace='simple_universe')
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
 
+.. literalinclude:: ../../catalyst/examples/simple_universe.py
+   :language: python
 
 
 .. _portfolio_optimization:
@@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used
 in writting the following article: 
 `Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
 
-.. code-block:: python
+Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
 
-  '''
-     You can run this code using the Python interpreter:
-
-     $ python portfolio_optimization.py
-  '''
-
-  from __future__ import division
-  import os
-  import pytz
-  import numpy as np
-  import pandas as pd
-  from scipy.optimize import minimize
-  import matplotlib.pyplot as plt
-  from datetime import datetime
-
-  from catalyst.api import record, symbol, symbols, order_target_percent
-  from catalyst.utils.run_algo import run_algorithm
-
-  np.set_printoptions(threshold='nan', suppress=True)
-
-
-  def initialize(context):
-     # Portfolio assets list
-     context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
-                              'xmr_usdt')
-     context.nassets = len(context.assets)
-     # Set the time window that will be used to compute expected return 
-     # and asset correlations
-     context.window = 180
-     # Set the number of days between each portfolio rebalancing
-     context.rebalance_period = 30                   
-     context.i = 0
-
-     
-  def handle_data(context, data):
-     # Only rebalance at the beggining of the algorithm execution and 
-     # every multiple of the rebalance period
-     if context.i == 0 or context.i%context.rebalance_period == 0:
-         n = context.window
-         prices = data.history(context.assets, fields='price', 
-                               bar_count=n+1, frequency='1d') 
-         pr = np.asmatrix(prices)
-         t_prices = prices.iloc[1:n+1]
-         t_val = t_prices.values
-         tminus_prices = prices.iloc[0:n]
-         tminus_val = tminus_prices.values
-         # Compute daily returns (r)
-         r = np.asmatrix(t_val/tminus_val-1)
-         # Compute the expected returns of each asset with the average 
-         # daily return for the selected time window
-         m = np.asmatrix(np.mean(r, axis=0))
-         # ###
-         stds = np.std(r, axis=0)
-         # Compute excess returns matrix (xr)
-         xr = r - m
-         # Matrix algebra to get variance-covariance matrix
-         cov_m = np.dot(np.transpose(xr),xr)/n
-         # Compute asset correlation matrix (informative only)
-         corr_m = cov_m/np.dot(np.transpose(stds),stds)
-         
-         # Define portfolio optimization parameters
-         n_portfolios = 50000
-         results_array = np.zeros((3+context.nassets,n_portfolios))
-         for p in xrange(n_portfolios):
-             weights = np.random.random(context.nassets)
-             weights /= np.sum(weights)
-             w = np.asmatrix(weights)
-             p_r = np.sum(np.dot(w,np.transpose(m)))*365
-             p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
-             
-             #store results in results array
-             results_array[0,p] = p_r
-             results_array[1,p] = p_std
-             #store Sharpe Ratio (return / volatility) - risk free rate element 
-             #excluded for simplicity
-             results_array[2,p] = results_array[0,p] / results_array[1,p]
-             i = 0
-             for iw in weights:
-                 results_array[3+i,p] = weights[i]
-                 i += 1
-         
-         #convert results array to Pandas DataFrame
-         results_frame = pd.DataFrame(np.transpose(results_array),
-                            columns=['r','stdev','sharpe']+context.assets)
-         #locate position of portfolio with highest Sharpe Ratio
-         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
-         #locate positon of portfolio with minimum standard deviation
-         min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
-         
-         #order optimal weights for each asset
-         for asset in context.assets:
-             if data.can_trade(asset):
-                 order_target_percent(asset, max_sharpe_port[asset])
-         
-         #create scatter plot coloured by Sharpe Ratio
-         plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
-         plt.xlabel('Volatility')
-         plt.ylabel('Returns')
-         plt.colorbar()
-         #plot red star to highlight position of portfolio with highest Sharpe Ratio
-         plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
-         #plot green star to highlight position of minimum variance portfolio
-         plt.show()
-         print(max_sharpe_port)
-         record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
-     context.i += 1
-     
-         
-  def analyze(context=None, results=None):
-     # Form DataFrame with selected data
-     data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
-     
-     # Save results in CSV file
-     filename = os.path.splitext(os.path.basename(__file__))[0]
-     data.to_csv(filename + '.csv')
-
-
-  # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
-  start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
-  end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) 
-  results = run_algorithm(initialize=initialize,
-                          handle_data=handle_data,
-                          analyze=analyze,
-                          start=start,
-                          end=end,
-                          exchange_name='poloniex',
-                          capital_base=100000, )
+.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
+   :language: python
 
 .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
    :align: center

docs/source/features.rst

@@ -44,11 +44,11 @@ For additional details on the functionality added on recent releases, see the
 Upcoming features
 ~~~~~~~~~~~~~~~~~
 
-* Additional datasets beyond pricing data (Dec. 2017)
-* API documentation (Jan. 2017)
-* Support for decentralized exchanges (Jan. 2017)
-* Support for data ingestion of community-contributed data sets (Jan. 2017)
-* Pipeline support (Jan. 2018)
+* Additional datasets beyond pricing data (Q1 2018)
+* API documentation (Q1 2018)
+* Support for decentralized exchanges (Q1 2018)
+* Support for data ingestion of community-contributed data sets (Q1 2018)
+* Pipeline support (Q1 2018)
 * Web UI (Q2 2018)
 
 

docs/source/index.rst

@@ -1,6 +1,8 @@
 .. include:: ../../README.rst
+
 |
 |
+
 Table of Contents
 -----------------