merged from develop

Example: Simple Universe

README.rst

@@ -1 +1,72 @@
-All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_ can be found in the `catalyst-docs wiki <https://github.com/enigmampc/catalyst-docs/wiki>`_.
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+    :target: https://enigmampc.github.io/catalyst
+    :align: center
+    :alt: Enigma | Catalyst
+
+|version tag|
+|version status|
+|discord|
+|twitter|
+
+|
+
+Catalyst is an algorithmic trading library for crypto-assets written in Python.
+It allows trading strategies to be easily expressed and backtested against 
+historical data (with daily and minute resolution), providing analytics and 
+insights regarding a particular strategy's performance. Catalyst also supports
+live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
+and Poloniex) with more being added over time. Catalyst empowers users to share 
+and curate data and build profitable, data-driven investment strategies. Please 
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
+refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
+further technical details.
+
+Catalyst builds on top of the well-established 
+`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
+minimize structural changes to the general API to maximize compatibility with 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
+for questions around Catalyst, algorithmic trading and technical support.
+
+Overview
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.
+-  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
+   performance across trading algorithms.
+
+Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
+
+
+
+
+.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
+   :target: https://discordapp.com/invite/SJK32GY
+
+.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
+   :target: https://twitter.com/enigmampc
+
+

catalyst/__init__.py

@@ -29,11 +29,14 @@ from ._version import get_versions
 from . algorithm import TradingAlgorithm
 from . import api
 
+from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
+
+__version__ = get_versions()['version']
+del get_versions
 
 # PERF: Fire a warning if calendars were instantiated during catalyst import.
 # Having calendars doesn't break anything per-se, but it makes catalyst imports
 # noticeably slower, which becomes particularly noticeable in the Zipline CLI.
-from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
 if global_calendar_dispatcher._calendars:
     import warnings
     warnings.warn(
@@ -44,10 +47,6 @@ if global_calendar_dispatcher._calendars:
 del global_calendar_dispatcher
 
 
-__version__ = get_versions()['version']
-del get_versions
-
-
 def load_ipython_extension(ipython):
     from .__main__ import catalyst_magic
     ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -69,7 +68,6 @@ if os.name == 'nt':
     _()
     del _
 
-
 __all__ = [
     'TradingAlgorithm',
     'api',
@@ -80,7 +78,3 @@ __all__ = [
     'run_algorithm',
     'utils',
 ]
-
-from ._version import get_versions
-__version__ = get_versions()['version']
-del get_versions

catalyst/__main__.py

@@ -9,7 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -29,16 +29,17 @@ except NameError:
 @click.option(
     '--strict-extensions/--non-strict-extensions',
     is_flag=True,
-    help='If --strict-extensions is passed then catalyst will not run if it'
-         ' cannot load all of the specified extensions. If this is not passed or'
-         ' --non-strict-extensions is passed then the failure will be logged but'
-         ' execution will continue.',
+    help='If --strict-extensions is passed then catalyst will not run '
+         'if it cannot load all of the specified extensions. If this is '
+         'not passed or --non-strict-extensions is passed then the '
+         'failure will be logged but execution will continue.',
 )
 @click.option(
     '--default-extension/--no-default-extension',
     is_flag=True,
     default=True,
-    help="Don't load the default catalyst extension.py file in $ZIPLINE_HOME.",
+    help="Don't load the default catalyst extension.py file "
+         "in $CATALYST_HOME.",
 )
 @click.version_option()
 def main(extension, strict_extensions, default_extension):
@@ -123,9 +124,9 @@ def ipython_only(option):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '--data-frequency',
@@ -137,7 +138,6 @@ def ipython_only(option):
 @click.option(
     '--capital-base',
     type=float,
-    default=10e6,
     show_default=True,
     help='The starting capital for the simulation.',
 )
@@ -175,8 +175,8 @@ def ipython_only(option):
     default='-',
     metavar='FILENAME',
     show_default=True,
-    help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
+    help="The location to write the perf data. If this is '-' the perf"
+         " will be written to stdout.",
 )
 @click.option(
     '--print-algo/--no-print-algo',
@@ -193,8 +193,7 @@ def ipython_only(option):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange.',
 )
 @click.option(
     '-n',
@@ -239,16 +238,27 @@ def run(ctx,
         # does not pass either of these and then passes the first only
         # to be told they need to pass the second argument also
         ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'",
+            "must specify dates with '-s' / '--start' and '-e' / '--end'"
+            " in backtest mode",
         )
     if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'")
+        ctx.fail("must specify a start date with '-s' / '--start'"
+                 " in backtest mode")
     if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'")
+        ctx.fail("must specify an end date with '-e' / '--end'"
+                 " in backtest mode")
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
+    if base_currency is None:
+        ctx.fail("must specify a base currency with '-c' in backtest mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    click.echo('Running in backtesting mode.')
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -272,7 +282,9 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=False
+        live_graph=False,
+        simulate_orders=True,
+        stats_output=None,
     )
 
     if output == '-':
@@ -300,11 +312,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                     # these options are set when running in line magic mode
-                     # set a non None algo text to use the ipython user_ns
-                     '--algotext', '',
-                     '--local-namespace',
-                 ] if cell is None else []) + line.split(),
+                # these options are set when running in line magic mode
+                # set a non None algo text to use the ipython user_ns
+                '--algotext', '',
+                '--local-namespace',
+            ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,
@@ -324,6 +336,12 @@ def catalyst_magic(line, cell=None):
     type=click.File('r'),
     help='The file that contains the algorithm to run.',
 )
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
 @click.option(
     '-t',
     '--algotext',
@@ -334,9 +352,9 @@ def catalyst_magic(line, cell=None):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '-o',
@@ -362,8 +380,7 @@ def catalyst_magic(line, cell=None):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange.',
 )
 @click.option(
     '-n',
@@ -382,9 +399,17 @@ def catalyst_magic(line, cell=None):
     default=False,
     help='Display live graph.',
 )
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
 @click.pass_context
 def live(ctx,
          algofile,
+         capital_base,
          algotext,
          define,
          output,
@@ -393,7 +418,8 @@ def live(ctx,
          exchange_name,
          algo_namespace,
          base_currency,
-         live_graph):
+         live_graph,
+         simulate_orders):
     """Trade live with the given algorithm.
     """
     if (algotext is not None) == (algofile is not None):
@@ -404,11 +430,22 @@ def live(ctx,
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
+
     if algo_namespace is None:
         ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
     if base_currency is None:
         ctx.fail("must specify a base currency '-c' in live execution mode")
 
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.')
+
+    else:
+        click.echo('Running in live trading mode.')
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -418,7 +455,7 @@ def live(ctx,
         algotext=algotext,
         defines=define,
         data_frequency=None,
-        capital_base=None,
+        capital_base=capital_base,
         data=None,
         bundle=None,
         bundle_timestamp=None,
@@ -432,7 +469,9 @@ def live(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        simulate_orders=simulate_orders,
+        stats_output=None,
     )
 
     if output == '-':
@@ -447,9 +486,7 @@ def live(ctx,
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
+    help='The name of the exchange bundle to ingest.',
 )
 @click.option(
     '-f',
@@ -485,18 +522,40 @@ def live(ctx,
     help='A list of symbols to exclude from the ingestion '
          '(optional comma separated list)',
 )
+@click.option(
+    '--csv',
+    default=None,
+    help='The path of a CSV file containing the data. If specified, start, '
+         'end, include-symbols and exclude-symbols will be ignored. Instead,'
+         'all data in the file will be ingested.',
+)
 @click.option(
     '--show-progress/--no-show-progress',
     default=True,
     help='Print progress information to the terminal.'
 )
-def ingest_exchange(exchange_name, data_frequency, start, end,
-                    include_symbols, exclude_symbols, show_progress):
+@click.option(
+    '--verbose/--no-verbose`',
+    default=False,
+    help='Show a progress indicator for every currency pair.'
+)
+@click.option(
+    '--validate/--no-validate`',
+    default=False,
+    help='Report potential anomalies found in data bundles.'
+)
+@click.pass_context
+def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
+                    include_symbols, exclude_symbols, csv, show_progress,
+                    verbose, validate):
     """
     Ingest data for the given exchange.
     """
-    exchange = get_exchange(exchange_name)
-    exchange_bundle = ExchangeBundle(exchange)
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    exchange_bundle = ExchangeBundle(exchange_name)
 
     click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
@@ -505,10 +564,59 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
         exclude_symbols=exclude_symbols,
         start=start,
         end=end,
-        show_progress=show_progress
+        show_progress=show_progress,
+        show_breakdown=verbose,
+        show_report=validate,
+        csv=csv
     )
 
 
+@main.command(name='clean-algo')
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='The label of the algorithm to for which to clean the state.'
+)
+@click.pass_context
+def clean_algo(ctx, algo_namespace):
+    click.echo(
+        'Cleaning algo state: {}'.format(algo_namespace)
+    )
+    delete_algo_folder(algo_namespace)
+    click.echo('Done')
+
+
+@main.command(name='clean-exchange')
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the exchange bundle to ingest.',
+)
+@click.option(
+    '-f',
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute'}),
+    default=None,
+    help='The bundle data frequency to remove. If not specified, it will '
+         'remove both daily and minute bundles.',
+)
+@click.pass_context
+def clean_exchange(ctx, exchange_name, data_frequency):
+    """Clean up bundles from 'ingest-exchange'.
+    """
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    exchange_bundle = ExchangeBundle(exchange_name)
+
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
+    exchange_bundle.clean(
+        data_frequency=data_frequency,
+    )
+    click.echo('Done')
+
+
 @main.command()
 @click.option(
     '-b',
@@ -521,9 +629,7 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
+    help='The name of the exchange bundle to ingest.',
 )
 @click.option(
     '-c',
@@ -598,7 +704,7 @@ def ingest(ctx, bundle, exchange_name, compile_locally, assets_version,
          ' This may not be passed with -e / --before or -a / --after',
 )
 def clean(bundle, before, after, keep_last):
-    """Clean up data downloaded with the ingest command.
+    """Clean up bundles from 'ingest'.
     """
     bundles_module.clean(
         bundle,

catalyst/algorithm.py

@@ -124,7 +124,6 @@ from catalyst.utils.events import (
 from catalyst.utils.factory import create_simulation_parameters
 from catalyst.utils.math_utils import (
     tolerant_equals,
-    round_if_near_integer,
     round_nearest
 )
 from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
@@ -1485,7 +1484,6 @@ class TradingAlgorithm(object):
         """
         Converts the number of shares to the smallest tradable lot size for
         the asset being ordered.
-        
         """
         return round_nearest(amount, asset.min_trade_size)
 
@@ -1523,6 +1521,7 @@ class TradingAlgorithm(object):
                              self.updated_portfolio(),
                              self.get_datetime(),
                              self.trading_client.current_data)
+
     @staticmethod
     def __convert_order_params_for_blotter(limit_price, stop_price, style):
         """

catalyst/assets/_assets.pyx

@@ -17,6 +17,8 @@
 """
 Cythonized Asset object.
 """
+import hashlib
+
 cimport cython
 from cpython.number cimport PyNumber_Index
 from cpython.object cimport (
@@ -36,6 +38,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
+from catalyst.exchange.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
@@ -393,11 +396,18 @@ cdef class Future(Asset):
 
 cdef class TradingPair(Asset):
     cdef readonly float leverage
-    cdef readonly object market_currency
+    cdef readonly object quote_currency
     cdef readonly object base_currency
     cdef readonly object end_daily
     cdef readonly object end_minute
     cdef readonly object exchange_symbol
+    cdef readonly float maker
+    cdef readonly float taker
+    cdef readonly int trading_state
+    cdef readonly object data_source
+    cdef readonly float max_trade_size
+    cdef readonly float lot
+    cdef readonly int decimals
 
     _kwargnames = frozenset({
         'sid',
@@ -410,12 +420,19 @@ cdef class TradingPair(Asset):
         'exchange',
         'exchange_full',
         'leverage',
-        'market_currency',
+        'quote_currency',
         'base_currency',
         'end_daily',
         'end_minute',
         'exchange_symbol',
-        'min_trade_size'
+        'min_trade_size',
+        'max_trade_size',
+        'lot',
+        'maker',
+        'taker',
+        'trading_state',
+        'data_source',
+        'decimals'
     })
     def __init__(self,
                  object symbol,
@@ -431,10 +448,17 @@ cdef class TradingPair(Asset):
                  object first_traded=None,
                  object auto_close_date=None,
                  object exchange_full=None,
-                 object min_trade_size=None):
+                 float min_trade_size=0.0001,
+                 float max_trade_size=1000000,
+                 float maker=0.0015,
+                 float taker=0.0025,
+                 float lot=0,
+                 int decimals = 8,
+                 int trading_state=0,
+                 object data_source='catalyst'):
         """
         Replicates the Asset constructor with some built-in conventions
-        and a new 'leverage' attribute.
+        and adds properties for leverage and fees.
 
         Symbol
         ------
@@ -466,8 +490,6 @@ cdef class TradingPair(Asset):
         highest volume and market cap generally benefit from high leverage.
         New currencies from ICO generally cannot be leveraged.
 
-        The leverage value is either None or and integer.
-
         Leverage allows you to open a larger position with a smaller amount
         of funds. For example, if you open a $5,000 position in BTC/USD
         with 5:1 leverage, only one-fifth of this amount, or $1000, will be
@@ -477,6 +499,11 @@ cdef class TradingPair(Asset):
         the position. If you open with 1:1 leverage, $5,000 of your balance
         will be tied to the position.
 
+        Fees
+        ----
+        Exchanges generally charge a taker (taking from the order book) or
+        maker (adding to the order book) fee.
+
         :param symbol:
         :param exchange:
         :param start_date:
@@ -491,17 +518,23 @@ cdef class TradingPair(Asset):
         :param auto_close_date:
         :param exchange_full:
         :param min_trade_size:
+        :param max_trade_size:
+        :param maker:
+        :param taker:
+        :param data_source
+        :param decimals
+        :param lot
         """
 
         symbol = symbol.lower()
         try:
-            self.market_currency, self.base_currency = symbol.split('_')
+            self.base_currency, self.quote_currency = symbol.split('_')
         except Exception as e:
             raise InvalidSymbolError(symbol=symbol, error=e)
 
         if sid == 0 or sid is None:
             try:
-                sid = abs(hash(symbol)) % (10 ** 4)
+                sid = get_sid(symbol)
             except Exception as e:
                 raise SidHashError(symbol=symbol)
 
@@ -509,11 +542,14 @@ cdef class TradingPair(Asset):
             asset_name = ' / '.join(symbol.split('_')).upper()
 
         if start_date is None:
-            start_date = pd.Timestamp.utcnow()
+            start_date = pd.to_datetime('2009-1-1', utc=True)
 
         if end_date is None:
             end_date = pd.Timestamp.utcnow() + timedelta(days=365)
 
+        if lot == 0 and min_trade_size > 0:
+            lot = min_trade_size
+
         super().__init__(
             sid,
             exchange,
@@ -524,19 +560,26 @@ cdef class TradingPair(Asset):
             first_traded=first_traded,
             auto_close_date=auto_close_date,
             exchange_full=exchange_full,
-            min_trade_size=min_trade_size
+            min_trade_size=min_trade_size,
         )
 
+        self.maker = maker
+        self.taker = taker
         self.leverage = leverage
         self.end_daily = end_daily
         self.end_minute = end_minute
         self.exchange_symbol = exchange_symbol
+        self.trading_state = trading_state
+        self.data_source = data_source
+        self.max_trade_size = max_trade_size
+        self.lot = lot
+        self.decimals = decimals
 
     def __repr__(self):
         return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
                'Introduced On: {start_date}, ' \
-               'Market Currency: {market_currency}, ' \
                'Base Currency: {base_currency}, ' \
+               'Quote Currency: {quote_currency}, ' \
                'Exchange Leverage: {leverage}, ' \
                'Minimum Trade Size: {min_trade_size} ' \
                'Last daily ingestion: {end_daily} ' \
@@ -545,7 +588,7 @@ cdef class TradingPair(Asset):
             sid=self.sid,
             exchange=self.exchange,
             start_date=self.start_date,
-            market_currency=self.market_currency,
+            quote_currency=self.quote_currency,
             base_currency=self.base_currency,
             leverage=self.leverage,
             min_trade_size=self.min_trade_size,
@@ -553,6 +596,32 @@ cdef class TradingPair(Asset):
             end_minute=self.end_minute
         )
 
+    cpdef to_dict(self):
+        """
+        Convert to a python dict.
+        """
+        #TODO: missing fields
+        super_dict = super(TradingPair, self).to_dict()
+        super_dict['end_daily'] = self.end_daily
+        super_dict['end_minute'] = self.end_minute
+        super_dict['leverage'] = self.leverage
+        super_dict['min_trade_size'] = self.min_trade_size
+        return super_dict
+
+    def is_exchange_open(self, dt_minute):
+        """
+        Parameters
+        ----------
+        dt_minute: pd.Timestamp (UTC, tz-aware)
+            The minute to check.
+
+        Returns
+        -------
+        boolean: whether the asset's exchange is open at the given minute.
+        """
+        #TODO: make more dymanic to catch holds
+        return True
+
     cpdef __reduce__(self):
         """
         Function used by pickle to determine how to serialize/deserialize this
@@ -560,6 +629,7 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
+        #TODO: make sure that all fields set there
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
@@ -570,7 +640,12 @@ cdef class TradingPair(Asset):
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
-                                 self.min_trade_size))
+                                 self.min_trade_size,
+                                 self.max_trade_size,
+                                 self.lot,
+                                 self.decimals,
+                                 self.taker,
+                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -1,5 +1,18 @@
 # -*- coding: utf-8 -*-
 
+import os
 import logbook
 
-LOG_LEVEL = logbook.INFO
+''' You can override the LOG level from your environment.
+    For example, if you want to see the DEBUG messages, run:
+    $ export CATALYST_LOG_LEVEL=10
+'''
+LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
+
+SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
+              '{exchange}/symbols.json'
+
+DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+DATE_FORMAT = '%Y-%m-%d'
+
+AUTO_INGEST = False

catalyst/curate/poloniex.py

@@ -1,39 +1,47 @@
-import json, time, csv
+import os
+import time
+import shutil
+import json
+import csv
 from datetime import datetime
+
 import pandas as pd
-import os, time, shutil, requests, logbook
+import requests
+import logbook
+
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 
-DT_START        = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END          = int(time.time())
-CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
-CSV_OUT_FOLDER  = '/Volumes/enigma/data/poloniex/'
-CONN_RETRIES    = 2
+DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
+DT_END = pd.to_datetime('today').value // 10 ** 9
+CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
+CONN_RETRIES = 2
 
 logbook.StderrHandler().push_application()
 log = logbook.Logger(__name__)
 
+
 class PoloniexCurator(object):
     '''
     OHLCV data feed generator for crypto data. Based on Poloniex market data
     '''
 
-    _api_path       = 'https://poloniex.com/public?'
-    currency_pairs  = []
+    _api_path = 'https://poloniex.com/public?'
+    currency_pairs = []
 
     def __init__(self):
         if not os.path.exists(CSV_OUT_FOLDER):
             try:
                 os.makedirs(CSV_OUT_FOLDER)
             except Exception as e:
-                log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
+                log.error('Failed to create data folder: {}'.format(
+                            CSV_OUT_FOLDER))
                 log.exception(e)
 
-    '''
-        Retrieves and returns all currency pairs from the exchange 
-    '''
     def get_currency_pairs(self):
+        '''
+        Retrieves and returns all currency pairs from the exchange
+        '''
         url = self._api_path + 'command=returnTicker'
 
         try:
@@ -44,102 +52,153 @@ class PoloniexCurator(object):
             return None
 
         data = response.json()
-        self.currency_pairs  = []
+        self.currency_pairs = []
         for ticker in data:
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
 
-        log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
+        log.debug('Currency pairs retrieved successfully: {}'.format(
+                    len(self.currency_pairs)
+                    ))
 
-
-    '''
-        Helper function that reads tradeID and date fields from CSV readline
-    '''
     def _retrieve_tradeID_date(self, row):
+        '''
+        Helper function that reads tradeID and date fields from CSV readline
+        '''
         tId = int(row.split(',')[0])
-        d   = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9
+        d = pd.to_datetime(row.split(',')[1],
+                           infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
-    '''
-        Retrieves TradeHistory from exchange for a given currencyPair between start and end dates.
-        If no start date is provided, uses a system-wide one (beginning of time for cryptotrading)
-        If no end date is provided, 'now' is used
+    def retrieve_trade_history(self, currencyPair, start=DT_START,
+                               end=DT_END, temp=None):
+        '''
+        Retrieves TradeHistory from exchange for a given currencyPair
+        between start and end dates. If no start date is provided, uses
+        a system-wide one (beginning of time for cryptotrading).
+        If no end date is provided, 'now' is used.
+
         Stores results in CSV file on disk.
-        This function is called recursively to work around the limitations imposed by the provider API.
-    '''
-    def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None):
+
+        This function is called recursively to work around the
+        limitations imposed by the provider API.
+        '''
         csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
 
         '''
-            Check what data we already have on disk, reading first and last lines from file.
-            Data is stored on file from NEWEST to OLDEST.
+        Check what data we already have on disk, reading first and last
+        lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
-            with open(csv_fn, 'ab+') as f: 
+            with open(csv_fn, 'ab+') as f:
                 f.seek(0, os.SEEK_END)
-                if(f.tell() > 2):                           # First check file is not zero size
-                    f.seek(0)                               # Go to the beginning to read first line
-                    last_tradeID, end_file    = self._retrieve_tradeID_date(f.readline())
-                    f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                    while f.read(1) != b"\n":               # Until EOL is found...
-                        f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
-                    first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
+                if(f.tell() > 2):                   # Check file size is not 0
+                    f.seek(0)                       # Go to start to read
+                    last_tradeID, end_file = self._retrieve_tradeID_date(
+                                                    f.readline())
+                    f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
+                    while f.read(1) != b"\n":       # Until EOL is found...
+                        # ...jump back the read byte plus one more.
+                        f.seek(-2, os.SEEK_CUR)
+                    first_tradeID, start_file = self._retrieve_tradeID_date(
+                                                    f.readline())
 
-                    if( first_tradeID == 1 and end_file + 3600 > DT_END ):
+                    if(end_file + 3600 * 6 > DT_END
+                        and (first_tradeID == 1
+                             or (currencyPair == 'BTC_HUC'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_RIC'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_XCP'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_NAV'
+                                 and first_tradeID == 4569)
+                             or (currencyPair == 'BTC_POT'
+                                 and first_tradeID == 23511))):
                         return
 
         except Exception as e:
-            log.error('Error opening file: %s' % csv_fn)
+            log.error('Error opening file: {}'.format(csv_fn))
             log.exception(e)
 
         '''
-        Poloniex API limits querying TradeHistory to intervals smaller than 1 month,
-        so we make sure that start date is never more than 1 month apart from end date
+        Poloniex API limits querying TradeHistory to intervals smaller
+        than 1 month, so we make sure that start date is never more than
+        1 month apart from end date
         '''
-        if( end - start > 2419200 ):       # 60 s/min * 60 min/hr * 24 hr/day * 28 days     
+        if(end - start > 2419200):   # 60s/min * 60min/hr * 24hr/day * 28days
             newstart = end - 2419200
         else:
             newstart = start
 
-        log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t '
-                    + time.ctime(newstart) + ' - '+ time.ctime(end))
+        log.debug('{}: Retrieving from {} to {}\t {} - {}'.format(
+                    currencyPair, str(newstart), str(end),
+                    time.ctime(newstart), time.ctime(end)))
 
-        url = self._api_path + 'command=returnTradeHistory&currencyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end)
+        url = '{path}command=returnTradeHistory&currencyPair={pair}' \
+              '&start={start}&end={end}'.format(
+                    path=self._api_path,
+                    pair=currencyPair,
+                    start=str(newstart),
+                    end=str(end)
+                )
 
-        try:
-            response = requests.get(url)
-        except Exception as e:
-            log.error('Failed to retrieve trade history data for %s' % currencyPair)
-            log.exception(e)
+        attempts = 0
+        success = 0
+        while attempts < CONN_RETRIES:
+            try:
+                response = requests.get(url)
+            except Exception as e:
+                log.error('Failed to retrieve trade history data'
+                          'for {}'.format(currencyPair))
+                log.exception(e)
+                attempts += 1
+            else:
+                try:
+                    if(isinstance(response.json(), dict)
+                       and response.json()['error']):
+                        log.error('Failed to to retrieve trade history data '
+                                  'for {}: {}'.format(
+                                    currencyPair,
+                                    response.json()['error']
+                                    ))
+                        attempts += 1
+                except Exception as e:
+                    log.exception(e)
+                    attempts += 1
+                else:
+                    success = 1
+                    break
+
+        if not success:
             return None
-        else:
-            if isinstance(response.json(), dict) and response.json()['error']:
-                log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error']))
-                exit(1)
 
         '''
-            If we get to transactionId == 1, and we already have that on disk, 
-            we got to the end of TradeHistory for this coin.
+        If we get to transactionId == 1, and we already have that on
+        disk, we got to the end of TradeHistory for this coin.
         '''
-        if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID): 
+        if('first_tradeID' in locals()
+                and response.json()[-1]['tradeID'] == first_tradeID):
             return
 
         '''
             There are primarily two scenarios:
-                a) There is newer data available that we need to add at the beginning
-                   of the file. We'll retrieve all what we need until we get to what 
-                   we already have, writing it to a temporary file; and we will write
-                   that at the beginning of our existing file.
-                b) We are going back in time, appending at the end of our existing
-                   TradeHistory until the first transaction for this currencyPair
+                a) There is newer data available that we need to add at
+                   the beginning of the file. We'll retrieve all what we
+                   need until we get to what we already have, writing it
+                   to a temporary file; and we will write that at the
+                   beginning of our existing file.
+                b) We are going back in time, appending at the end of
+                   our existing TradeHistory until the first transaction
+                   for this currencyPair
         '''
-        try: 
-            if( 'end_file' in locals() and end_file + 3600 < end):
+        try:
+            if('end_file' in locals() and end_file + 3600 < end):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
                 for item in response.json():
-                    if( item['tradeID'] <= last_tradeID ):
+                    if(item['tradeID'] <= last_tradeID):
                         continue
                     tempcsv.writerow([
                         item['tradeID'],
@@ -148,24 +207,28 @@ class PoloniexCurator(object):
                         item['rate'],
                         item['amount'],
                         item['total'],
-                        item['globalTradeID']        
+                        item['globalTradeID'],
                     ])
-                if( response.json()[-1]['tradeID'] > last_tradeID ):
-                    end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
-                    self.retrieve_trade_history(currencyPair, start, end, temp=temp) 
+                if(response.json()[-1]['tradeID'] > last_tradeID):
+                    end = pd.to_datetime(response.json()[-1]['date'],
+                                         infer_datetime_format=True
+                                         ).value // 10**9
+                    self.retrieve_trade_history(currencyPair, start,
+                                                end, temp=temp)
                 else:
-                    with open(csv_fn,'rb+') as f:
-                        shutil.copyfileobj(f,temp)
+                    with open(csv_fn, 'rb+') as f:
+                        shutil.copyfileobj(f, temp)
                         f.seek(0)
                         temp.seek(0)
-                        shutil.copyfileobj(temp,f)
+                        shutil.copyfileobj(temp, f)
                     temp.close()
                     end = start_file
             else:
                 with open(csv_fn, 'ab') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in response.json():
-                        if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ):
+                        if('first_tradeID' in locals()
+                                and item['tradeID'] >= first_tradeID):
                             continue
                         csvwriter.writerow([
                             item['tradeID'],
@@ -176,52 +239,67 @@ class PoloniexCurator(object):
                             item['total'],
                             item['globalTradeID']
                         ])
-                end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
+                end = pd.to_datetime(response.json()[-1]['date'],
+                                     infer_datetime_format=True).value//10**9
 
         except Exception as e:
-            log.error('Error opening %s' % csv_fn)
+            log.error('Error opening {}'.format(csv_fn))
             log.exception(e)
 
         '''
-            If we got here, we aren't done yet. Call recursively with 'end' times
-            that go sequentially back in time.
+            If we got here, we aren't done yet. Call recursively with
+            'end' times that go sequentially back in time.
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
-
-    '''
+    def generate_ohlcv(self, df):
+        '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
         by resampling with 1-minute period
-    '''
-    def generate_ohlcv(self, df):
-        df.set_index('date', inplace=True)                      # Index by date
-        vol = df['total'].to_frame('volume')                    # Will deal with vol separately, as ohlc() messes it up
-        df.drop('total', axis=1, inplace=True)                  # Drop volume data from dataframe
-        ohlc = df.resample('T').ohlc()                          # Resample OHLC in 1min bins
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])         # Raname columns by dropping 'rate'
-        closes = ohlc['close'].fillna(method='pad')             # Pad forward missing 'close'
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))           # Fill N/A with last close
-        vol = vol.resample('T').sum().fillna(0)                 # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)                   # Concatenate OHLC + Volume
+        '''
+        df.set_index('date', inplace=True)             # Index by date
+        vol = df['total'].to_frame('volume')           # set Vol aside
+        df.drop('total', axis=1, inplace=True)         # Drop volume data
+        ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
+        ohlc.cols = ohlc.cols.map(lambda t: t[1])      # Raname cols
+        closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
+        ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
+        vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin
+        ohlcv = pd.concat([ohlc, vol], axis=1)         # Concat OHLC + Vol
         return ohlcv
 
-
-    '''
+    def write_ohlcv_file(self, currencyPair):
+        '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
-    '''
-    def write_ohlcv_file(self, currencyPair):        
+        '''
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
-        csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if( os.path.isfile(csv_1min) ):
-            log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
+        csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
+        if(os.path.getmtime(csv_1min) > time.time() - 7200):
+            log.debug(currencyPair+': 1min data file already up to date. '
+                      'Delete the file if you want to rebuild it.')
         else:
-            df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
-                    dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
-            df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
-            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
+            df = pd.read_csv(csv_trades,
+                             names=['tradeID',
+                                    'date',
+                                    'type',
+                                    'rate',
+                                    'amount',
+                                    'total',
+                                    'globalTradeID'],
+                             dtype={'tradeID': int,
+                                    'date': str,
+                                    'type': str,
+                                    'rate': float,
+                                    'amount': float,
+                                    'total': float,
+                                    'globalTradeID': int}
+                             )
+            df.drop(['tradeID', 'type', 'amount', 'globalTradeID'],
+                    axis=1, inplace=True)
+            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
             ohlcv = self.generate_ohlcv(df)
-            try: 
-                with open(csv_1min, 'ab') as csvfile:
+            try:
+                with open(csv_1min, 'w') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in ohlcv.itertuples():
                         if item.Index == 0:
@@ -235,25 +313,30 @@ class PoloniexCurator(object):
                             item.volume,
                         ])
             except Exception as e:
-                log.error('Error opening %s' % csv_fn)
+                log.error('Error opening {}'.format(csv_1min))
                 log.exception(e)
-            log.debug(currencyPair+': Generated 1min OHLCV data.')
+            log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
 
-
-    '''
-    Returns a data frame for a given currencyPair from data on disk
-    '''
     def onemin_to_dataframe(self, currencyPair, start, end):
-        csv_fn     = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df         = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
-        df['date'] = pd.to_datetime(df['date'],unit='s')
+        '''
+        Returns a data frame for a given currencyPair from data on disk
+        '''
+        csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
+        df = pd.read_csv(csv_fn, names=['date',
+                                        'open',
+                                        'high',
+                                        'low',
+                                        'close',
+                                        'volume'])
+        df['date'] = pd.to_datetime(df['date'], unit='s')
         df.set_index('date', inplace=True)
-        return df[start : end]
+        return df[start:end]
 
-    '''
-    Generates a symbols.json file with corresponding start_date for each currencyPair
-    '''
     def generate_symbols_json(self, filename=None):
+        '''
+        Generates a symbols.json file with corresponding start_date
+        for each currencyPair
+        '''
         symbol_map = {}
 
         if(filename is None):
@@ -262,33 +345,37 @@ class PoloniexCurator(object):
         with open(filename, 'w') as symbols:
             for currencyPair in self.currency_pairs:
                 start = None
-                csv_fn     = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
-                with open(csv_fn, 'r') as f: 
+                csv_fn = '{}crypto_trades-{}.csv'.format(
+                                                    CSV_OUT_FOLDER,
+                                                    currencyPair)
+                with open(csv_fn, 'r') as f:
                     f.seek(0, os.SEEK_END)
-                    if(f.tell() > 2):                           # First check file is not zero size
-                        f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                        while f.read(1) != b"\n":               # Until EOL is found...
-                            f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
-                        start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True)
+                    if(f.tell() > 2):               # Check file size is not 0
+                        f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
+                        while f.read(1) != b"\n":   # Until EOL is found...
+                            # ...jump back the read byte plus one more.
+                            f.seek(-2, os.SEEK_CUR)
+                        start = pd.to_datetime(f.readline().split(',')[1],
+                                               infer_datetime_format=True)
 
                 if(start is None):
                     start = time.gmtime()
                 base, market = currencyPair.lower().split('_')
-                symbol = '{market}_{base}'.format( market=market, base=base )
+                symbol = '{market}_{base}'.format(market=market, base=base)
                 symbol_map[currencyPair] = dict(
-                    symbol = symbol,
-                    start_date = start.strftime("%Y-%m-%d")
+                    symbol=symbol,
+                    start_date=start.strftime("%Y-%m-%d")
                 )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':'))    
+            json.dump(symbol_map, symbols, sort_keys=True, indent=2,
+                      separators=(',', ':'))
 
 
 if __name__ == '__main__':
     pc = PoloniexCurator()
     pc.get_currency_pairs()
-    #pc.generate_symbols_json()
-    
+    # pc.generate_symbols_json()
+
     for currencyPair in pc.currency_pairs:
         pc.retrieve_trade_history(currencyPair)
+        log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
-
-    

catalyst/data/bundles/__init__.py

@@ -1,6 +1,5 @@
 # These imports are necessary to force module-scope register calls to happen.
 from . import quandl  # noqa
-from . import poloniex
 from .core import (
     UnknownBundle,
     bundles,

catalyst/data/bundles/base.py

@@ -13,10 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from itertools import count
 import tarfile
-from time import time, sleep
+from time import sleep
 
 from abc import abstractmethod, abstractproperty
 import logbook
@@ -37,6 +36,7 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 
+
 class BaseBundle(object):
     def __init__(self, asset_filter=[]):
         self._asset_filter = asset_filter
@@ -104,11 +104,11 @@ class BaseBundle(object):
 
     def post_process_symbol_metadata(self, metadata, data):
         return metadata
-    
+
     @abstractmethod
     def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date):
         raise NotImplementedError()
-    
+
     def ingest(self,
                environ,
                asset_db_writer,
@@ -128,7 +128,7 @@ class BaseBundle(object):
             retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
 
             if is_compile:
-                # User has instructed local compilation and ingestion of bundle.
+                # User has instructed local compilation & ingestion of bundle.
                 # Fetch raw metadata for all symbols.
                 raw_metadata = self._fetch_metadata_frame(
                     api_key,
@@ -157,9 +157,9 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # Post-process metadata using cached symbol frames, and write to
-                # disk.  This metadata must be written before any attempt to write
-                # minute data.
+                # Post-process metadata using cached symbol frames, and write
+                # to disk.  This metadata must be written before any attempt
+                # to write minute data.
                 metadata = self._post_process_metadata(
                     raw_metadata,
                     cache,
@@ -184,10 +184,11 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # For legacy purposes, this call is required to ensure the database
-                # contains an appropriately initialized file structure.  We don't
-                # forsee a usecase for adjustments at this time, but may later
-                # choose to expose this functionality in the future.
+                # For legacy purposes, this call is required to ensure the
+                # database contains an appropriately initialized file
+                # structure.  We don't forsee a usecase for adjustments at
+                # this time, but may later choose to expose this functionality
+                # in the future.
                 adjustment_writer.write(
                     splits=(
                         pd.concat(self.splits, ignore_index=True)
@@ -232,12 +233,12 @@ class BaseBundle(object):
             tar.extractall(output_dir)
 
     def _fetch_metadata_frame(self,
-                             api_key,
-                             cache,
-                             retries=DEFAULT_RETRIES,
-                             environ=None,
-                             show_progress=False):
-        
+                              api_key,
+                              cache,
+                              retries=DEFAULT_RETRIES,
+                              environ=None,
+                              show_progress=False):
+
         # Setup raw metadata iterator to fetch pages if necessary.
         raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
 
@@ -251,7 +252,7 @@ class BaseBundle(object):
             show_percent=False,
         ) as blocks:
             metadata = pd.concat(blocks, ignore_index=True)
-        
+
         return metadata
 
     def _fetch_metadata_iter(self, api_key, cache, retries, environ):
@@ -269,21 +270,20 @@ class BaseBundle(object):
                             page_number,
                         )
                         break
-                    except ValueError as e:
+                    except ValueError:
                         raw = pd.DataFrame([])
                         break
-                    except Exception as e:
+                    except Exception:
                         log.exception(
                             'Failed to load metadata from {}. '
                             'Retrying.'.format(self.name)
-                        ) 
+                        )
                 else:
                     raise ValueError(
                         'Failed to download metadata page {} after {} '
                         'attempts.'.format(page_number, retries)
                     )
 
-
             if raw.empty:
                 # Empty DataFrame signals completion.
                 break
@@ -305,7 +305,7 @@ class BaseBundle(object):
             columns=self.md_column_names,
             index=metadata.index,
         )
-        
+
         # Iterate over the available symbols, loading the asset's raw symbol
         # data from the cache.  The final metadata is computed and recorded in
         # the appropriate row depending on the asset's id.
@@ -318,22 +318,22 @@ class BaseBundle(object):
             show_percent=False,
         ) as symbols_map:
             for asset_id, symbol in symbols_map:
-                # Attempt to load data from disk, the cache should have an entry
-                # for each symbol at this point of the execution. If one does
-                # not exist, we should fail.
+                # Attempt to load data from disk, the cache should have an
+                # entry for each symbol at this point of the execution. If one
+                # does not exist, we should fail.
                 key = '{sym}.daily.frame'.format(sym=symbol)
                 try:
                     raw_data = cache[key]
                 except KeyError:
                     raise ValueError(
-                      'Unable to find cached data for symbol: {0}'.format(symbol)
-                    )
+                      'Unable to find cached data for symbol:'
+                      ' {0}'.format(symbol))
 
                 # Perform and require post-processing of metadata.
                 final_symbol_metadata = self.post_process_symbol_metadata(
                     asset_id,
                     metadata.iloc[asset_id],
-                    raw_data,        
+                    raw_data,
                 )
 
                 # Record symbol's final metadata.
@@ -363,8 +363,8 @@ class BaseBundle(object):
             # returns the cached data unaltered.  The `should_sleep` flag
             # indicates that an API call was attempted, and that we should be
             # ensure aren't exceeding our rate limit before proceeding to the
-            # next symbol. If the raw_data is updated, it is cached before being
-            # returned.
+            # next symbol. If the raw_data is updated, it is cached before
+            # being returned.
             raw_data, should_sleep = self._maybe_update_symbol_frame(
                 start_time,
                 api_key,
@@ -414,7 +414,7 @@ class BaseBundle(object):
         last = start_session
         if raw_data is not None and len(raw_data) > 0:
             last = raw_data.index[-1].tz_localize('UTC')
-        
+
         should_sleep = False
 
         # Determine time at which cached data will be considered stale.
@@ -455,7 +455,7 @@ class BaseBundle(object):
                             retries=DEFAULT_RETRIES):
 
         # Data for symbol is old enough to attempt an update or is not
-        # present in the cache.  Fetch raw data for a single symbol 
+        # present in the cache.  Fetch raw data for a single symbol
         # with requested intervals and frequency. Retry as necessary.
         for _ in range(retries):
             try:
@@ -468,7 +468,6 @@ class BaseBundle(object):
                     data_frequency,
                 )
                 raw_data.index = pd.to_datetime(raw_data.index, utc=True)
-                #raw_data.index = raw_data.index.tz_localize('UTC')
 
                 # Filter incoming data to fit start and end sessions.
                 raw_data = raw_data[
@@ -482,7 +481,7 @@ class BaseBundle(object):
 
                 return raw_data
 
-            except Exception as e:
+            except Exception:
                 log.exception(
                     'Exception raised fetching {name} data. Retrying.'
                     .format(name=self.name)

catalyst/data/bundles/base_pricing.py

@@ -16,6 +16,7 @@
 from catalyst.data.bundles.base import BaseBundle
 from catalyst.utils.memoize import lazyval
 
+
 class BasePricingBundle(BaseBundle):
     @lazyval
     def md_dtypes(self):
@@ -38,6 +39,7 @@ class BasePricingBundle(BaseBundle):
             ('volume', 'float64'),
         ]
 
+
 class BaseCryptoPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):
@@ -55,6 +57,7 @@ class BaseCryptoPricingBundle(BasePricingBundle):
     def dividends(self):
         return []
 
+
 class BaseEquityPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):

catalyst/data/bundles/core.py

@@ -37,6 +37,7 @@ from catalyst.utils.cli import maybe_show_progress
 
 ONE_MEGABYTE = 1024 * 1024
 
+
 def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
     return pth.data_path(
         asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -135,6 +136,7 @@ def ingestions_for_bundle(bundle, environ=None):
         reverse=True,
     )
 
+
 def download_with_progress(url, chunk_size, **progress_kwargs):
     """
     Download streaming data from a URL, printing progress information to the
@@ -705,4 +707,5 @@ def _make_bundle_core():
     )
 
 
-bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
+bundles, register_bundle, register, unregister, ingest, load, clean = \
+    _make_bundle_core()

catalyst/data/bundles/poloniex.py

@@ -14,19 +14,17 @@
 # limitations under the License.
 
 import sys
-
-from datetime import datetime
+from six.moves.urllib.parse import urlencode
 
 import pandas as pd
 
-from six.moves.urllib.parse import urlencode
-
 from catalyst.data.bundles.core import register_bundle
 from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
 from catalyst.utils.memoize import lazyval
 
 from catalyst.curate.poloniex import PoloniexCurator
 
+
 class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def name(self):
@@ -46,7 +44,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def tar_url(self):
         return (
-            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
+            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
+            'poloniex/poloniex-bundle.tar.gz'
         )
 
     @lazyval
@@ -67,12 +66,11 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         raw = raw.sort_index().reset_index()
         raw.rename(
-            columns={'index':'symbol'},
+            columns={'index': 'symbol'},
             inplace=True,
         )
 
         raw = raw[raw['isFrozen'] == 0]
-
         return raw
 
     def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
@@ -98,7 +96,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                                frequency):
 
         # TODO: replace this with direct exchange call
-        # The end date and frequency should be used to calculate the number of bars
+        # The end date and frequency should be used to
+        # calculate the number of bars
         if(frequency == 'minute'):
             pc = PoloniexCurator()
             raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
@@ -116,8 +115,9 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             )
             raw.set_index('date', inplace=True)
 
-        # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
-        # on disk, which we compensate here to get the right pricing amounts
+        # BcolzDailyBarReader introduces a 1/1000 factor in the way
+        # pricing is stored on disk, which we compensate here to get
+        # the right pricing amounts
         # ref: data/us_equity_pricing.py
         scale = 1
         raw.loc[:, 'open'] /= scale
@@ -139,7 +139,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         return self._format_polo_query(query_params)
 
-
     def _format_data_url(self,
                          api_key,
                          symbol,
@@ -162,27 +161,26 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             ('end', end_date.value / 10**9),
             ('period', period),
         ]
-            
+
         return self._format_polo_query(query_params)
-    
+
     def _format_polo_query(self, query_params):
         # TODO: got against the exchange object
         return 'https://poloniex.com/public?{query}'.format(
             query=urlencode(query_params),
         )
 
-''' 
-As a second parameter, you can pass an array of currency pairs 
-that will be processed as an asset_filter to only process that 
+
+'''
+As a second parameter, you can pass an array of currency pairs
+that will be processed as an asset_filter to only process that
 subset of assets in the bundle, such as:
 register_bundle(PoloniexBundle, ['USDT_BTC',])
 
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-
 if 'ingest' in sys.argv and '-c' in sys.argv:
     register_bundle(PoloniexBundle)
 else:
     register_bundle(PoloniexBundle, create_writers=False)
-

catalyst/data/bundles/quandl.py

@@ -16,7 +16,6 @@
 from datetime import datetime
 
 import pandas as pd
-
 from six.moves.urllib.parse import urlencode
 
 from catalyst.data.bundles.core import register_bundle
@@ -26,25 +25,16 @@ from catalyst.utils.memoize import lazyval
 """
 Module for building a complete daily dataset from Quandl's WIKI dataset.
 """
-from itertools import count
-import tarfile
-from time import time, sleep
-from datetime import datetime
-
 from logbook import Logger
-import pandas as pd
-from six.moves.urllib.parse import urlencode
-
-from catalyst.utils.calendars import register_calendar_alias
-from catalyst.utils.cli import maybe_show_progress
-
-from . import core as bundles
 
 from catalyst.constants import LOG_LEVEL
+from catalyst.utils.calendars import register_calendar_alias
+
 
 log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
+
 class QuandlBundle(BaseEquityPricingBundle):
     @lazyval
     def name(self):
@@ -109,8 +99,8 @@ class QuandlBundle(BaseEquityPricingBundle):
         # Filter out invalid symbols
         raw = raw[~raw.symbol.isin(self._excluded_symbols)]
 
-        # cut out all the other stuff in the name column
-        # we need to escape the paren because it is actually splitting on a regex
+        # cut out all the other stuff in the name column. We need to
+        # escape the paren because it is actually splitting on a regex
         raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
 
         return raw
@@ -175,7 +165,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['sid'] = asset_id
         self.splits.append(df)
 
-
     def _update_dividends(self, asset_id, raw_data):
         divs = raw_data.ex_dividend
         df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -186,7 +175,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
         self.dividends.append(df)
 
-
     def _format_metadata_url(self, api_key, page_number):
         """Build the query RL for the quandl WIKI metadata.
         """
@@ -200,10 +188,10 @@ class QuandlBundle(BaseEquityPricingBundle):
             query_params = [('api_key', api_key)] + query_params
 
         return (
-            'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
+            'https://www.quandl.com/api/v3/datasets.csv?'
+            + urlencode(query_params)
         )
 
-
     def _format_wiki_url(self,
                          api_key,
                          symbol,
@@ -229,5 +217,6 @@ class QuandlBundle(BaseEquityPricingBundle):
             )
         )
 
+
 register_calendar_alias('QUANDL', 'NYSE')
 register_bundle(QuandlBundle)

catalyst/data/data_portal.py

@@ -656,11 +656,11 @@ class DataPortal(object):
         return spot_value
 
     def _get_minutely_spot_value(self,
-                                asset,
-                                column,
-                                dt,
-                                data_frequency,
-                                ffill=False):
+                                 asset,
+                                 column,
+                                 dt,
+                                 data_frequency,
+                                 ffill=False):
 
         reader = self._get_pricing_reader(data_frequency)
 
@@ -706,7 +706,7 @@ class DataPortal(object):
             asset,
             column,
             dt,
-            ffill, 
+            ffill,
             'minute',
         )
 

catalyst/data/dispatch_bar_reader.py

@@ -133,11 +133,13 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
 
         return results
 
+
 class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):
         return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
 
+
 class AssetDispatchSessionBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):

catalyst/data/loader.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import datetime
 import os
 from collections import OrderedDict
 
@@ -129,11 +128,13 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # before this date.
     '''
     if(bundle_data):
-        # If we are using the bundle to retrieve the cryptobenchmark, find the last
-        # date for which there is trading data in the bundle
-        asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
+        # If we are using the bundle to retrieve the cryptobenchmark, find
+        # the last date for which there is trading data in the bundle
+        asset = bundle_data.asset_finder.lookup_symbol(
+                    symbol=bm_symbol,as_of_date=None)
         ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
-        last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
+        last_date = pd.to_datetime(
+                    bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
@@ -142,27 +143,30 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.poloniex.poloniex import Poloniex
-        exchange = Poloniex('', '', '')
+        from catalyst.exchange.factory import get_exchange
+        exchange = get_exchange(
+            exchange_name='poloniex', base_currency='usdt'
+        )
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 
     # exchange.get_history_window() already ensures that we have the right data
     # for the right dates
-    br = exchange.get_history_window(
+    br = exchange.get_history_window_with_bundle(
         assets=[benchmark_asset],
         end_dt=last_date,
         bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
-        data_frequency='daily')
+        data_frequency='daily',
+        force_auto_ingest=True)
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
     br.loc[start_dt] = 0
     br = br.sort_index()
 
-    # Override first_date for treasury data since we have it for many more years
-    # and is independent of crypto data
+    # Override first_date for treasury data since we have it for many more
+    # years and is independent of crypto data
     first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
     tc = ensure_treasury_data(
         bm_symbol,
@@ -298,14 +302,14 @@ def ensure_crypto_benchmark_data(symbol,
 
     if (bundle == 'poloniex'):
         '''
-        If we're using the Poloniex bundle, we'll get the benchmark from the bundle
-        instead of downloading it from Poloniex every time we need it.
-        Poloniex has a captcha for API queries originating from outside the US that 
-        prevents users abroad from getting Catalyst to work
+        If we're using the Poloniex bundle, we'll get the benchmark from the
+        bundle instead of downloading it from Poloniex every time we need it.
+        Poloniex has a captcha for API queries originating from outside the US
+        that prevents users abroad from getting Catalyst to work
         '''
         logger.info(
-            (
-                'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
+            ('Retrieving benchmark data from bundle for {symbol!r}'
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
@@ -327,11 +331,12 @@ def ensure_crypto_benchmark_data(symbol,
                         last_date)]
 
     else:
-        # This is how it used to be: downloading the benchmark everytime. 
-        # Leaving this code here to be repurposed in the future for other bundles.
+        # This is how it used to be: downloading the benchmark everytime.
+        # Leaving this code here to be repurposed in the future for
+        # other bundles.
         logger.info(
-            (
-                'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
+            ('Downloading benchmark data for {symbol!r}'
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         raise DeprecationWarning('poloniex bundle deprecated')
@@ -428,67 +433,6 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
     return data
 
 
-def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
-                          environ=None):
-    """
-    Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
-
-    Parameters
-    ----------
-    symbol : str
-        The symbol for the benchmark to load.
-    first_date : pd.Timestamp
-        First required date for the cache.
-    last_date : pd.Timestamp
-        Last required date for the cache.
-    now : pd.Timestamp
-        The current time.  This is used to prevent repeated attempts to
-        re-download data that isn't available due to scheduling quirks or other
-        failures.
-    trading_day : pd.CustomBusinessDay
-        A trading day delta.  Used to find the day before first_date so we can
-        get the close of the day prior to first_date.
-
-    We attempt to download data unless we already have data stored at the data
-    cache for `symbol` whose first entry is before or on `first_date` and whose
-    last entry is on or after `last_date`.
-
-    If we perform a download and the cache criteria are not satisfied, we wait
-    at least one hour before attempting a redownload.  This is determined by
-    comparing the current time to the result of os.path.getmtime on the cache
-    path.
-    """
-    filename = get_benchmark_filename(symbol)
-    data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
-                             environ)
-    if data is not None:
-        return data
-
-    # If no cached data was found or it was missing any dates then download the
-    # necessary data.
-    logger.info(
-        ('Downloading benchmark data for {symbol!r} '
-         'from {first_date} to {last_date}'),
-        symbol=symbol,
-        first_date=first_date - trading_day,
-        last_date=last_date
-    )
-
-    try:
-        data = get_benchmark_returns(
-            symbol,
-            first_date - trading_day,
-            last_date,
-        )
-        data.to_csv(get_data_filepath(filename, environ))
-    except (OSError, IOError, HTTPError):
-        logger.exception('Failed to cache the new benchmark returns')
-        raise
-    if not has_data_for_dates(data, first_date, last_date):
-        logger.warn("Still don't have expected data after redownload!")
-    return data
-
-
 def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
     """
     Ensure we have treasury data from treasury module associated with

catalyst/data/minute_bars.py

@@ -341,12 +341,10 @@ class BcolzMinuteBarMetadata(object):
             'end_session': str(self.end_session.date()),
             # Write these values for backwards compatibility
             'first_trading_day': str(self.start_session.date()),
-            'market_opens': (
-                market_opens.values.astype('datetime64[m]').
-                    astype(np.int64).tolist()),
-            'market_closes': (
-                market_closes.values.astype('datetime64[m]').
-                    astype(np.int64).tolist()),
+            'market_opens': (market_opens.values.astype('datetime64[m]').
+                             astype(np.int64).tolist()),
+            'market_closes': (market_closes.values.astype('datetime64[m]').
+                              astype(np.int64).tolist()),
         }
         with open(self.metadata_path(rootdir), 'w+') as fp:
             json.dump(metadata, fp)
@@ -1256,8 +1254,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 values = carray[start_idx:end_idx + 1]
                 if indices_to_exclude is not None:
                     for excl_start, excl_stop in indices_to_exclude[::-1]:
-                        excl_slice = np.s_[
-                                     excl_start - start_idx:excl_stop - start_idx + 1]
+                        excl_slice = np.s_[excl_start - start_idx:excl_stop
+                                           - start_idx + 1]
                         values = np.delete(values, excl_slice)
 
                 where = values != 0
@@ -1320,9 +1318,8 @@ class H5MinuteBarUpdateWriter(object):
 
     def __init__(self, path, complevel=None, complib=None):
         self._complevel = complevel if complevel \
-                                       is not None else self._COMPLEVEL
-        self._complib = complib if complib \
-                                   is not None else self._COMPLIB
+                            is not None else self._COMPLEVEL
+        self._complib = complib if complib is not None else self._COMPLIB
         self._path = path
 
     def write(self, frames):

catalyst/data/us_equity_pricing.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division     # Python2 req to have division of ints yield float
+from __future__ import division  # Python2 req for division of ints yield float
 
 from errno import ENOENT
 from functools import partial
@@ -120,7 +120,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
-PRICE_ADJUSTMENT_FACTOR = 1000000000    # Provides 9 decimals resolution. Also affects _equities.pyx L220
+# Provides 9 decimals resolution. Also affects _equities.pyx L220
+PRICE_ADJUSTMENT_FACTOR = 1000000000
 
 
 def check_uint32_safe(value, colname):
@@ -130,6 +131,7 @@ def check_uint32_safe(value, colname):
             "for uint32" % (value, colname)
         )
 
+
 def check_uint64_safe(value, colname):
     if value >= UINT64_MAX:
         raise ValueError(
@@ -322,8 +324,8 @@ class BcolzDailyBarWriter(object):
         # Maps column name -> output carray.
         columns = {
             k: carray(array([], dtype=uint64))
-               if k in OHLCV
-               else carray(array([], dtype=uint32))
+            if k in OHLCV
+            else carray(array([], dtype=uint32))
             for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
         }
 
@@ -439,11 +441,13 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed = (raw_data[list(OHLC)]
+                     * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
-        processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed['volume'] = (raw_data.volume
+                               * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         return ctable.fromdataframe(processed)
 
 
@@ -496,7 +500,7 @@ class BcolzDailyBarReader(SessionBarReader):
 
     The data in these columns is interpreted as follows:
 
-    - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted 
+    - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
       as 10^9 * as-traded dollar value.
     - Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
     - Id is the asset id of the row.

catalyst/examples/README.rst

@@ -0,0 +1,3 @@
+An overview of most of the trading strategies in this folder can be found in the 
+`Examples Algorithms <https://enigmampc.github.io/catalyst/example-algos.html>`_
+section of our documentation website.

catalyst/examples/arbitrage_with_interface.py

@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
     else:
         raise ValueError('invalid order action')
 
-    base_currency = enter_exchange.base_currency
-    base_currency_amount = enter_exchange.portfolio.cash
+    quote_currency = enter_exchange.quote_currency
+    quote_currency_amount = enter_exchange.portfolio.cash
 
     exit_balances = exit_exchange.get_balances()
     exit_currency = context.trading_pairs[
-        context.selling_exchange].market_currency
+        context.selling_exchange].quote_currency
 
     if exit_currency in exit_balances:
-        market_currency_amount = exit_balances[exit_currency]
+        quote_currency_amount = exit_balances[exit_currency]
     else:
         log.warn(
             'the selling exchange {exchange_name} does not hold '
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
         )
         return
 
-    if base_currency_amount < (amount * entry_price):
-        adj_amount = base_currency_amount / entry_price
+    if quote_currency_amount < (amount * entry_price):
+        adj_amount = quote_currency_amount / entry_price
         log.warn(
-            'not enough {base_currency} ({base_currency_amount}) to buy '
+            'not enough {quote_currency} ({quote_currency_amount}) to buy '
             '{amount}, adjusting the amount to {adj_amount}'.format(
-                base_currency=base_currency,
-                base_currency_amount=base_currency_amount,
+                quote_currency=quote_currency,
+                quote_currency_amount=quote_currency_amount,
                 amount=amount,
                 adj_amount=adj_amount
             )
         )
         amount = adj_amount
 
-    elif market_currency_amount < amount:
+    elif quote_currency_amount < amount:
         log.warn(
             'not enough {currency} ({currency_amount}) to sell '
             '{amount}, aborting'.format(
                 currency=exit_currency,
-                currency_amount=market_currency_amount,
+                currency_amount=quote_currency_amount,
                 amount=amount
             )
         )
@@ -263,13 +263,20 @@ def analyze(context, stats):
     pass
 
 
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='poloniex,bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc',
-    live_graph=False
-)
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'live'
+    if MODE == 'live':
+        run_algorithm(
+            capital_base=0.1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex,bitfinex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=True,
+            stats_output=None,
+        )

catalyst/examples/buy_and_hodl.py

@@ -14,30 +14,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (order_target_value, symbol, record,
+                          cancel_order, get_open_orders, )
 
-from catalyst.api import (
-    order_target_value,
-    symbol,
-    record,
-    cancel_order,
-    get_open_orders,
-)
 
 def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
+    context.ASSET_NAME = 'btc_usd'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
     context.is_buying = True
     context.asset = symbol(context.ASSET_NAME)
 
     context.i = 0
 
+
 def handle_data(context, data):
     context.i += 1
 
@@ -49,55 +44,56 @@ def handle_data(context, data):
     orders = get_open_orders(context.asset) or []
     for order in orders:
         cancel_order(order)
-    
+
     # Stop buying after passing the reserve threshold
     cash = context.portfolio.cash
     if cash <= reserve_value:
         context.is_buying = False
 
     # Retrieve current asset price from pricing data
-    price = data[context.asset].price
+    price = data.current(context.asset, 'price')
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
+        print('buying')
         # Place order to make position in asset equal to target_hodl_value
         order_target_value(
             context.asset,
             target_hodl_value,
-            limit_price=price*1.1,
-            stop_price=price*0.9,
+            limit_price=price * 1.1,
         )
 
     record(
         price=price,
-        volume=data[context.asset].volume,
+        volume=data.current(context.asset, 'volume'),
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,
     )
 
+
 def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
 
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value (USD)')
+    ax1.set_ylabel('Portfolio\nValue\n(USD)')
 
     ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
+    ax2.set_ylabel('{asset}\n(USD)'.format(asset=context.ASSET_NAME))
+    results[['price']].plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     buys = trans.ix[
         [t[0]['amount'] > 0 for t in trans.transactions]
     ]
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
+    ax2.scatter(
+        buys.index.to_pydatetime(),
+        results.price[buys.index],
+        marker='^',
+        s=100,
+        c='g',
+        label=''
     )
 
     ax3 = plt.subplot(613, sharex=ax1)
@@ -124,14 +120,29 @@ def analyze(context=None, results=None):
         'algorithm',
         'benchmark',
     ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
+    ax5.set_ylabel('Percent\nChange')
 
     ax6 = plt.subplot(616, sharex=ax1)
     results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mCoins/5min)')
+    ax6.set_ylabel('Volume')
 
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
-    plt.show()
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bitfinex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_btc.py

@@ -1,10 +0,0 @@
-from catalyst.api import order, record, symbol
-
-
-def initialize(context):
-    context.asset = symbol('btc_usd')
-
-
-def handle_data(context, data):
-    order(context.asset, 1)
-    record(btc=data.current(context.asset, 'price'))

catalyst/examples/buy_btc_simple.py

@@ -1,8 +1,49 @@
+'''
+    This is a very simple example referenced in the beginner's tutorial:
+    https://enigmampc.github.io/catalyst/beginner-tutorial.html
+
+    Run this example, by executing the following from your terminal:
+      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
+      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+    If you want to run this code using another exchange, make sure that
+    the asset is available on that exchange. For example, if you were to run
+    it for exchange Poloniex, you would need to edit the following line:
+
+        context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
+
+    and specify exchange poloniex as follows:
+    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
+    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+    To see which assets are available on each exchange, visit:
+    https://www.enigma.co/catalyst/status
+'''
+from catalyst import run_algorithm
 from catalyst.api import order, record, symbol
+import pandas as pd
+
 
 def initialize(context):
     context.asset = symbol('btc_usd')
 
+
 def handle_data(context, data):
     order(context.asset, 1)
-    record(btc = data.current(context.asset, 'price'))
+    record(btc=data.current(context.asset, 'price'))
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='bitfinex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_low_sell_high.py

@@ -1,17 +1,19 @@
 '''
-This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
-Install it first by running: 
+This algorithm requires an additional library (ta-lib) beyond those
+required by catalyst. Install it first by running:
 $ pip install TA-Lib
 
-If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and needs to be installed.
-See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install 
-the required dependencies.
+If you get build errors like:
+    "fatal error: ta-lib/ta_libc.h: No such file or directory"
+it typically means that it can't find the underlying TA-Lib library and it
+needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
+instructions on how to install the required dependencies.
 '''
 
 import talib
 from logbook import Logger
 
+from catalyst import run_algorithm
 from catalyst.api import (
     order,
     order_target_percent,
@@ -20,6 +22,7 @@ from catalyst.api import (
     get_open_orders,
 )
 from catalyst.exchange.stats_utils import get_pretty_stats
+import pandas as pd
 
 algo_namespace = 'buy_low_sell_high_xrp'
 log = Logger(algo_namespace)
@@ -27,7 +30,7 @@ log = Logger(algo_namespace)
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USD'
+    context.ASSET_NAME = 'XRP_USDT'
     context.asset = symbol(context.ASSET_NAME)
 
     context.TARGET_POSITIONS = 5000
@@ -100,8 +103,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
+        elif (position.amount > 0
+              and price > cost_basis * (1 + context.PROFIT_TARGET)):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -156,3 +159,18 @@ def handle_data(context, data):
 def analyze(context, stats):
     log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
     pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_low_sell_high_live.py

@@ -41,7 +41,7 @@ def _handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='1d'
+        frequency='1D'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
@@ -88,8 +88,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
+        elif (position.amount > 0
+              and price > cost_basis * (1 + context.PROFIT_TARGET)):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -146,23 +146,15 @@ def analyze(context, stats):
     pass
 
 
-run_algorithm(
-    capital_base=100000,
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='poloniex',
-    start=pd.to_datetime('2017-5-01', utc=True),
-    end=pd.to_datetime('2017-10-16', utc=True),
-    base_currency='usdt',
-    data_frequency='daily'
-)
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='poloniex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc'
-# )
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.001,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=algo_namespace,
+        base_currency='btc',
+        simulate_orders=True,
+    )

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -1,173 +0,0 @@
-import talib
-from logbook import Logger
-import pandas as pd
-
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_low_sell_high_neo'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.asset = symbol('neo_btc', 'bitfinex')
-
-    context.TARGET_POSITIONS = 50000
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'close')
-    log.info('got price {price}'.format(price=price))
-
-    if price is None:
-        log.warn('no pricing data')
-        return
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=1,
-        frequency='1m'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.1
-    else:
-        buy_increment = None
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(price=price)
-
-    orders = get_open_orders(context.asset)
-    if len(orders) > 0:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        limit_price = price * (1 + context.SLIPPAGE_ALLOWED)
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=limit_price
-        )
-        pass
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-
-    pass
-
-
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='bitfinex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc',
-#     live_graph=False
-# )
-
-# Backtest
-run_algorithm(
-    capital_base=250,
-    data_frequency='minute',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    algo_namespace=algo_namespace,
-    base_currency='btc'
-)

catalyst/examples/dual_moving_average.py

@@ -0,0 +1,162 @@
+import numpy as np
+import pandas as pd
+from logbook import Logger
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (record, symbol, order_target_percent,
+                          get_open_orders)
+from catalyst.exchange.stats_utils import extract_transactions
+
+NAMESPACE = 'dual_moving_average'
+log = Logger(NAMESPACE)
+
+
+def initialize(context):
+    context.i = 0
+    context.asset = symbol('ltc_usd')
+    context.base_price = None
+
+
+def handle_data(context, data):
+    # define the windows for the moving averages
+    short_window = 50
+    long_window = 200
+
+    # Skip as many bars as long_window to properly compute the average
+    context.i += 1
+    if context.i < long_window:
+        return
+
+    # Compute moving averages calling data.history() for each
+    # moving average with the appropriate parameters. We choose to use
+    # minute bars for this simulation -> freq="1m"
+    # Returns a pandas dataframe.
+    short_mavg = data.history(context.asset,
+                              'price',
+                              bar_count=short_window,
+                              frequency="1m",
+                              ).mean()
+    long_mavg = data.history(context.asset,
+                             'price',
+                             bar_count=long_window,
+                             frequency="1m",
+                             ).mean()
+
+    # Let's keep the price of our asset in a more handy variable
+    price = data.current(context.asset, 'price')
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+    price_change = (price - context.base_price) / context.base_price
+
+    # Save values for later inspection
+    record(price=price,
+           cash=context.portfolio.cash,
+           price_change=price_change,
+           short_mavg=short_mavg,
+           long_mavg=long_mavg)
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.asset):
+        return
+
+    # We check what's our position on our portfolio and trade accordingly
+    pos_amount = context.portfolio.positions[context.asset].amount
+
+    # Trading logic
+    if short_mavg > long_mavg and pos_amount == 0:
+        # we buy 100% of our portfolio for this asset
+        order_target_percent(context.asset, 1)
+    elif short_mavg < long_mavg and pos_amount > 0:
+        # we sell all our positions for this asset
+        order_target_percent(context.asset, 0)
+
+
+def analyze(context, perf):
+
+    # Get the base_currency that was passed as a parameter to the simulation
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # First chart: Plot portfolio value using base_currency
+    ax1 = plt.subplot(411)
+    perf.loc[:, ['portfolio_value']].plot(ax=ax1)
+    ax1.legend_.remove()
+    ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
+    start, end = ax1.get_ylim()
+    ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Second chart: Plot asset price, moving averages and buys/sells
+    ax2 = plt.subplot(412, sharex=ax1)
+    perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
+        ax=ax2,
+        label='Price')
+    ax2.legend_.remove()
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.asset.symbol,
+        base=base_currency
+        ))
+    start, end = ax2.get_ylim()
+    ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    # Third chart: Compare percentage change between our portfolio
+    # and the price of the asset
+    ax3 = plt.subplot(413, sharex=ax1)
+    perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
+    ax3.legend_.remove()
+    ax3.set_ylabel('Percent Change')
+    start, end = ax3.get_ylim()
+    ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Fourth chart: Plot our cash
+    ax4 = plt.subplot(414, sharex=ax1)
+    perf.cash.plot(ax=ax4)
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+    start, end = ax4.get_ylim()
+    ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='usd',
+            start=pd.to_datetime('2017-9-22', utc=True),
+            end=pd.to_datetime('2017-9-23', utc=True),
+        )

catalyst/examples/dual_vwap.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2017 Enigma MPC, Inc.
-# Copyright 2014 Quantopian, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from catalyst.api import (
-    order_target_percent,
-    record,
-    symbol,
-    get_open_orders,
-    set_max_leverage,
-    schedule_function,
-    date_rules,
-    attach_pipeline,
-    pipeline_output,
-)
-
-from catalyst.pipeline import Pipeline
-from catalyst.pipeline.data import CryptoPricing
-from catalyst.pipeline.factors.crypto import VWAP
-
-
-def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
-    context.TARGET_INVESTMENT_RATIO = 0.8
-    context.SHORT_WINDOW = 30
-    context.LONG_WINDOW = 100
-
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
-    context.i = 0
-    context.asset = symbol(context.ASSET_NAME)
-
-    set_max_leverage(1.0)
-
-    attach_pipeline(make_pipeline(context), 'vwap_pipeline')
-
-    schedule_function(
-        rebalance,
-        time_rules=times_rules.every_minute(),
-    )
-
-
-def before_trading_start(context, data):
-    context.pipeline_data = pipeline_output('vwap_pipeline')
-
-def make_pipeline(context):
-    return Pipeline(
-        columns={
-            'price': CryptoPricing.open.latest,
-            'volume': CryptoPricing.volume.latest,
-            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
-            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
-        }
-    )
-
-def rebalance(context, data):
-    context.i += 1
-
-    # skip first LONG_WINDOW bars to fill windows
-    if context.i < context.LONG_WINDOW:
-        return
-
-    # get pipeline data for asset of interest
-    pipeline_data = context.pipeline_data
-    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
-
-    # retrieve long and short moving averages from pipeline
-    short_mavg = pipeline_data.short_mavg
-    long_mavg = pipeline_data.long_mavg
-    price = pipeline_data.price
-    volume = pipeline_data.volume
-
-    # check that order has not already been placed
-    open_orders = get_open_orders()
-    if context.asset not in open_orders:
-        # check that the asset of interest can currently be traded
-        if data.can_trade(context.asset):
-            # adjust portfolio based on comparison of long and short vwap
-            if short_mavg > long_mavg:
-                order_target_percent(
-                    context.asset,
-                    context.TARGET_INVESTMENT_RATIO,
-                )
-            elif short_mavg < long_mavg:
-                order_target_percent(
-                    context.asset,
-                    0.0,
-                )
-
-    record(
-        price=price,
-        cash=context.portfolio.cash,
-        leverage=context.account.leverage,
-        short_mavg=short_mavg,
-        long_mavg=long_mavg,
-        volume=volume,
-    )
-    
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(611)
-    results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio value (USD)')
-
-    ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
-
-    trans = results.ix[[t != [] for t in results.transactions]]
-    amounts = [t[0]['amount'] for t in trans.transactions]
-
-    buys = trans.ix[
-        [t[0]['amount'] > 0 for t in trans.transactions]
-    ]
-    sells = trans.ix[
-        [t[0]['amount'] < 0 for t in trans.transactions]
-    ]
-
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        context.TICK_SIZE * results.price[sells.index],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(613, sharex=ax1)
-    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-    ax3.set_ylabel('Leverage (USD)')
-
-    ax4 = plt.subplot(614, sharex=ax1)
-    results[['cash']].plot(ax=ax4)
-    ax4.set_ylabel('Cash (USD)')
-
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]] = results[[
-        'treasury_period_return',
-        'algorithm_period_return',
-        'benchmark_period_return',
-    ]]
-
-    ax5 = plt.subplot(615, sharex=ax1)
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(616, sharex=ax1)
-    results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mBTC/day)')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()

catalyst/examples/mean_reversion_simple.py

@@ -0,0 +1,289 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.stats_utils import extract_transactions
+
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('neo_eth')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERBOUGHT = 80
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    # context.set_commission(maker=0.1, taker=0.2)
+    context.set_slippage(spread=0.0001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'backtest'
+
+    if MODE == 'backtest':
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))
+
+    elif MODE == 'live':
+        run_algorithm(
+            capital_base=0.05,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='binance',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True,
+            stats_output=None
+        )

catalyst/examples/portfolio_optimization.py

@@ -0,0 +1,149 @@
+'''Use this code to execute a portfolio optimization model. This code
+   will select the portfolio with the maximum Sharpe Ratio. The parameters
+   are set to use 180 days of historical data and rebalance every 30 days.
+
+   This is the code used in the following article:
+   https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
+
+   You can run this code using the Python interpreter:
+
+   $ python portfolio_optimization.py
+'''
+
+from __future__ import division
+import os
+import pytz
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+from catalyst.api import record, symbols, order_target_percent
+from catalyst.utils.run_algo import run_algorithm
+
+np.set_printoptions(threshold='nan', suppress=True)
+
+
+def initialize(context):
+    # Portfolio assets list
+    context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
+                             'xmr_usdt')
+    context.nassets = len(context.assets)
+    # Set the time window that will be used to compute expected return
+    # and asset correlations
+    context.window = 180
+    # Set the number of days between each portfolio rebalancing
+    context.rebalance_period = 30
+    context.i = 0
+
+
+def handle_data(context, data):
+    # Only rebalance at the beggining of the algorithm execution and
+    # every multiple of the rebalance period
+    if context.i == 0 or context.i % context.rebalance_period == 0:
+        n = context.window
+        prices = data.history(context.assets, fields='price',
+                              bar_count=n + 1, frequency='1d')
+        pr = np.asmatrix(prices)
+        t_prices = prices.iloc[1:n + 1]
+        t_val = t_prices.values
+        tminus_prices = prices.iloc[0:n]
+        tminus_val = tminus_prices.values
+        # Compute daily returns (r)
+        r = np.asmatrix(t_val / tminus_val - 1)
+        # Compute the expected returns of each asset with the average
+        # daily return for the selected time window
+        m = np.asmatrix(np.mean(r, axis=0))
+        # ###
+        stds = np.std(r, axis=0)
+        # Compute excess returns matrix (xr)
+        xr = r - m
+        # Matrix algebra to get variance-covariance matrix
+        cov_m = np.dot(np.transpose(xr), xr) / n
+        # Compute asset correlation matrix (informative only)
+        corr_m = cov_m / np.dot(np.transpose(stds), stds)
+
+        # Define portfolio optimization parameters
+        n_portfolios = 50000
+        results_array = np.zeros((3 + context.nassets, n_portfolios))
+        for p in xrange(n_portfolios):
+            weights = np.random.random(context.nassets)
+            weights /= np.sum(weights)
+            w = np.asmatrix(weights)
+            p_r = np.sum(np.dot(w, np.transpose(m))) * 365
+            p_std = np.sqrt(np.dot(np.dot(w, cov_m),
+                                   np.transpose(w))) * np.sqrt(365)
+
+            # store results in results array
+            results_array[0, p] = p_r
+            results_array[1, p] = p_std
+            # store Sharpe Ratio (return / volatility) - risk free rate element
+            # excluded for simplicity
+            results_array[2, p] = results_array[0, p] / results_array[1, p]
+            i = 0
+            for iw in weights:
+                results_array[3 + i, p] = weights[i]
+                i += 1
+
+        # convert results array to Pandas DataFrame
+        results_frame = pd.DataFrame(np.transpose(results_array),
+                                     columns=['r', 'stdev', 'sharpe']
+                                             + context.assets)
+        # locate position of portfolio with highest Sharpe Ratio
+        max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
+        # locate positon of portfolio with minimum standard deviation
+        # min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+
+        # order optimal weights for each asset
+        for asset in context.assets:
+            if data.can_trade(asset):
+                order_target_percent(asset, max_sharpe_port[asset])
+
+        # create scatter plot coloured by Sharpe Ratio
+        plt.scatter(results_frame.stdev,
+                    results_frame.r,
+                    c=results_frame.sharpe,
+                    cmap='RdYlGn')
+        plt.xlabel('Volatility')
+        plt.ylabel('Returns')
+        plt.colorbar()
+        # plot red star to highlight position of portfolio
+        # with highest Sharpe Ratio
+        plt.scatter(max_sharpe_port[1],
+                    max_sharpe_port[0],
+                    marker='o',
+                    color='b',
+                    s=200)
+        # plot green star to highlight position of minimum variance portfolio
+        plt.show()
+        print(max_sharpe_port)
+        record(pr=pr,
+               r=r,
+               m=m,
+               stds=stds,
+               max_sharpe_port=max_sharpe_port,
+               corr_m=corr_m)
+    context.i += 1
+
+
+def analyze(context=None, results=None):
+    # Form DataFrame with selected data
+    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
+                    'portfolio_value']]
+
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename(__file__))[0]
+    data.to_csv(filename + '.csv')
+
+
+if __name__ == '__main__':
+    # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
+    start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+    end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
+    results = run_algorithm(initialize=initialize,
+                            handle_data=handle_data,
+                            analyze=analyze,
+                            start=start,
+                            end=end,
+                            exchange_name='poloniex',
+                            capital_base=100000, )

catalyst/examples/rsi_profit_target.py

@@ -0,0 +1,265 @@
+from datetime import timedelta
+
+import pandas as pd
+import numpy as np
+import talib
+from logbook import Logger
+
+from catalyst.api import (
+    order,
+    symbol,
+    record,
+    get_open_orders,
+)
+from catalyst.utils.run_algo import run_algorithm
+
+algo_namespace = 'rsi'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('initializing algo')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
+
+    context.MAX_HOLDINGS = 0.2
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERSOLD_BBANDS = 45
+    context.RSI_OVERBOUGHT_BBANDS = 55
+    context.SLIPPAGE_ALLOWED = 0.03
+
+    context.TARGET = 0.15
+    context.STOP_LOSS = 0.1
+    context.STOP = 0.03
+    context.position = None
+
+    context.last_bar = None
+
+    context.errors = []
+    pass
+
+
+def _handle_buy_sell_decision(context, data, signal, price):
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        log.info('skipping bar until all open orders execute')
+        return
+
+    positions = context.portfolio.positions
+    if context.position is None and context.asset in positions:
+        position = positions[context.asset]
+        context.position = dict(
+            cost_basis=position['cost_basis'],
+            amount=position['amount'],
+            stop=None
+        )
+
+    # action = None
+    if context.position is not None:
+        cost_basis = context.position['cost_basis']
+        amount = context.position['amount']
+        log.info(
+            'found {amount} positions with cost basis {cost_basis}'.format(
+                amount=amount,
+                cost_basis=cost_basis
+            )
+        )
+        stop = context.position['stop']
+
+        target = cost_basis * (1 + context.TARGET)
+        if price >= target:
+            context.position['cost_basis'] = price
+            context.position['stop'] = context.STOP
+
+        stop_target = context.STOP_LOSS if stop is None else context.STOP
+        if price < cost_basis * (1 - stop_target):
+            log.info('executing stop loss')
+            order(
+                asset=context.asset,
+                amount=-amount,
+                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            # action = 0
+            context.position = None
+
+    else:
+        if signal == 'long':
+            log.info('opening position')
+            buy_amount = context.MAX_HOLDINGS / price
+            order(
+                asset=context.asset,
+                amount=buy_amount,
+                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
+            )
+            context.position = dict(
+                cost_basis=price,
+                amount=buy_amount,
+                stop=None
+            )
+            # action = 0
+
+
+def _handle_data_rsi_only(context, data):
+    price = data.current(context.asset, 'close')
+    log.info('got price {price}'.format(price=price))
+
+    if price is np.nan:
+        log.warn('no pricing data')
+        return
+
+    if context.base_price is None:
+        context.base_price = price
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=20,
+            frequency='30T'
+        )
+    except Exception as e:
+        log.warn('historical data not available: '.format(e))
+        return
+
+    rsi = talib.RSI(prices.values, timeperiod=16)[-1]
+    log.info('got rsi {}'.format(rsi))
+
+    signal = None
+    if rsi < context.RSI_OVERSOLD:
+        signal = 'long'
+
+    # Making sure that the price is still current
+    price = data.current(context.asset, 'close')
+    cash = context.portfolio.cash
+    log.info(
+        'base currency available: {cash}, cap: {cap}'.format(
+            cash=cash,
+            cap=context.MAX_HOLDINGS
+        )
+    )
+    volume = data.current(context.asset, 'volume')
+    price_change = (price - context.base_price) / context.base_price
+    record(
+        price=price,
+        price_change=price_change,
+        rsi=rsi,
+        volume=volume,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+    )
+
+    _handle_buy_sell_decision(context, data, signal, price)
+
+
+def handle_data(context, data):
+    dt = data.current_dt
+
+    if context.last_bar is None or (
+            context.last_bar + timedelta(minutes=15)) <= dt:
+        context.last_bar = dt
+    else:
+        return
+
+    log.info('BAR {}'.format(dt))
+    try:
+        _handle_data_rsi_only(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    results.loc[:, 'price'].plot(ax=ax2)
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    trans = results.loc[[t != [] for t in results.transactions], :]
+    buys = trans.loc[[t[0]['amount'] > 0 for t in trans.transactions], :]
+    sells = trans.loc[[t[0]['amount'] < 0 for t in trans.transactions], :]
+    # buys = results.loc[results['action'] == 1, :]
+    # sells = results.loc[results['action'] == 0, :]
+
+    ax2.plot(
+        buys.index,
+        results.loc[buys.index, 'price'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        results.loc[sells.index, 'price'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results.loc[:, ['alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Alpha / Beta ')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results.loc[:, ['starting_cash', 'cash']].plot(ax=ax4)
+    ax4.set_ylabel('Base Currency ({})'.format(base_currency))
+
+    results['algorithm'] = results.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results.loc[:, 'rsi'].plot(ax=ax6)
+    ax6.set_ylabel('RSI')
+
+    ax6.plot(
+        buys.index,
+        results.loc[buys.index, 'rsi'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax6.plot(
+        sells.index,
+        results.loc[sells.index, 'rsi'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # Backtest
+    run_algorithm(
+        capital_base=0.5,
+        data_frequency='minute',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        algo_namespace=algo_namespace,
+        base_currency='btc',
+        start=pd.to_datetime('2017-9-1', utc=True),
+        end=pd.to_datetime('2017-10-1', utc=True),
+    )

catalyst/examples/simple_loop.py

@@ -1,13 +1,16 @@
-import pandas as pd
 import talib
+import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.api import symbol
+from catalyst.api import symbol, record
+from catalyst.exchange.stats_utils import get_pretty_stats, \
+    extract_transactions
 
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('xrp_btc')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
 
 
 def handle_data(context, data):
@@ -19,33 +22,104 @@ def handle_data(context, data):
     prices = data.history(
         context.asset,
         fields='price',
-        bar_count=15,
-        frequency='1d'
+        bar_count=20,
+        frequency='30T'
     )
+    last_traded = prices.index[-1]
+    print('last candle date: {}'.format(last_traded))
+
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     print('got rsi: {}'.format(rsi))
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        price_change=price_change,
+        cash=cash
+    )
+
+
+def analyze(context, perf):
+    import matplotlib.pyplot as plt
+    print('the stats: {}'.format(get_pretty_stats(perf)))
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
     pass
 
 
-# run_algorithm(
-#     capital_base=250,
-#     start=pd.to_datetime('2015-08-01', utc=True),
-#     end=pd.to_datetime('2017-9-30', utc=True),
-#     data_frequency='daily',
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=None,
-#     exchange_name='poloniex',
-#     algo_namespace='simple_loop',
-#     base_currency='eth'
-# )
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=None,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace='simple_loop',
-    base_currency='eth',
-    live_graph=False
-)
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=1,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=None,
+        exchange_name='poloniex',
+        live=True,
+        algo_namespace='simple_loop',
+        base_currency='eth',
+        live_graph=False,
+        simulate_orders=True
+    )

catalyst/examples/simple_universe.py

@@ -0,0 +1,171 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.3
+
+This example aims to provide an easy way for users to learn how to
+collect data from any given exchange and select a subset of the available
+currency pairs for trading. You simply need to specify the exchange and
+the market (base_currency) that you want to focus on. You will then see
+how to create a universe of assets, and filter it based the market you
+desire.
+
+The example prints out the closing price of all the pairs for a given
+market in a given exchange every 30 minutes. The example also contains
+the OHLCV data with minute-resolution for the past seven days which
+could be used to create indicators. Use this code as the backbone to
+create your own trading strategy.
+
+The lookback_date variable is used to ensure data for a coin existed on
+the lookback period specified.
+
+To run, execute the following two commands in a terminal (inside catalyst
+environment). The first one retrieves all the pricing data needed for this
+script to run (only needs to be run once), and the second one executes this
+script with the parameters specified in the run_algorithm() call at the end
+of the file:
+
+catalyst ingest-exchange -x bitfinex -f minute
+
+python simple_universe.py
+
+"""
+from datetime import timedelta
+
+import numpy as np
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+from catalyst.api import (symbols, )
+
+
+def initialize(context):
+    context.i = -1  # minute counter
+    context.exchange = context.exchanges.values()[0].name.lower()
+    context.base_currency = context.exchanges.values()[0].base_currency.lower()
+
+
+def handle_data(context, data):
+    context.i += 1
+    lookback_days = 7  # 7 days
+
+    # current date & time in each iteration formatted into a string
+    now = data.current_dt
+    date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = now - timedelta(days=lookback_days)
+    # keep only the date as a string, discard the time
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
+
+    one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
+    # update universe everyday at midnight
+    if not context.i % one_day_in_minutes:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+
+    # get lookback_days of history data: that is 'lookback' number of bins
+    lookback = one_day_in_minutes / minutes * lookback_days
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # Get 30 minute interval OHLCV data. This is the standard data
+            # required for candlestick or indicators/signals. Return Pandas
+            # DataFrames. 30T means 30-minute re-sampling of one minute data.
+            # Adjust it to your desired time interval as needed.
+            opened = fill(data.history(coin,
+                                       'open',
+                                       bar_count=lookback,
+                                       frequency='30T')).values
+            high = fill(data.history(coin,
+                                     'high',
+                                     bar_count=lookback,
+                                     frequency='30T')).values
+            low = fill(data.history(coin,
+                                    'low',
+                                    bar_count=lookback,
+                                    frequency='30T')).values
+            close = fill(data.history(coin,
+                                      'price',
+                                      bar_count=lookback,
+                                      frequency='30T')).values
+            volume = fill(data.history(coin,
+                                       'volume',
+                                       bar_count=lookback,
+                                       frequency='30T')).values
+
+            # close[-1] is the last value in the set, which is the equivalent
+            # to current price (as in the most recent value)
+            # displays the minute price for each pair every 30 minutes
+            print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
+                  '\tV:{v}'.format(
+                    now=now,
+                    pair=pair,
+                    o=opened[-1],
+                    h=high[-1],
+                    l=low[-1],
+                    c=close[-1],
+                    v=volume[-1],
+                  ))
+
+            # -------------------------------------------------------------
+            # --------------- Insert Your Strategy Here -------------------
+            # -------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    # get all the pairs for the given exchange
+    json_symbols = get_exchange_symbols(context.exchange)
+    # convert into a DataFrame for easier processing
+    df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
+    df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
+                                   axis=1)
+    df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
+                                     axis=1)
+
+    # Filter all the pairs to get only the ones for a given base_currency
+    df = df[df['base_currency'] == context.base_currency]
+
+    # Filter all pairs to ensure that pair existed in the current date range
+    df = df[df.start_date < lookback_date]
+    df = df[df.end_daily >= current_date]
+    context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
+
+    return df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace(
+                     [np.inf, -np.inf], np.nan
+                    ).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-11-10', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=100.0,  # amount of base_currency
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')

catalyst/examples/talib_simple.py

@@ -0,0 +1,366 @@
+# Run Command
+# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
+#   -f talib_simple.py -x poloniex
+#
+# Description
+# Simple TALib Example showing how to use various indicators
+# in you strategy. Based loosly on
+# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
+
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import talib as ta
+from logbook import Logger
+from matplotlib.dates import date2num
+from matplotlib.finance import candlestick_ohlc
+
+from catalyst import run_algorithm
+from catalyst.api import (
+    order,
+    order_target_percent,
+    symbol,
+)
+from catalyst.exchange.stats_utils import get_pretty_stats
+
+algo_namespace = 'talib_sample'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('Starting TALib Simple Example')
+
+    context.ASSET_NAME = 'BTC_USDT'
+    context.asset = symbol(context.ASSET_NAME)
+
+    context.ORDER_SIZE = 10
+    context.SLIPPAGE_ALLOWED = 0.05
+
+    context.swallow_errors = True
+    context.errors = []
+
+    # Bars to look at per iteration should be bigger than SMA_SLOW
+    context.BARS = 365
+    context.COUNT = 0
+
+    # Technical Analysis Settings
+    context.SMA_FAST = 50
+    context.SMA_SLOW = 100
+    context.RSI_PERIOD = 14
+    context.RSI_OVER_BOUGHT = 80
+    context.RSI_OVER_SOLD = 20
+    context.RSI_AVG_PERIOD = 15
+    context.MACD_FAST = 12
+    context.MACD_SLOW = 26
+    context.MACD_SIGNAL = 9
+    context.STOCH_K = 14
+    context.STOCH_D = 3
+    context.STOCH_OVER_BOUGHT = 80
+    context.STOCH_OVER_SOLD = 20
+
+    pass
+
+
+def _handle_data(context, data):
+    # Get price, open, high, low, close
+    prices = data.history(
+        context.asset,
+        bar_count=context.BARS,
+        fields=['price', 'open', 'high', 'low', 'close'],
+        frequency='1d')
+
+    # Create a analysis data frame
+    analysis = pd.DataFrame(index=prices.index)
+
+    # SMA FAST
+    analysis['sma_f'] = ta.SMA(prices.close.as_matrix(), context.SMA_FAST)
+    # SMA SLOW
+    analysis['sma_s'] = ta.SMA(prices.close.as_matrix(), context.SMA_SLOW)
+
+    # Relative Strength Index
+    analysis['rsi'] = ta.RSI(prices.close.as_matrix(), context.RSI_PERIOD)
+    # RSI SMA
+    analysis['sma_r'] = ta.SMA(analysis.rsi.as_matrix(),
+                               context.RSI_AVG_PERIOD)
+
+    # MACD, MACD Signal, MACD Histogram
+    analysis['macd'], analysis['macdSignal'], analysis['macdHist'] = ta.MACD(
+        prices.close.as_matrix(), fastperiod=context.MACD_FAST,
+        slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
+
+    # Stochastics %K %D
+    # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
+    # %D = 3-day SMA of %K
+    analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
+        prices.high.as_matrix(), prices.low.as_matrix(),
+        prices.close.as_matrix(), slowk_period=context.STOCH_K,
+        slowd_period=context.STOCH_D)
+
+    # SMA FAST over SLOW Crossover
+    analysis['sma_test'] = np.where(analysis.sma_f > analysis.sma_s, 1, 0)
+
+    # MACD over Signal Crossover
+    analysis['macd_test'] = np.where((analysis.macd > analysis.macdSignal), 1,
+                                     0)
+
+    # Stochastics OVER BOUGHT & Decreasing
+    analysis['stoch_over_bought'] = np.where(
+        (analysis.stoch_k > context.STOCH_OVER_BOUGHT) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # Stochastics OVER SOLD & Increasing
+    analysis['stoch_over_sold'] = np.where(
+        (analysis.stoch_k < context.STOCH_OVER_SOLD) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # RSI OVER BOUGHT & Decreasing
+    analysis['rsi_over_bought'] = np.where(
+        (analysis.rsi > context.RSI_OVER_BOUGHT) & (
+            analysis.rsi < analysis.rsi.shift(1)), 1, 0)
+
+    # RSI OVER SOLD & Increasing
+    analysis['rsi_over_sold'] = np.where(
+        (analysis.rsi < context.RSI_OVER_SOLD) & (
+            analysis.rsi > analysis.rsi.shift(1)), 1, 0)
+
+    # Save the prices and analysis to send to analyze
+    context.prices = prices
+    context.analysis = analysis
+    context.price = data.current(context.asset, 'price')
+
+    makeOrders(context, analysis)
+
+    # Log the values of this bar
+    logAnalysis(analysis)
+
+
+def handle_data(context, data):
+    log.info('handling bar {}'.format(data.current_dt))
+    try:
+        _handle_data(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    log.info('completed bar {}, total execution errors {}'.format(
+        data.current_dt,
+        len(context.errors)
+    ))
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context, results):
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename('talib_simple'))[0]
+    results.to_csv(filename + '.csv')
+
+    log.info('the daily stats:\n{}'.format(get_pretty_stats(results)))
+    chart(context, context.prices, context.analysis, results)
+    pass
+
+
+def makeOrders(context, analysis):
+    if context.asset in context.portfolio.positions:
+
+        # Current position
+        position = context.portfolio.positions[context.asset]
+
+        if (position == 0):
+            log.info('Position Zero')
+            return
+
+        # Cost Basis
+        cost_basis = position.cost_basis
+
+        log.info(
+            'Holdings: {amount} @ {cost_basis}'.format(
+                amount=position.amount,
+                cost_basis=cost_basis
+            )
+        )
+
+        # Sell when holding and got sell singnal
+        if isSell(context, analysis):
+            profit = (context.price * position.amount) - (
+                cost_basis * position.amount)
+            order_target_percent(
+                asset=context.asset,
+                target=0,
+                limit_price=context.price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            log.info(
+                'Sold {amount} @ {price} Profit: {profit}'.format(
+                    amount=position.amount,
+                    price=context.price,
+                    profit=profit
+                )
+            )
+        else:
+            log.info('no buy or sell opportunity found')
+    else:
+        # Buy when not holding and got buy signal
+        if isBuy(context, analysis):
+            order(
+                asset=context.asset,
+                amount=context.ORDER_SIZE,
+                limit_price=context.price * (1 + context.SLIPPAGE_ALLOWED)
+            )
+            log.info(
+                'Bought {amount} @ {price}'.format(
+                    amount=context.ORDER_SIZE,
+                    price=context.price
+                )
+            )
+
+
+def isBuy(context, analysis):
+    # Bullish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 1):
+        # Bullish MACD
+        if (getLast(analysis, 'macd_test') == 1):
+            return True
+
+    # # Bullish Stochastics
+    # if(getLast(analysis, 'stoch_over_sold') == 1):
+    #     return True
+
+    # # Bullish RSI
+    # if(getLast(analysis, 'rsi_over_sold') == 1):
+    #     return True
+
+    return False
+
+
+def isSell(context, analysis):
+    # Bearish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 0):
+        # Bearish MACD
+        if (getLast(analysis, 'macd_test') == 0):
+            return True
+
+    # # Bearish Stochastics
+    # if(getLast(analysis, 'stoch_over_bought') == 0):
+    #     return True
+
+    # # Bearish RSI
+    # if(getLast(analysis, 'rsi_over_bought') == 0):
+    #     return True
+
+    return False
+
+
+def chart(context, prices, analysis, results):
+    results.portfolio_value.plot()
+
+    # Data for matplotlib finance plot
+    dates = date2num(prices.index.to_pydatetime())
+
+    # Create the Open High Low Close Tuple
+    prices_ohlc = [tuple([dates[i],
+                          prices.open[i],
+                          prices.high[i],
+                          prices.low[i],
+                          prices.close[i]]) for i in range(len(dates))]
+
+    fig = plt.figure(figsize=(14, 18))
+
+    # Draw the candle sticks
+    ax1 = fig.add_subplot(411)
+    ax1.set_ylabel(context.ASSET_NAME, size=20)
+    candlestick_ohlc(ax1, prices_ohlc, width=0.4, colorup='g', colordown='r')
+
+    # Draw Moving Averages
+    analysis.sma_f.plot(ax=ax1, c='r')
+    analysis.sma_s.plot(ax=ax1, c='g')
+
+    # RSI
+    ax2 = fig.add_subplot(412)
+    ax2.set_ylabel('RSI', size=12)
+    analysis.rsi.plot(ax=ax2, c='g',
+                      label='Period: ' + str(context.RSI_PERIOD))
+    analysis.sma_r.plot(ax=ax2, c='r',
+                        label='MA: ' + str(context.RSI_AVG_PERIOD))
+    ax2.axhline(y=30, c='b')
+    ax2.axhline(y=50, c='black')
+    ax2.axhline(y=70, c='b')
+    ax2.set_ylim([0, 100])
+    handles, labels = ax2.get_legend_handles_labels()
+    ax2.legend(handles, labels)
+
+    # Draw MACD computed with Talib
+    ax3 = fig.add_subplot(413)
+    ax3.set_ylabel('MACD: ' + str(context.MACD_FAST) + ', ' + str(
+        context.MACD_SLOW) + ', ' + str(context.MACD_SIGNAL), size=12)
+    analysis.macd.plot(ax=ax3, color='b', label='Macd')
+    analysis.macdSignal.plot(ax=ax3, color='g', label='Signal')
+    analysis.macdHist.plot(ax=ax3, color='r', label='Hist')
+    ax3.axhline(0, lw=2, color='0')
+    handles, labels = ax3.get_legend_handles_labels()
+    ax3.legend(handles, labels)
+
+    # Stochastic plot
+    ax4 = fig.add_subplot(414)
+    ax4.set_ylabel('Stoch (k,d)', size=12)
+    analysis.stoch_k.plot(ax=ax4, label='stoch_k:' + str(context.STOCH_K),
+                          color='r')
+    analysis.stoch_d.plot(ax=ax4, label='stoch_d:' + str(context.STOCH_D),
+                          color='g')
+    handles, labels = ax4.get_legend_handles_labels()
+    ax4.legend(handles, labels)
+    ax4.axhline(y=20, c='b')
+    ax4.axhline(y=50, c='black')
+    ax4.axhline(y=80, c='b')
+
+    plt.show()
+
+
+def logAnalysis(analysis):
+    # Log only the last value in the array
+    log.info('- sma_f:          {:.2f}'.format(getLast(analysis, 'sma_f')))
+    log.info('- sma_s:          {:.2f}'.format(getLast(analysis, 'sma_s')))
+
+    log.info('- rsi:            {:.2f}'.format(getLast(analysis, 'rsi')))
+    log.info('- sma_r:          {:.2f}'.format(getLast(analysis, 'sma_r')))
+
+    log.info('- macd:           {:.2f}'.format(getLast(analysis, 'macd')))
+    log.info(
+        '- macdSignal:     {:.2f}'.format(getLast(analysis, 'macdSignal')))
+    log.info('- macdHist:       {:.2f}'.format(getLast(analysis, 'macdHist')))
+
+    log.info('- stoch_k:        {:.2f}'.format(getLast(analysis, 'stoch_k')))
+    log.info('- stoch_d:        {:.2f}'.format(getLast(analysis, 'stoch_d')))
+
+    log.info('- sma_test:       {}'.format(getLast(analysis, 'sma_test')))
+    log.info('- macd_test:      {}'.format(getLast(analysis, 'macd_test')))
+
+    log.info('- stoch_over_bought:   {}'.format(
+        getLast(analysis, 'stoch_over_bought')))
+    log.info(
+        '- stoch_over_sold:   {}'.format(getLast(analysis, 'stoch_over_sold')))
+
+    log.info('- rsi_over_bought:       {}'.format(
+        getLast(analysis, 'rsi_over_bought')))
+    log.info(
+        '- rsi_over_sold:       {}'.format(getLast(analysis, 'rsi_over_sold')))
+
+
+def getLast(arr, name):
+    return arr[name][arr[name].index[-1]]
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        base_currency='usdt',
+        start=pd.to_datetime('2016-11-1', utc=True),
+        end=pd.to_datetime('2017-11-10', utc=True),
+    )

catalyst/exchange/asset_finder_exchange.py

@@ -41,14 +41,15 @@ class AssetFinderExchange(object):
         SidsNotFound
             When a requested sid is not found and default_none=False.
         """
-        for sid in sids:
-            if sid in self._asset_cache:
-                log.debug('got asset from cache: {}'.format(sid))
-            else:
-                log.debug('fetching asset: {}'.format(sid))
+        # for sid in sids:
+        #     if sid in self._asset_cache:
+        #         log.debug('got asset from cache: {}'.format(sid))
+        #     else:
+        #         log.debug('fetching asset: {}'.format(sid))
         return list()
 
-    def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
         """Lookup an asset by symbol.
 
         Parameters
@@ -84,10 +85,15 @@ class AssetFinderExchange(object):
         """
         log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
 
-        key = ','.join([exchange.name, symbol])
+        if data_frequency is not None:
+            key = ','.join([exchange.name, symbol, data_frequency])
+
+        else:
+            key = ','.join([exchange.name, symbol])
+
         if key in self._asset_cache:
             return self._asset_cache[key]
         else:
-            asset = exchange.get_asset(symbol)
+            asset = exchange.get_asset(symbol, data_frequency)
             self._asset_cache[key] = asset
             return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -14,6 +14,7 @@ import six
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
@@ -23,22 +24,23 @@ from catalyst.exchange.exchange_errors import (
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
+from catalyst.utils.deprecate import deprecated
+
 requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 
+@deprecated
 class Bitfinex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
         self.url = BITFINEX_URL
@@ -46,8 +48,13 @@ class Bitfinex(Exchange):
         self.secret = secret.encode('UTF-8')
         self.name = 'bitfinex'
         self.color = 'green'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -61,7 +68,7 @@ class Bitfinex(Exchange):
         self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def _request(self, operation, data, version='v1'):
         payload_object = {
@@ -167,7 +174,8 @@ class Bitfinex(Exchange):
 
         executed_price = float(order_status['avg_execution_price'])
 
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
+        # TODO: bitfinex does not specify comission.
+        # I could calculate it but not sure if it's worth it.
         commission = None
 
         date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
@@ -240,7 +248,7 @@ class Bitfinex(Exchange):
         # TODO: fetch account data and keep in cache
         return None
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count=None,
                     start_dt=None, end_dt=None):
         """
         Retrieve OHLVC candles from Bitfinex
@@ -255,33 +263,40 @@ class Bitfinex(Exchange):
         '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
          '1M'
         """
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
 
-        freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
+        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
+                               '360T', '720T', '1D', '7D', '14D', '30D']
+        if freq not in allowed_frequencies:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
         if freq_match:
             number = int(freq_match.group(1))
             unit = freq_match.group(2)
 
-            if unit == 'd':
-                converted_unit = 'D'
+            if unit == 'T':
+                if number in [60, 180, 360, 720]:
+                    number = number / 60
+                    converted_unit = 'h'
+                else:
+                    converted_unit = 'm'
             else:
                 converted_unit = unit
 
             frequency = '{}{}'.format(number, converted_unit)
-            allowed_frequencies = ['1m', '5m', '15m', '30m', '1h', '3h', '6h',
-                                   '12h', '1D', '7D', '14D', '1M']
 
-            if frequency not in allowed_frequencies:
-                raise InvalidHistoryFrequencyError(
-                    frequency=data_frequency
-                )
-        elif data_frequency == 'minute':
-            frequency = '1m'
-        elif data_frequency == 'daily':
-            frequency = '1D'
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
@@ -587,17 +602,17 @@ class Bitfinex(Exchange):
             else:
                 try:
                     start_date = cached_symbols[symbol]['start_date']
-                except KeyError as e:
+                except KeyError:
                     start_date = time.strftime('%Y-%m-%d')
 
             try:
                 end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[symbol] = dict(
@@ -648,15 +663,16 @@ class Bitfinex(Exchange):
 
         """
             Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than now: time.time()
+            the startmonth we got above. Avoid end dates greater than
+            now: time.time()
         """
-        url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
+        url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
+               '&end={end}').format(
             url=self.url,
             symbol=symbol_v2,
             start=startmonth - 3600 * 24 * 31 * 1000,
             end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000))
-        )
+                    int(time.time() * 1000)))
 
         try:
             self.ask_request()

catalyst/exchange/bittrex/bittrex.py

@@ -1,4 +1,5 @@
 import json
+import time
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
@@ -13,18 +14,22 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
 
+# TODO: consider using this: https://github.com/mondeja/bittrex_v2
+from catalyst.utils.deprecate import deprecated
+
 log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 
 
+@deprecated
 class Bittrex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Bittrex_api(key=key, secret=secret)
         self.name = 'bittrex'
         self.color = 'blue'
         self.base_currency = base_currency
@@ -43,7 +48,10 @@ class Bittrex(Exchange):
         self.assets = dict()
         self.load_assets()
 
-        self.bundle = ExchangeBundle(self)
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
+        self.bundle = ExchangeBundle(self.name)
 
     @property
     def account(self):
@@ -65,10 +73,10 @@ class Bittrex(Exchange):
         return exchange_symbol.lower()
 
     def get_balances(self):
+        balances = self.api.getbalances()
         try:
             log.debug('retrieving wallet balances')
             self.ask_request()
-            balances = self.api.getbalances()
 
         except Exception as e:
             raise ExchangeRequestError(error=e)
@@ -207,45 +215,59 @@ class Bittrex(Exchange):
                 error=status['message']
             )
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
-                    start_date=None):
+    def get_candles(self, freq, assets, bar_count=None,
+                    start_dt=None, end_dt=None):
         """
         Supported Intervals
         -------------------
         day, oneMin, fiveMin, thirtyMin, hour
 
-        :param data_frequency:
+        :param freq:
         :param assets:
         :param bar_count:
+        :param start_dt
+        :param end_dt
         :return:
         """
-        log.info('retrieving candles')
 
-        if data_frequency == 'minute' or data_frequency == '1m':
+        # TODO: this has no effect at the moment
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
+
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        if freq == '1T':
             frequency = 'oneMin'
-        elif data_frequency == '5m':
+        elif freq == '5T':
             frequency = 'fiveMin'
-        elif data_frequency == '30m':
+        elif freq == '30T':
             frequency = 'thirtyMin'
-        elif data_frequency == '1h':
+        elif freq == '60T':
             frequency = 'hour'
-        elif data_frequency == 'daily' or data_frequency == '1D':
+        elif freq == '1D':
             frequency = 'day'
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
         for asset in asset_list:
+            end = int(time.mktime(end_dt.timetuple()))
             url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_=1499127220008'.format(
-                url=URL2,
-                symbol=self.get_symbol(asset),
-                frequency=frequency
-            )
+                  '&tickInterval={frequency}&_={end}'.format(
+                    url=URL2,
+                    symbol=self.get_symbol(asset),
+                    frequency=frequency,
+                    end=end, )
 
             try:
                 data = json.loads(urllib.request.urlopen(url).read().decode())
@@ -272,9 +294,11 @@ class Bittrex(Exchange):
                 return ohlc
 
             ordered_candles = list(reversed(candles))
+            ohlc_map = dict()
             if bar_count is None:
                 ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
             else:
+                # TODO: optimize
                 ohlc_bars = []
                 for candle in ordered_candles[:bar_count]:
                     ohlc = ohlc_from_candle(candle)
@@ -336,12 +360,12 @@ class Bittrex(Exchange):
 
             try:
                 end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[exchange_symbol] = dict(

catalyst/exchange/bittrex/bittrex_api.py

@@ -3,11 +3,12 @@ import json
 import time
 import hmac
 import hashlib
-
-from six.moves import urllib
+import ssl
 
 # Workaround for backwards compatibility
 # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
+from six.moves import urllib
+
 urlopen = urllib.request.urlopen
 
 
@@ -39,13 +40,17 @@ class Bittrex_api(object):
         if method not in self.public:
             url += '&apikey=' + self.key
             url += '&nonce=' + str(int(time.time()))
-            signature = hmac.new(self.secret, url, hashlib.sha512).hexdigest()
+
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 url.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
             headers = {'apisign': signature}
         else:
             headers = {}
 
         req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(req).read())
+        response = json.loads(urlopen(
+            req, context=ssl._create_unverified_context()).read())
 
         if response["result"]:
             return response["result"]

catalyst/exchange/bittrex/extensions-example.py

@@ -4,4 +4,4 @@ from catalyst.exchange.exchange_bundle import exchange_bundle
 symbols = (
     'neo_btc',
 )
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
+register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/bundle_utils.py

@@ -7,22 +7,42 @@ import numpy as np
 import pandas as pd
 import pytz
 
-from catalyst.data.bundles import from_bundle_ingest_dirname
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_errors import NoDataAvailableOnExchange
 from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
-from catalyst.utils.deprecate import deprecated
-from catalyst.utils.paths import data_path
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 API_URL = 'http://data.enigma.co/api/v1'
 
 
 def get_date_from_ms(ms):
+    """
+    The date from the number of miliseconds from the epoch.
+
+    Parameters
+    ----------
+    ms: int
+
+    Returns
+    -------
+    datetime
+
+    """
     return datetime.fromtimestamp(ms / 1000.0)
 
 
 def get_seconds_from_date(date):
+    """
+    The number of seconds from the epoch.
+
+    Parameters
+    ----------
+    date: datetime
+
+    Returns
+    -------
+    int
+
+    """
     epoch = datetime.utcfromtimestamp(0)
     epoch = epoch.replace(tzinfo=pytz.UTC)
 
@@ -33,16 +53,19 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
     """
     Download and extract a bcolz bundle.
 
-    :param exchange_name:
-    :param symbol:
-    :param data_frequency:
-    :param period:
-    :return:
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
 
-    Note:
+    Returns
+    -------
+    str
         Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-    """
 
+    """
     root = get_exchange_bundles_folder(exchange_name)
     name = '{exchange}-{frequency}-{symbol}-{period}'.format(
         exchange=exchange_name,
@@ -55,9 +78,8 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
     if not os.path.isdir(path):
         url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
               'exchange-{exchange}/{name}.tar.gz'.format(
-            exchange=exchange_name,
-            name=name
-        )
+                exchange=exchange_name,
+                name=name)
 
         bytes = download_without_progress(url)
         with tarfile.open('r', fileobj=bytes) as tar:
@@ -67,113 +89,191 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
 
 
 def get_delta(periods, data_frequency):
+    """
+    Get a time delta based on the specified data frequency.
+
+    Parameters
+    ----------
+    periods: int
+    data_frequency: str
+
+    Returns
+    -------
+    timedelta
+
+    """
     return timedelta(minutes=periods) \
         if data_frequency == 'minute' else timedelta(days=periods)
 
 
-def get_periods_range(start_dt, end_dt, data_frequency):
-    freq = 'T' if data_frequency == 'minute' else 'D'
+def get_periods_range(start_dt, end_dt, freq):
+    """
+    Get a date range for the specified parameters.
+
+    Parameters
+    ----------
+    start_dt: datetime
+    end_dt: datetime
+    freq: str
+
+    Returns
+    -------
+    DateTimeIndex
+
+    """
+    if freq == 'minute':
+        freq = 'T'
+
+    elif freq == 'daily':
+        freq = 'D'
 
     return pd.date_range(start_dt, end_dt, freq=freq)
 
 
-def get_periods(start_dt, end_dt, data_frequency):
-    delta = end_dt - start_dt
+def get_periods(start_dt, end_dt, freq):
+    """
+    The number of periods in the specified range.
 
-    if data_frequency == 'minute':
-        delta_periods = delta.total_seconds() / 60
+    Parameters
+    ----------
+    start_dt: datetime
+    end_dt: datetime
+    freq: str
 
-    elif data_frequency == 'daily':
-        delta_periods = delta.total_seconds() / 60 / 60 / 24
+    Returns
+    -------
+    int
 
-    else:
-        raise ValueError('frequency not supported')
-
-    return int(delta_periods)
+    """
+    return len(get_periods_range(start_dt, end_dt, freq))
 
 
-def get_start_dt(end_dt, bar_count, data_frequency):
+def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
+    """
+    The start date based on specified end date and data frequency.
+
+    Parameters
+    ----------
+    end_dt: datetime
+    bar_count: int
+    data_frequency: str
+
+    Returns
+    -------
+    datetime
+
+    """
     periods = bar_count
     if periods > 1:
         delta = get_delta(periods, data_frequency)
         start_dt = end_dt - delta
+
+        if not include_first:
+            start_dt += get_delta(1, data_frequency)
     else:
         start_dt = end_dt
 
     return start_dt
 
 
-def get_adj_dates(start, end, assets, data_frequency):
+def get_period_label(dt, data_frequency):
     """
-    Contains a date range to the trading availability of the specified pairs.
+    The period label for the specified date and frequency.
+
+    Parameters
+    ----------
+    dt: datetime
+    data_frequency: str
+
+    Returns
+    -------
+    str
 
-    :param start:
-    :param end:
-    :param assets:
-    :param data_frequency:
-    :return:
     """
-    earliest_trade = None
-    last_entry = None
-    for asset in assets:
-        if earliest_trade is None or earliest_trade > asset.start_date:
-            earliest_trade = asset.start_date
-
-        end_asset = asset.end_minute if data_frequency == 'minute' else \
-            asset.end_daily
-        if end_asset is not None and \
-                (last_entry is None or end_asset > last_entry):
-            last_entry = end_asset
-
-    if start is None or earliest_trade > start:
-        start = earliest_trade
-
-    if end is None or (last_entry is not None and end > last_entry):
-        end = last_entry
-
-    if end is None or start >= end:
-        raise NoDataAvailableOnExchange(
-            exchange=asset.exchange.title(),
-            symbol=[asset.symbol.encode('utf-8')],
-            data_frequency=data_frequency,
-        )
-
-    return start, end
+    if data_frequency == 'minute':
+        return '{}-{:02d}'.format(dt.year, dt.month)
+    else:
+        return '{}'.format(dt.year)
 
 
-def get_month_start_end(dt):
+def get_month_start_end(dt, first_day=None, last_day=None):
     """
-    Returns the first and last day of the month for the specified date.
+    The first and last day of the month for the specified date.
+
+    Parameters
+    ----------
+    dt: datetime
+    first_day: datetime
+    last_day: datetime
+
+    Returns
+    -------
+    datetime, datetime
 
-    :param dt:
-    :return:
     """
     month_range = calendar.monthrange(dt.year, dt.month)
-    month_start = pd.to_datetime(datetime(
-        dt.year, dt.month, 1, 0, 0, 0, 0
-    ), utc=True)
 
-    month_end = pd.to_datetime(datetime(
-        dt.year, dt.month, month_range[1], 23, 59, 0, 0
-    ), utc=True)
+    if first_day:
+        month_start = first_day
+    else:
+        month_start = pd.to_datetime(datetime(
+            dt.year, dt.month, 1, 0, 0, 0, 0
+        ), utc=True)
+
+    if last_day:
+        month_end = last_day
+    else:
+        month_end = pd.to_datetime(datetime(
+            dt.year, dt.month, month_range[1], 23, 59, 0, 0
+        ), utc=True)
+
+        if month_end > pd.Timestamp.utcnow():
+            month_end = pd.Timestamp.utcnow().floor('1D')
 
     return month_start, month_end
 
 
-def get_year_start_end(dt):
+def get_year_start_end(dt, first_day=None, last_day=None):
     """
-    Returns the first and last day of the year for the specified date.
+    The first and last day of the year for the specified date.
+
+    Parameters
+    ----------
+
+    dt: datetime
+    first_day: datetime
+    last_day: datetime
+
+    Returns
+    -------
+    datetime, datetime
 
-    :param dt:
-    :return:
     """
-    year_start = pd.to_datetime(date(dt.year, 1, 1), utc=True)
-    year_end = pd.to_datetime(date(dt.year, 12, 31), utc=True)
+    year_start = first_day if first_day \
+        else pd.to_datetime(date(dt.year, 1, 1), utc=True)
+    year_end = last_day if last_day \
+        else pd.to_datetime(date(dt.year, 12, 31), utc=True)
+
+    if year_end > pd.Timestamp.utcnow():
+        year_end = pd.Timestamp.utcnow().floor('1D')
 
     return year_start, year_end
 
 
 def get_df_from_arrays(arrays, periods):
+    """
+    A DataFrame from the specified OHCLV arrays.
+
+    Parameters
+    ----------
+    arrays: Object
+    periods: DateTimeIndex
+
+    Returns
+    -------
+    DataFrame
+
+    """
     ohlcv = dict()
     for index, field in enumerate(
             ['open', 'high', 'low', 'close', 'volume']):
@@ -191,64 +291,68 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
     Evaluate whether price data of an asset is included has been ingested in
     the exchange bundle for the given date range.
 
-    :param asset:
-    :param start_dt:
-    :param end_dt:
-    :param reader:
-    :return:
+    Parameters
+    ----------
+    asset: TradingPair
+    start_dt: datetime
+    end_dt: datetime
+    reader: BcolzBarMinuteReader
+
+    Returns
+    -------
+    bool
+
     """
     has_data = True
-    if has_data and reader is not None:
-        try:
-            start_close = \
-                reader.get_value(asset.sid, start_dt, 'close')
+    dates = [start_dt, end_dt]
 
-            if np.isnan(start_close):
+    while dates and has_data:
+        try:
+            dt = dates.pop(0)
+            close = reader.get_value(asset.sid, dt, 'close')
+
+            if np.isnan(close):
                 has_data = False
 
-            else:
-                end_close = reader.get_value(asset.sid, end_dt, 'close')
-
-                if np.isnan(end_close):
-                    has_data = False
-
-        except Exception as e:
+        except Exception:
             has_data = False
 
-    else:
-        has_data = False
-
     return has_data
 
 
-@deprecated
-def find_most_recent_time(bundle_name):
+def get_assets(exchange, include_symbols, exclude_symbols):
     """
-    Find most recent "time folder" for a given bundle.
+    Get assets from an exchange, including or excluding the specified
+    symbols.
 
-    :param bundle_name:
-        The name of the targeted bundle.
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
 
-    :return folder:
-        The name of the time folder.
     """
-    try:
-        bundle_folders = os.listdir(
-            data_path([bundle_name]),
-        )
-    except OSError:
-        return None
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
 
-    most_recent_bundle = dict()
-    for folder in bundle_folders:
-        date = from_bundle_ingest_dirname(folder)
-        if not most_recent_bundle or date > \
-                most_recent_bundle[most_recent_bundle.keys()[0]]:
-            most_recent_bundle = dict()
-            most_recent_bundle[folder] = date
+        return exchange.get_assets(include_symbols_list)
 
-    if most_recent_bundle:
-        return most_recent_bundle.keys()[0]
     else:
-        return None
+        all_assets = exchange.get_assets()
 
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -0,0 +1,638 @@
+import re
+from collections import defaultdict
+
+import ccxt
+import pandas as pd
+import six
+from ccxt import ExchangeNotAvailable, InvalidOrder
+from logbook import Logger
+from six import string_types
+
+from catalyst.algorithm import MarketOrder
+from catalyst.assets._assets import TradingPair
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
+    ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
+    ExchangeNotFoundError, CreateOrderError
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.exchange_utils import mixin_market_params, \
+    from_ms_timestamp, get_epoch
+from catalyst.finance.order import Order, ORDER_STATUS
+
+log = Logger('CCXT', level=LOG_LEVEL)
+
+SUPPORTED_EXCHANGES = dict(
+    binance=ccxt.binance,
+    bitfinex=ccxt.bitfinex,
+    bittrex=ccxt.bittrex,
+    poloniex=ccxt.poloniex,
+    bitmex=ccxt.bitmex,
+    gdax=ccxt.gdax,
+)
+
+
+class CCXT(Exchange):
+    def __init__(self, exchange_name, key, secret, base_currency):
+        log.debug(
+            'finding {} in CCXT exchanges:\n{}'.format(
+                exchange_name, ccxt.exchanges
+            )
+        )
+        try:
+            # Making instantiation as explicit as possible for code tracking.
+            if exchange_name in SUPPORTED_EXCHANGES:
+                exchange_attr = SUPPORTED_EXCHANGES[exchange_name]
+
+            else:
+                exchange_attr = getattr(ccxt, exchange_name)
+
+            self.api = exchange_attr({
+                'apiKey': key,
+                'secret': secret,
+            })
+
+        except Exception:
+            raise ExchangeNotFoundError(exchange_name=exchange_name)
+
+        self._symbol_maps = [None, None]
+
+        try:
+            markets_symbols = self.api.load_markets()
+            log.debug('the markets:\n{}'.format(markets_symbols))
+
+        except ExchangeNotAvailable as e:
+            raise ExchangeRequestError(error=e)
+
+        self.name = exchange_name
+
+        self.markets = self.api.fetch_markets()
+        self.load_assets()
+
+        self.base_currency = base_currency
+        self.transactions = defaultdict(list)
+
+        self.num_candles_limit = 2000
+        self.max_requests_per_minute = 60
+        self.request_cpt = dict()
+
+        self.bundle = ExchangeBundle(self.name)
+
+    def account(self):
+        return None
+
+    def time_skew(self):
+        return None
+
+    def get_market(self, symbol):
+        """
+        The CCXT market.
+
+        Parameters
+        ----------
+        symbol:
+            The CCXT symbol.
+
+        Returns
+        -------
+        dict[str, Object]
+
+        """
+        s = self.get_symbol(symbol)
+        market = next(
+            (market for market in self.markets if market['symbol'] == s),
+            None,
+        )
+        return market
+
+    def get_symbol(self, asset_or_symbol):
+        """
+        The CCXT symbol.
+
+        Parameters
+        ----------
+        asset_or_symbol
+
+        Returns
+        -------
+
+        """
+        symbol = asset_or_symbol if isinstance(
+            asset_or_symbol, string_types
+        ) else asset_or_symbol.symbol
+
+        parts = symbol.split('_')
+        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+
+    def get_catalyst_symbol(self, market_or_symbol):
+        """
+        The Catalyst symbol.
+
+        Parameters
+        ----------
+        market_or_symbol
+
+        Returns
+        -------
+
+        """
+        if isinstance(market_or_symbol, string_types):
+            parts = market_or_symbol.split('/')
+            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+        else:
+            return '{}_{}'.format(
+                market_or_symbol['base'].lower(),
+                market_or_symbol['quote'].lower(),
+            )
+
+    def get_timeframe(self, freq):
+        """
+        The CCXT timeframe from the Catalyst frequency.
+
+        Parameters
+        ----------
+        freq: str
+            The Catalyst frequency (Pandas convention)
+
+        Returns
+        -------
+        str
+
+        """
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) \
+                if freq_match.group(1) else 1
+
+            unit = freq_match.group(2)
+
+        else:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+        if unit.lower() == 'd':
+            timeframe = '{}d'.format(candle_size)
+
+        elif unit.lower() == 'm' or unit == 'T':
+            timeframe = '{}m'.format(candle_size)
+
+        elif unit.lower() == 'h' or unit == 'T':
+            timeframe = '{}h'.format(candle_size)
+
+        return timeframe
+
+    def get_candles(self, freq, assets, bar_count=None, start_dt=None,
+                    end_dt=None):
+        is_single = (isinstance(assets, TradingPair))
+        if is_single:
+            assets = [assets]
+
+        symbols = self.get_symbols(assets)
+        timeframe = self.get_timeframe(freq)
+
+        ms = None
+        if start_dt is not None:
+            delta = start_dt - get_epoch()
+            ms = int(delta.total_seconds()) * 1000
+
+        candles = dict()
+        for asset in assets:
+            try:
+                ohlcvs = self.api.fetch_ohlcv(
+                    symbol=symbols[0],
+                    timeframe=timeframe,
+                    since=ms,
+                    limit=bar_count,
+                    params={}
+                )
+
+                candles[asset] = []
+                for ohlcv in ohlcvs:
+                    candles[asset].append(dict(
+                        last_traded=pd.to_datetime(
+                            ohlcv[0], unit='ms', utc=True
+                        ),
+                        open=ohlcv[1],
+                        high=ohlcv[2],
+                        low=ohlcv[3],
+                        close=ohlcv[4],
+                        volume=ohlcv[5]
+                    ))
+
+            except Exception as e:
+                raise ExchangeRequestError(error=e)
+
+        if is_single:
+            return six.next(six.itervalues(candles))
+
+        else:
+            return candles
+
+    def _fetch_symbol_map(self, is_local):
+        try:
+            return self.fetch_symbol_map(is_local)
+        except ExchangeSymbolsNotFound:
+            return None
+
+    def get_asset_defs(self, market):
+        """
+        The local and Catalyst definitions of the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dicts.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        asset_defs = []
+
+        for is_local in (False, True):
+            asset_def = self.get_asset_def(market, is_local)
+            asset_defs.append((asset_def, is_local))
+
+        return asset_defs
+
+    def get_asset_def(self, market, is_local=False):
+        """
+        The asset definition (in symbols.json files) corresponding
+        to the the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dict.
+        is_local
+            Whether to search in local or Catalyst asset definitions.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        exchange_symbol = market['id']
+
+        symbol_map = self._fetch_symbol_map(is_local)
+        if symbol_map is not None:
+            assets_lower = {k.lower(): v for k, v in symbol_map.items()}
+            key = exchange_symbol.lower()
+
+            asset = assets_lower[key] if key in assets_lower else None
+            if asset is not None:
+                return asset
+
+            else:
+                return None
+
+        else:
+            return None
+
+    def create_trading_pair(self, market, asset_def=None, is_local=False):
+        """
+        Creating a TradingPair from market and asset data.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+        asset_def: dict[str, Object]
+        is_local: bool
+
+        Returns
+        -------
+
+        """
+        data_source = 'local' if is_local else 'catalyst'
+        params = dict(
+            exchange=self.name,
+            data_source=data_source,
+            exchange_symbol=market['id'],
+        )
+        mixin_market_params(self.name, params, market)
+
+        if asset_def is not None:
+            params['symbol'] = asset_def['symbol']
+
+            params['start_date'] = asset_def['start_date'] \
+                if 'start_date' in asset_def else None
+
+            params['end_date'] = asset_def['end_date'] \
+                if 'end_date' in asset_def else None
+
+            params['leverage'] = asset_def['leverage'] \
+                if 'leverage' in asset_def else 1.0
+
+            params['asset_name'] = asset_def['asset_name'] \
+                if 'asset_name' in asset_def else None
+
+            params['end_daily'] = asset_def['end_daily'] \
+                if 'end_daily' in asset_def \
+                   and asset_def['end_daily'] != 'N/A' else None
+
+            params['end_minute'] = asset_def['end_minute'] \
+                if 'end_minute' in asset_def \
+                   and asset_def['end_minute'] != 'N/A' else None
+
+        else:
+            params['symbol'] = self.get_catalyst_symbol(market)
+            # TODO: add as an optional column
+            params['leverage'] = 1.0
+
+        return TradingPair(**params)
+
+    def load_assets(self):
+        self.assets = []
+
+        for market in self.markets:
+            asset_defs = self.get_asset_defs(market)
+
+            asset = None
+            for asset_def in asset_defs:
+                if asset_def[0] is not None or not asset_defs[1]:
+                    try:
+                        asset = self.create_trading_pair(
+                            market=market,
+                            asset_def=asset_def[0],
+                            is_local=asset_def[1]
+                        )
+                        self.assets.append(asset)
+
+                    except TypeError as e:
+                        log.warn('unable to add asset: {}'.format(e))
+
+            if asset is None:
+                asset = self.create_trading_pair(market=market)
+                self.assets.append(asset)
+
+    def get_balances(self):
+        try:
+            log.debug('retrieving wallets balances')
+            balances = self.api.fetch_balance()
+
+            balances_lower = dict()
+            for key in balances:
+                balances_lower[key.lower()] = balances[key]
+
+        except Exception as e:
+            log.debug('error retrieving balances: {}', e)
+            raise ExchangeRequestError(error=e)
+
+        return balances_lower
+
+    def _create_order(self, order_status):
+        """
+        Create a Catalyst order object from a CCXT order dictionary
+
+        Parameters
+        ----------
+        order_status: dict[str, Object]
+            The order dict from the CCXT api.
+
+        Returns
+        -------
+        Order
+            The Catalyst order object
+
+        """
+        if order_status['status'] == 'canceled':
+            status = ORDER_STATUS.CANCELLED
+
+        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
+            log.debug('found executed order {}'.format(order_status))
+            status = ORDER_STATUS.FILLED
+
+        elif order_status['status'] == 'open':
+            status = ORDER_STATUS.OPEN
+
+        else:
+            raise ValueError('invalid state for order')
+
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if order_status['side'] == 'sell':
+            amount = -amount
+            filled = -filled
+
+        price = order_status['price']
+        order_type = order_status['type']
+
+        limit_price = price if order_type == 'limit' else None
+        stop_price = None  # TODO: add support
+
+        executed_price = order_status['cost'] / order_status['amount']
+        commission = order_status['fee']
+        date = from_ms_timestamp(order_status['timestamp'])
+
+        # order_id = str(order_status['info']['clientOrderId'])
+        order_id = order_status['id']
+
+        # TODO: this won't work, redo the packages with a different key.
+        symbol = order_status['info']['symbol'] \
+            if 'symbol' in order_status['info'] \
+            else order_status['info']['Exchange']
+
+        order = Order(
+            dt=date,
+            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            amount=amount,
+            stop=stop_price,
+            limit=limit_price,
+            filled=filled,
+            id=order_id,
+            commission=commission
+        )
+        order.status = status
+
+        return order, executed_price
+
+    def create_order(self, asset, amount, is_buy, style):
+        symbol = self.get_symbol(asset)
+
+        if isinstance(style, ExchangeLimitOrder):
+            price = style.get_limit_price(is_buy)
+            order_type = 'limit'
+
+        elif isinstance(style, MarketOrder):
+            price = None
+            order_type = 'market'
+
+        else:
+            raise InvalidOrderStyle(
+                exchange=self.name,
+                style=style.__class__.__name__
+            )
+
+        side = 'buy' if amount > 0 else 'sell'
+
+        if hasattr(self.api, 'amount_to_lots'):
+            adj_amount = self.api.amount_to_lots(
+                symbol=symbol,
+                amount=abs(amount),
+            )
+            if adj_amount != abs(amount):
+                log.info(
+                    'adjusted order amount {} to {} based on lot size'.format(
+                        abs(amount), adj_amount,
+                    )
+                )
+        else:
+            adj_amount = abs(amount)
+
+        try:
+            result = self.api.create_order(
+                symbol=symbol,
+                type=order_type,
+                side=side,
+                amount=adj_amount,
+                price=price
+            )
+        except ExchangeNotAvailable as e:
+            log.debug('unable to create order: {}'.format(e))
+            raise ExchangeRequestError(error=e)
+
+        except InvalidOrder as e:
+            log.warn('the exchange rejected the order: {}'.format(e))
+            raise CreateOrderError(exchange=self.name, error=e)
+
+        if 'info' not in result:
+            raise ValueError('cannot use order without info attribute')
+
+        final_amount = adj_amount if side == 'buy' else -adj_amount
+        order_id = result['id']
+        order = Order(
+            dt=pd.Timestamp.utcnow(),
+            asset=asset,
+            amount=final_amount,
+            stop=style.get_stop_price(is_buy),
+            limit=style.get_limit_price(is_buy),
+            id=order_id
+        )
+        return order
+
+    def get_open_orders(self, asset):
+        try:
+            symbol = self.get_symbol(asset)
+            result = self.api.fetch_open_orders(
+                symbol=symbol,
+                since=None,
+                limit=None,
+                params=dict()
+            )
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        orders = []
+        for order_status in result:
+            order, executed_price = self._create_order(order_status)
+            if asset is None or asset == order.sid:
+                orders.append(order)
+
+        return orders
+
+    def get_order(self, order_id, asset_or_symbol=None):
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, the request might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            order_status = self.api.fetch_order(id=order_id, symbol=symbol)
+            order, executed_price = self._create_order(order_status)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        return order, executed_price
+
+    def cancel_order(self, order_param, asset_or_symbol=None):
+        order_id = order_param.id \
+            if isinstance(order_param, Order) else order_param
+
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, cancelling order might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            self.api.cancel_order(id=order_id, symbol=symbol)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+    def tickers(self, assets):
+        """
+        Retrieve current tick data for the given assets
+
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[dict[str, float]
+
+        """
+        tickers = dict()
+        for asset in assets:
+            try:
+                ccxt_symbol = self.get_symbol(asset)
+                ticker = self.api.fetch_ticker(ccxt_symbol)
+
+                ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
+
+                if 'last_price' not in ticker:
+                    # TODO: any more exceptions?
+                    ticker['last_price'] = ticker['last']
+
+                # Using the volume represented in the base currency
+                ticker['volume'] = ticker['baseVolume'] \
+                    if 'baseVolume' in ticker else 0
+
+                tickers[asset] = ticker
+
+            except ExchangeNotAvailable as e:
+                log.warn(
+                    'unable to fetch ticker: {} {}'.format(
+                        self.name, asset.symbol
+                    )
+                )
+                raise ExchangeRequestError(error=e)
+
+        return tickers
+
+    def get_account(self):
+        return None
+
+    def get_orderbook(self, asset, order_type='all', limit=None):
+        ccxt_symbol = self.get_symbol(asset)
+
+        params = dict()
+        if limit is not None:
+            params['depth'] = limit
+
+        order_book = self.api.fetch_order_book(ccxt_symbol, params)
+
+        order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
+        result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
+        for index, order_type in enumerate(order_types):
+            if limit is not None and index > limit - 1:
+                break
+
+            result[order_type] = []
+            for entry in order_book[order_type]:
+                result[order_type].append(dict(
+                    rate=float(entry[0]),
+                    quantity=float(entry[1])
+                ))
+
+        return result

catalyst/exchange/exchange_algorithm.py

@@ -10,11 +10,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import pickle
 import signal
 import sys
-from collections import deque
 from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
@@ -22,36 +20,32 @@ from time import sleep
 
 import logbook
 import pandas as pd
-from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
-from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
-    BcolzMinuteBarReader
-from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangePortfolioDataError,
-    ExchangeTransactionError,
-    OrphanOrderError)
-from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
-    ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
-    save_algo_object, get_algo_object, get_algo_folder, get_algo_df, \
-    save_algo_df
+    OrderTypeNotSupported, )
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.exchange_utils import (
+    save_algo_object,
+    get_algo_object,
+    get_algo_folder,
+    get_algo_df,
+    save_algo_df,
+    group_assets_by_exchange, )
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
+    stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
-from catalyst.utils.api_support import (
-    api_method,
-    disallowed_in_before_trading_start)
-from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
-    expect_types
+from catalyst.utils.api_support import api_method
+from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
 
@@ -66,9 +60,90 @@ class ExchangeAlgorithmExecutor(AlgorithmSimulator):
 class ExchangeTradingAlgorithmBase(TradingAlgorithm):
     def __init__(self, *args, **kwargs):
         self.exchanges = kwargs.pop('exchanges', None)
+        self.simulate_orders = kwargs.pop('simulate_orders', None)
 
         super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
 
+        self.current_day = None
+
+        if self.simulate_orders is None \
+                and self.sim_params.arena == 'backtest':
+            self.simulate_orders = True
+
+        self.blotter = ExchangeBlotter(
+            data_frequency=self.data_frequency,
+            # Default to NeverCancel in catalyst
+            cancel_policy=self.cancel_policy,
+            simulate_orders=self.simulate_orders,
+            exchanges=self.exchanges
+        )
+
+    @staticmethod
+    def __convert_order_params_for_blotter(limit_price, stop_price, style):
+        """
+        Helper method for converting deprecated limit_price and stop_price
+        arguments into ExecutionStyle instances.
+
+        This function assumes that either style == None or (limit_price,
+        stop_price) == (None, None).
+        """
+        if stop_price:
+            raise OrderTypeNotSupported(order_type='stop')
+
+        if style:
+            if limit_price is not None:
+                raise ValueError(
+                    'An order style and a limit price was included in the '
+                    'order. Please pick one to avoid any possible conflict.'
+                )
+
+            # Currently limiting order types or limit and market to
+            # be in-line with CXXT and many exchanges. We'll consider
+            # adding more order types in the future.
+            if not isinstance(style, ExchangeLimitOrder) or \
+                    not isinstance(style, MarketOrder):
+                raise OrderTypeNotSupported(
+                    order_type=style.__class__.__name__
+                )
+
+            return style
+
+        if limit_price:
+            return ExchangeLimitOrder(limit_price)
+        else:
+            return MarketOrder()
+
+    @api_method
+    def set_commission(self, maker=None, taker=None):
+        key = self.blotter.commission_models.keys()[0]
+        if maker is not None:
+            self.blotter.commission_models[key].maker = maker
+
+        if taker is not None:
+            self.blotter.commission_models[key].taker = taker
+
+    @api_method
+    def set_slippage(self, spread=None):
+        key = self.blotter.slippage_models.keys()[0]
+        if spread is not None:
+            self.blotter.slippage_models[key].spread = spread
+
+    def _calculate_order(self, asset, amount,
+                         limit_price=None, stop_price=None, style=None):
+        # Raises a ZiplineError if invalid parameters are detected.
+        self.validate_order_params(asset,
+                                   amount,
+                                   limit_price,
+                                   stop_price,
+                                   style)
+
+        # Convert deprecated limit_price and stop_price parameters to use
+        # ExecutionStyle objects.
+        style = self.__convert_order_params_for_blotter(limit_price,
+                                                        stop_price,
+                                                        style)
+        return amount, style
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies
@@ -113,13 +188,16 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             else self.sim_params.end_session
 
         if exchange_name is None:
-            exchange = self.exchanges.values()[0]
+            exchange = list(self.exchanges.values())[0]
         else:
             exchange = self.exchanges[exchange_name]
 
+        data_frequency = self.data_frequency \
+            if self.sim_params.arena == 'backtest' else None
         return self.asset_finder.lookup_symbol(
             symbol=symbol_str,
             exchange=exchange,
+            data_frequency=data_frequency,
             as_of_date=_lookup_date
         )
 
@@ -127,7 +205,13 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         """
         Creates a dictionary representing the state of the tracker.
 
+        Parameters
+        ----------
+        start_dt: datetime
+        end_dt: datetime
 
+        Notes
+        -----
         I rewrote this in an attempt to better control the stats.
         I don't want things to happen magically through complex logic
         pertaining to backtesting.
@@ -176,17 +260,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
-        stats['transactions'] = dict()
+        stats['transactions'] = []
         for date in period.processed_transactions:
             if start_dt <= date < end_dt:
-                stats['transactions'][date] = \
-                    period.processed_transactions[date]
+                transactions = period.processed_transactions[date]
+                for t in transactions:
+                    stats['transactions'].append(t.to_dict())
 
-        stats['orders'] = dict()
+        stats['orders'] = []
         for date in period.orders_by_modified:
             if start_dt <= date < end_dt:
-                stats['orders'][date] = \
-                    period.orders_by_modified[date]
+                orders = period.orders_by_modified[date]
+                for order in orders:
+                    stats['orders'].append(orders[order].to_dict())
 
         return stats
 
@@ -195,58 +281,56 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
-        self.blotter = ExchangeBlotter(
-            data_frequency=self.data_frequency,
-            # Default to NeverCancel in catalyst
-            cancel_policy=self.cancel_policy,
-        )
+        self.frame_stats = list()
         log.info('initialized trading algorithm in backtest mode')
 
-    def _calculate_order(self, asset, amount,
-                         limit_price=None, stop_price=None, style=None):
-        # Raises a ZiplineError if invalid parameters are detected.
-        self.validate_order_params(asset,
-                                   amount,
-                                   limit_price,
-                                   stop_price,
-                                   style)
-
-        # Convert deprecated limit_price and stop_price parameters to use
-        # ExecutionStyle objects.
-        style = self.__convert_order_params_for_blotter(limit_price,
-                                                        stop_price,
-                                                        style)
-        return amount, style
-
-    @staticmethod
-    def __convert_order_params_for_blotter(limit_price, stop_price, style):
-        """
-        Helper method for converting deprecated limit_price and stop_price
-        arguments into ExecutionStyle instances.
-
-        This function assumes that either style == None or (limit_price,
-        stop_price) == (None, None).
-        """
-        if style:
-            assert (limit_price, stop_price) == (None, None)
-            return style
-        if limit_price and stop_price:
-            return ExchangeStopLimitOrder(limit_price, stop_price)
-        if limit_price:
-            return ExchangeLimitOrder(limit_price)
-        if stop_price:
-            return ExchangeStopOrder(stop_price)
+    def is_last_frame_of_day(self, data):
+        # TODO: adjust here to support more intervals
+        next_frame_dt = data.current_dt + timedelta(minutes=1)
+        if next_frame_dt.date() > data.current_dt.date():
+            return True
         else:
-            return MarketOrder()
+            return False
+
+    def handle_data(self, data):
+        super(ExchangeTradingAlgorithmBacktest, self).handle_data(data)
+
+        if self.data_frequency == 'minute':
+            frame_stats = self.prepare_period_stats(
+                data.current_dt, data.current_dt + timedelta(minutes=1)
+            )
+            self.frame_stats.append(frame_stats)
+
+        self.current_day = data.current_dt.floor('1D')
+
+    def _create_stats_df(self):
+        stats = pd.DataFrame(self.frame_stats)
+        stats.set_index('period_close', inplace=True, drop=False)
+        return stats
+
+    def analyze(self, perf):
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
+        super(ExchangeTradingAlgorithmBacktest, self).analyze(stats)
+
+    def run(self, data=None, overwrite_sim_params=True):
+        perf = super(ExchangeTradingAlgorithmBacktest, self).run(
+            data, overwrite_sim_params
+        )
+        # Rebuilding the stats to support minute data
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
+        return stats
 
 
 class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
+        self.stats_output = kwargs.pop('stats_output', None)
 
         self._clock = None
-        self.minute_stats = deque(maxlen=60)
+        self.frame_stats = list()
 
         self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
 
@@ -264,38 +348,27 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.retry_order = 2
         self.retry_delay = 5
 
-        self.stats_minutes = 5
+        self.stats_minutes = 10
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
-        # TODO: fix precision before re-enabling
-        # self._create_minute_writer()
 
         signal.signal(signal.SIGINT, self.signal_handler)
 
         log.info('initialized trading algorithm in live mode')
 
-    def _create_minute_writer(self):
-        root = get_exchange_minute_writer_root(self.exchange.name)
-        filename = os.path.join(root, 'metadata.json')
-
-        if os.path.isfile(filename):
-            writer = BcolzMinuteBarWriter.open(
-                root, self.sim_params.end_session)
-        else:
-            # TODO: need to be able to write more precise numbers
-            writer = BcolzMinuteBarWriter(
-                rootdir=root,
-                calendar=self.trading_calendar,
-                minutes_per_day=1440,
-                start_session=self.sim_params.start_session,
-                end_session=self.sim_params.end_session,
-                write_metadata=True
-            )
-
-        self.exchange.minute_writer = writer
-        self.exchange.minute_reader = BcolzMinuteBarReader(root)
-
     def signal_handler(self, signal, frame):
+        """
+        Handles the keyboard interruption signal.
+
+        Parameters
+        ----------
+        signal
+        frame
+
+        Returns
+        -------
+
+        """
         self.is_running = False
 
         if self._analyze is None:
@@ -343,7 +416,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         # This method is taken from TradingAlgorithm.
         # The clock has been replaced to use RealtimeClock
-        # TODO: should we apply a time skew? not sure to understand the utility.
+        # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
         if self.live_graph:
@@ -381,43 +454,83 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         return self.trading_client.transform()
 
     def updated_portfolio(self):
-        """
-        We skip the entire performance tracker business and update the
-        portfolio directly.
-        :return:
-        """
-        # TODO: build cumulative portfolio
         return self.perf_tracker.get_portfolio(False)
 
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def _synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self, attempt_index=0):
+        """
+        Synchronizes the portfolio tracked by the algorithm to refresh
+        its current value.
+
+        This includes updating the last_sale_price of all tracked
+        positions, returning the available cash, and raising error
+        if the data goes out of sync.
+
+        Parameters
+        ----------
+        attempt_index: int
+
+        Returns
+        -------
+        float
+            The amount of base currency available for trading.
+
+        float
+            The total value of all tracked positions.
+
+        """
+        tracker = self.perf_tracker.position_tracker
+        total_cash = 0.0
+        total_positions_value = 0.0
+
         try:
+            # Position keys correspond to assets
+            positions = self.portfolio.positions
+            assets = list(positions)
+            exchange_assets = group_assets_by_exchange(assets)
             for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
+                assets = exchange_assets[exchange_name] \
+                    if exchange_name in exchange_assets else []
 
-                exchange.synchronize_portfolio()
+                exchange_positions = \
+                    [positions[asset] for asset in assets]
 
-                # Applying the updated last_sales_price to the positions
-                # in the performance tracker. This seems a bit redundant
-                # but it will make sense when we have multiple exchange portfolios
-                # feeding into the same performance tracker.
-                tracker = self.perf_tracker.todays_performance.position_tracker
-                for asset in exchange.portfolio.positions:
-                    position = exchange.portfolio.positions[asset]
+                check_cash = (not self.simulate_orders)
+
+                exchange = self.exchanges[exchange_name]  # Type: Exchange
+                cash, positions_value = exchange.calculate_totals(
+                    positions=exchange_positions,
+                    check_cash=check_cash,
+                )
+                total_positions_value += positions_value
+
+                if cash is not None:
+                    total_cash += cash
+
+                for position in exchange_positions:
                     tracker.update_position(
-                        asset=asset,
+                        asset=position.asset,
                         last_sale_date=position.last_sale_date,
                         last_sale_price=position.last_sale_price
                     )
+
+            if cash is None:
+                total_cash = self.portfolio.cash
+
+            elif total_cash < self.portfolio.cash:
+                raise ValueError('Cash on exchanges is lower than the algo.')
+
+            return total_cash, total_positions_value
+
         except ExchangeRequestError as e:
             log.warn(
                 'update portfolio attempt {}: {}'.format(attempt_index, e)
             )
             if attempt_index < self.retry_synchronize_portfolio:
                 sleep(self.retry_delay)
-                self._synchronize_portfolio(attempt_index + 1)
+                return self.synchronize_portfolio(attempt_index + 1)
             else:
                 raise ExchangePortfolioDataError(
                     data_type='update-portfolio',
@@ -425,31 +538,18 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                     error=e
                 )
 
-    def _check_open_orders(self, attempt_index=0):
-        try:
-            orders = list()
-            for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
-                exchange_orders = exchange.check_open_orders()
-
-                orders += exchange_orders
-
-            return orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self._check_open_orders(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
-
     def add_pnl_stats(self, period_stats):
+        """
+        Save p&l stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         starting = period_stats['starting_cash']
         current = period_stats['portfolio_value']
         appreciation = (current / starting) - 1
@@ -466,6 +566,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
 
     def add_custom_signals_stats(self, period_stats):
+        """
+        Save custom signals stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         log.debug('adding custom signals stats: {}'.format(self.recorded_vars))
         df = pd.DataFrame(
             data=[self.recorded_vars],
@@ -477,6 +588,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      self.custom_signals_stats)
 
     def add_exposure_stats(self, period_stats):
+        """
+        Save exposure stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         data = dict(
             long_exposure=period_stats['long_exposure'],
             base_currency=period_stats['ending_cash']
@@ -489,19 +611,39 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.exposure_stats = pd.concat([self.exposure_stats, df])
 
-        save_algo_df(self.algo_namespace, 'exposure_stats',
-                     self.exposure_stats)
+        save_algo_df(
+            self.algo_namespace, 'exposure_stats', self.exposure_stats
+        )
 
     def handle_data(self, data):
+        """
+        Wrapper around the handle_data method of each algo.
+
+        Parameters
+        ----------
+        data
+
+        """
         if not self.is_running:
             return
 
-        self._synchronize_portfolio()
+        # Resetting the frame stats every day to minimize memory footprint
+        today = data.current_dt.floor('1D')
+        if self.current_day is not None and today > self.current_day:
+            self.frame_stats = list()
 
-        transactions = self._check_open_orders()
-        for transaction in transactions:
-            self.perf_tracker.process_transaction(transaction)
+        new_transactions, new_commissions, closed_orders = \
+            self.blotter.get_transactions(data)
 
+        if len(new_transactions) > 0:
+            self.perf_tracker.update_performance()
+
+        cash, positions_value = self.synchronize_portfolio()
+        log.info(
+            'got totals from exchanges, cash: {} positions: {}'.format(
+                cash, positions_value
+            )
+        )
         if self._handle_data:
             self._handle_data(self, data)
 
@@ -511,51 +653,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.validate_account_controls()
 
         try:
-            # Since the clock runs 24/7, I trying to disable the daily
-            # Performance tracker and keep only minute and cumulative
-            self.perf_tracker.update_performance()
-
-            minute_stats = self.prepare_period_stats(
-                data.current_dt, data.current_dt + timedelta(minutes=1))
-
-            # Saving the last hour in memory
-            self.minute_stats.append(minute_stats)
-
-            self.add_pnl_stats(minute_stats)
-            if self.recorded_vars:
-                self.add_custom_signals_stats(minute_stats)
-                recorded_cols = self.recorded_vars.keys()
-            else:
-                recorded_cols = None
-
-            self.add_exposure_stats(minute_stats)
-
-            print_df = pd.DataFrame(list(self.minute_stats))
-            log.info(
-                'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
-                    stats_minutes=self.stats_minutes,
-                    stats=get_pretty_stats(
-                        stats_df=print_df,
-                        recorded_cols=recorded_cols,
-                        num_rows=self.stats_minutes
-                    )
-                ))
-
-            today = pd.to_datetime('today', utc=True)
-            daily_stats = self.prepare_period_stats(
-                start_dt=today,
-                end_dt=pd.Timestamp.utcnow()
-            )
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key=today.strftime('%Y-%m-%d'),
-                obj=daily_stats,
-                rel_path='daily_perf'
-            )
-
+            self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
+        # TODO: pickle does not seem to work in python 3
         try:
             save_algo_object(
                 algo_name=self.algo_namespace,
@@ -565,92 +667,85 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         except Exception as e:
             log.warn('unable to save minute perfs to disk: {}'.format(e))
 
-        try:
-            for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
-                save_algo_object(
-                    algo_name=self.algo_namespace,
-                    key='portfolio_{}'.format(exchange_name),
-                    obj=exchange.portfolio
-                )
-        except Exception as e:
-            log.warn('unable to save portfolio to disk: {}'.format(e))
+        self.current_day = data.current_dt.floor('1D')
 
-    def _order(self,
-               asset,
-               amount,
-               limit_price=None,
-               stop_price=None,
-               style=None,
-               attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(asset, amount, limit_price,
-                                  stop_price,
-                                  style)
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-                return self._order(
-                    asset, amount, limit_price, stop_price, style,
-                    attempt_index + 1)
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
+    def _process_stats(self, data):
+        today = data.current_dt.floor('1D')
 
-    @api_method
-    @disallowed_in_before_trading_start(OrderInBeforeTradingStart())
-    @expect_types(asset=TradingPair)
-    def order(self,
-              asset,
-              amount,
-              limit_price=None,
-              stop_price=None,
-              style=None):
-        """
-        We use the exchange specific portfolio to place orders.
-        The cumulative portfolio does not contain open orders but exchange
-        portfolios do.
+        # Since the clock runs 24/7, I trying to disable the daily
+        # Performance tracker and keep only minute and cumulative
+        self.perf_tracker.update_performance()
 
-        :param asset: TradingPair
-        :param amount: float
-        :param limit_price: float
-        :param stop_price: float
-        :param style: Style
-        :return order: Order
-            The catalyst order object or None
-        """
+        frame_stats = self.prepare_period_stats(
+            data.current_dt, data.current_dt + timedelta(minutes=1))
 
-        amount, style = self._calculate_order(asset, amount,
-                                              limit_price, stop_price,
-                                              style)
+        # Saving the last hour in memory
+        self.frame_stats.append(frame_stats)
 
-        order_id = self._order(asset, amount, limit_price, stop_price, style)
+        self.add_pnl_stats(frame_stats)
+        if self.recorded_vars:
+            self.add_custom_signals_stats(frame_stats)
+            recorded_cols = list(self.recorded_vars.keys())
 
-        exchange = self.exchanges[asset.exchange]
-        exchange_portfolio = exchange.portfolio
-        if order_id is not None:
-
-            if order_id in exchange_portfolio.open_orders:
-                order = exchange_portfolio.open_orders[order_id]
-                self.perf_tracker.process_order(order)
-                return order
-
-            else:
-                raise OrphanOrderError(
-                    order_id=order_id,
-                    exchange=exchange.name
-                )
         else:
-            log.warn('unable to order {} {} on exchange {}'.format(
-                amount, asset.symbol, asset.exchange))
-            return None
+            recorded_cols = None
+
+        self.add_exposure_stats(frame_stats)
+
+        log.info(
+            'statistics for the last {stats_minutes} minutes:\n'
+            '{stats}'.format(
+                stats_minutes=self.stats_minutes,
+                stats=get_pretty_stats(
+                    stats=self.frame_stats,
+                    recorded_cols=recorded_cols,
+                    num_rows=self.stats_minutes
+                )
+            ))
+
+        # Saving the daily stats in a format usable for performance
+        # analysis.
+        daily_stats = self.prepare_period_stats(
+            start_dt=today,
+            end_dt=data.current_dt
+        )
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=daily_stats,
+            rel_path='daily_perf'
+        )
+
+        return recorded_cols
+
+    def _save_stats_csv(self, recorded_cols):
+        # Writing the stats output
+        csv_bytes = None
+        try:
+            csv_bytes = stats_to_algo_folder(
+                stats=self.frame_stats,
+                algo_namespace=self.algo_namespace,
+                recorded_cols=recorded_cols,
+            )
+        except Exception as e:
+            log.warn('unable save stats locally: {}'.format(e))
+
+        try:
+            if self.stats_output is not None:
+                if 's3://' in self.stats_output:
+                    stats_to_s3(
+                        uri=self.stats_output,
+                        stats=self.frame_stats,
+                        algo_namespace=self.algo_namespace,
+                        recorded_cols=recorded_cols,
+                        bytes_to_write=csv_bytes
+                    )
+                else:
+                    raise ValueError(
+                        'Only S3 stats output is supported for now.'
+                    )
+        except Exception as e:
+            log.warn('unable save stats externally: {}'.format(e))
 
     @api_method
     def batch_market_order(self, share_counts):
@@ -688,15 +783,53 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                         'get_open_orders. Use `asset` instead.')
     @api_method
     def get_open_orders(self, asset=None):
+        """Retrieve all of the current open orders.
+
+        Parameters
+        ----------
+        asset : Asset
+            If passed and not None, return only the open orders for the given
+            asset instead of all open orders.
+
+        Returns
+        -------
+        open_orders : dict[list[Order]] or list[Order]
+            If no asset is passed this will return a dict mapping Assets
+            to a list containing all the open orders for the asset.
+            If an asset is passed then this will return a list of the open
+            orders for this asset.
+        """
         return self._get_open_orders(asset)
 
     @api_method
     def get_order(self, order_id, exchange_name):
+        """Lookup an order based on the order id returned from one of the
+        order functions.
+
+        Parameters
+        ----------
+        order_id : str
+            The unique identifier for the order.
+
+        Returns
+        -------
+        order : Order
+            The order object.
+        execution_price: float
+            The execution price per share of the order
+        """
         exchange = self.exchanges[exchange_name]
         return exchange.get_order(order_id)
 
     @api_method
     def cancel_order(self, order_param, exchange_name):
+        """Cancel an open order.
+
+        Parameters
+        ----------
+        order_param : str or Order
+            The order_id or order object to cancel.
+        """
         exchange = self.exchanges[exchange_name]
 
         order_id = order_param

catalyst/exchange/exchange_bcolz.py

@@ -16,7 +16,7 @@ class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
             end_session = end_session.floor('1d')
 
         minutes_per_day = 1440 if self._data_frequency == 'minute' else 1
-        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 1000000)
+        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000)
         calendar = get_calendar('OPEN')
 
         super(BcolzExchangeBarWriter, self) \
@@ -39,17 +39,25 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
         return self._data_frequency
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
+        """
+        Parameters
+        ----------
+        fields : list of str
+           'open', 'high', 'low', 'close', or 'volume'
+        start_dt: Timestamp
+           Beginning of the window range.
+        end_dt: Timestamp
+           End of the window range.
+        sids : list of int
+           The asset identifiers in the window.
 
-        # if self._data_frequency == 'minute':
-        #     return super(BcolzExchangeBarReader, self) \
-        #         .load_raw_arrays(fields, start_dt, end_dt, sids)
-        #
-        # else:
-        #     return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
-
-        return self._load_raw_arrays(fields, start_dt, end_dt, sids)
-
-    def _load_raw_arrays(self, fields, start_dt, end_dt, sids):
+        Returns
+        -------
+        list of np.ndarray
+            A list with an entry per field of ndarrays with shape
+            (minutes in range, sids) with a dtype of float64, containing the
+            values for the respective field over start and end dt range.
+        """
         start_idx = self._find_position_of_minute(start_dt)
         end_idx = self._find_position_of_minute(end_dt)
 
@@ -79,8 +87,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
                 if mask is None:
                     mask = a != 0
 
+                inverse_ratio = self._ohlc_ratio_inverse_for_sid(sid)
                 out[:len(mask), i][mask] = (
-                    a[mask] * self._ohlc_ratio_inverse_for_sid(sid)
+                    a[mask] * inverse_ratio
                 )
 
             if field in fields:

catalyst/exchange/exchange_blotter.py

@@ -1,21 +1,21 @@
+from time import sleep
+
+import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange_errors import ExchangeRequestError, \
+    ExchangePortfolioDataError, ExchangeTransactionError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
+from catalyst.finance.order import ORDER_STATUS, Order
 from catalyst.finance.slippage import SlippageModel
-from catalyst.finance.transaction import Transaction
+from catalyst.finance.transaction import create_transaction, Transaction
+from catalyst.utils.input_validation import expect_types
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
-# It seems like we need to accept greater slippage risk in cryptos
-# Orders won't often close at Equity levels.
-# TODO: consider adjusting dynamically based on trading pair
-DEFAULT_SLIPPAGE_SPREAD = 0.02
-DEFAULT_MAKER_FEE = 0.001
-DEFAULT_TAKER_FEE = 0.002
-
 
 class TradingPairFeeSchedule(CommissionModel):
     """
@@ -23,23 +23,24 @@ class TradingPairFeeSchedule(CommissionModel):
 
     Parameters
     ----------
-    fee : float, optional
-        The percentage fee.
+    maker : float, optional
+        The percentage maker fee.
+
+    taker: float, optional
+        The percentage taker fee.
     """
 
-    def __init__(self,
-                 maker_fee=DEFAULT_MAKER_FEE,
-                 taker_fee=DEFAULT_TAKER_FEE):
-        self.maker_fee = maker_fee
-        self.taker_fee = taker_fee
+    def __init__(self, maker=None, taker=None):
+        self.maker = maker
+        self.taker = taker
 
     def __repr__(self):
         return (
-            '{class_name}(maker_fee={maker_fee}, '
-            'taker_fee={taker_fee})'.format(
+            '{class_name}(maker={maker}, '
+            'taker={taker})'.format(
                 class_name=self.__class__.__name__,
-                maker_fee=self.maker_fee,
-                taker_fee=self.taker_fee,
+                maker=self.maker,
+                taker=self.taker,
             )
         )
 
@@ -47,16 +48,25 @@ class TradingPairFeeSchedule(CommissionModel):
         """
         Calculate the final fee based on the order parameters.
 
-        :param order:
-        :param transaction:
+        :param order: Order
+        :param transaction: Transaction
 
         :return float:
             The total commission.
         """
         cost = abs(transaction.amount) * transaction.price
 
+        asset = order.asset
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+
+        multiplier = maker \
+            if ((order.amount > 0 and order.limit < transaction.price)
+                or (order.amount < 0 and order.limit > transaction.price)) \
+               and order.limit_reached else taker
+
         # Assuming just the taker fee for now
-        fee = cost * self.taker_fee
+        fee = cost * multiplier
         return fee
 
 
@@ -70,7 +80,7 @@ class TradingPairFixedSlippage(SlippageModel):
         spread / 2 will be added to buys and subtracted from sells.
     """
 
-    def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
+    def __init__(self, spread=0.0001):
         super(TradingPairFixedSlippage, self).__init__()
         self.spread = spread
 
@@ -97,12 +107,8 @@ class TradingPairFixedSlippage(SlippageModel):
 
             execution_price, execution_volume = self.process_order(data, order)
 
-            transaction = Transaction(
-                asset=order.asset,
-                amount=abs(execution_volume),
-                dt=dt,
-                price=execution_price,
-                order_id=order.id
+            transaction = create_transaction(
+                order, dt, execution_price, execution_volume
             )
 
             self._volume_for_bar += abs(transaction.amount)
@@ -125,6 +131,14 @@ class TradingPairFixedSlippage(SlippageModel):
 
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
+        self.simulate_orders = kwargs.pop('simulate_orders', False)
+
+        self.exchanges = kwargs.pop('exchanges', None)
+        if not self.exchanges:
+            raise ValueError(
+                'ExchangeBlotter must have an `exchanges` attribute.'
+            )
+
         super(ExchangeBlotter, self).__init__(*args, **kwargs)
 
         # Using the equity models for now
@@ -136,3 +150,148 @@ class ExchangeBlotter(Blotter):
         self.commission_models = {
             TradingPair: TradingPairFeeSchedule()
         }
+
+        self.retry_delay = 5
+        self.retry_check_open_orders = 5
+
+    def exchange_order(self, asset, amount, style=None, attempt_index=0):
+        try:
+            exchange = self.exchanges[asset.exchange]
+            return exchange.order(
+                asset, amount, style
+            )
+        except ExchangeRequestError as e:
+            log.warn(
+                'order attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_order:
+                sleep(self.retry_delay)
+
+                return self.exchange_order(
+                    asset, amount, style, attempt_index + 1
+                )
+            else:
+                raise ExchangeTransactionError(
+                    transaction_type='order',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    @expect_types(asset=TradingPair)
+    def order(self, asset, amount, style, order_id=None):
+        log.debug('ordering {} {}'.format(amount, asset.symbol))
+        if amount == 0:
+            log.warn('skipping 0 amount orders')
+            return None
+
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).order(
+                asset, amount, style, order_id
+            )
+
+        else:
+            order = self.exchange_order(
+                asset, amount, style
+            )
+
+            self.open_orders[order.asset].append(order)
+            self.orders[order.id] = order
+            self.new_orders.append(order)
+
+            return order.id
+
+    def check_open_orders(self):
+        """
+        Loop through the list of open orders in the Portfolio object.
+        For each executed order found, create a transaction and apply to the
+        Portfolio.
+
+        Returns
+        -------
+        list[Transaction]
+
+        """
+        for asset in self.open_orders:
+            exchange = self.exchanges[asset.exchange]
+
+            for order in self.open_orders[asset]:
+                log.debug('found open order: {}'.format(order.id))
+
+                new_order, executed_price = exchange.get_order(order.id, asset)
+                log.debug(
+                    'got updated order {} {}'.format(
+                        new_order, executed_price
+                    )
+                )
+                order.status = new_order.status
+
+                if order.status == ORDER_STATUS.FILLED:
+                    order.commission = new_order.commission
+                    if order.amount != new_order.amount:
+                        log.warn(
+                            'executed order amount {} differs '
+                            'from original'.format(
+                                new_order.amount, order.amount
+                            )
+                        )
+                        order.amount = new_order.amount
+
+                    transaction = Transaction(
+                        asset=order.asset,
+                        amount=order.amount,
+                        dt=pd.Timestamp.utcnow(),
+                        price=executed_price,
+                        order_id=order.id,
+                        commission=order.commission
+                    )
+                    yield order, transaction
+
+                elif order.status == ORDER_STATUS.CANCELLED:
+                    yield order, None
+
+                else:
+                    delta = pd.Timestamp.utcnow() - order.dt
+                    log.info(
+                        'order {order_id} still open after {delta}'.format(
+                            order_id=order.id,
+                            delta=delta
+                        )
+                    )
+
+    def get_exchange_transactions(self, attempt_index=0):
+        closed_orders = []
+        transactions = []
+        commissions = []
+
+        try:
+            for order, txn in self.check_open_orders():
+                order.dt = txn.dt
+
+                transactions.append(txn)
+
+                if not order.open:
+                    closed_orders.append(order)
+
+            return transactions, commissions, closed_orders
+
+        except ExchangeRequestError as e:
+            log.warn(
+                'check open orders attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_check_open_orders:
+                sleep(self.retry_delay)
+                return self.get_exchange_transactions(attempt_index + 1)
+
+            else:
+                raise ExchangePortfolioDataError(
+                    data_type='order-status',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def get_transactions(self, bar_data):
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).get_transactions(bar_data)
+
+        else:
+            return self.get_exchange_transactions()

catalyst/exchange/exchange_data_portal.py

@@ -1,16 +1,3 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import abc
 from time import sleep
 
@@ -19,13 +6,15 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
+from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
     PricingDataNotLoadedError)
+from catalyst.exchange.exchange_utils import get_frequency, \
+    resample_history_df, group_assets_by_exchange
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
@@ -33,7 +22,6 @@ log = Logger('DataPortalExchange', level=LOG_LEVEL)
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
 
-        self.exchanges = kwargs.pop('exchanges', None)
         # TODO: put somewhere accessible by each algo
         self.retry_get_history_window = 5
         self.retry_get_spot_value = 5
@@ -51,21 +39,14 @@ class DataPortalExchangeBase(DataPortal):
                             ffill=True,
                             attempt_index=0):
         try:
-            exchange_assets = dict()
-            for asset in assets:
-                if asset.exchange not in exchange_assets:
-                    exchange_assets[asset.exchange] = list()
-
-                exchange_assets[asset.exchange].append(asset)
-
+            exchange_assets = group_assets_by_exchange(assets)
             if len(exchange_assets) > 1:
                 df_list = []
                 for exchange_name in exchange_assets:
-                    exchange = self.exchanges[exchange_name]
                     assets = exchange_assets[exchange_name]
 
                     df_exchange = self.get_exchange_history_window(
-                        exchange,
+                        exchange_name,
                         assets,
                         end_dt,
                         bar_count,
@@ -80,9 +61,9 @@ class DataPortalExchangeBase(DataPortal):
                 return pd.concat(df_list)
 
             else:
-                exchange = self.exchanges[exchange_assets.keys()[0]]
+                exchange_name = list(exchange_assets.keys())[0]
                 return self.get_exchange_history_window(
-                    exchange,
+                    exchange_name,
                     assets,
                     end_dt,
                     bar_count,
@@ -134,7 +115,7 @@ class DataPortalExchangeBase(DataPortal):
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -148,9 +129,8 @@ class DataPortalExchangeBase(DataPortal):
                         attempt_index=0):
         try:
             if isinstance(assets, TradingPair):
-                exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
-                    exchange, [assets], field, dt, data_frequency)
+                    assets.exchange, [assets], field, dt, data_frequency)
 
                 if not spot_values:
                     return np.nan
@@ -165,18 +145,17 @@ class DataPortalExchangeBase(DataPortal):
 
                     exchange_assets[asset.exchange].append(asset)
 
-                if len(exchange_assets.keys()) == 1:
-                    exchange = self.exchanges[exchange_assets.keys()[0]]
+                if len(list(exchange_assets.keys())) == 1:
+                    exchange_name = list(exchange_assets.keys())[0]
                     return self.get_exchange_spot_value(
-                        exchange, assets, field, dt, data_frequency)
+                        exchange_name, assets, field, dt, data_frequency)
 
                 else:
                     spot_values = []
                     for exchange_name in exchange_assets:
-                        exchange = self.exchanges[exchange_name]
                         assets = exchange_assets[exchange_name]
                         exchange_spot_values = self.get_exchange_spot_value(
-                            exchange,
+                            exchange_name,
                             assets,
                             field,
                             dt,
@@ -211,7 +190,7 @@ class DataPortalExchangeBase(DataPortal):
         return self._get_spot_value(assets, field, dt, data_frequency)
 
     @abc.abstractmethod
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
         return
 
@@ -226,10 +205,11 @@ class DataPortalExchangeBase(DataPortal):
 
 class DataPortalExchangeLive(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchanges = kwargs.pop('exchanges', None)
         super(DataPortalExchangeLive, self).__init__(*args, **kwargs)
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -237,6 +217,27 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange.
+
+        Parameters
+        ----------
+        exchange_name: Exchange
+        assets: list[TradingPair]
+        end_dt: datetime
+        bar_count: int
+        frequency: str
+        field: str
+        data_frequency: str
+        ffill: bool
+
+        Returns
+        -------
+        DataFrame
+
+        """
+        exchange = self.exchanges[exchange_name]
+
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -244,11 +245,28 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
             frequency,
             field,
             data_frequency,
-            ffill)
+            False)
         return df
 
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
+        """
+        A spot value for the exchange.
+
+        Parameters
+        ----------
+        exchange_name: str
+        assets: list[TradingPair]
+        field: str
+        dt: datetime
+        data_frequency: str
+
+        Returns
+        -------
+        float
+
+        """
+        exchange = self.exchanges[exchange_name]
         exchange_spot_values = exchange.get_spot_value(
             assets, field, dt, data_frequency)
 
@@ -257,16 +275,16 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
 class DataPortalExchangeBacktest(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchange_names = kwargs.pop('exchange_names', None)
+
         super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs)
 
         self.exchange_bundles = dict()
-
         self.history_loaders = dict()
         self.minute_history_loaders = dict()
 
-        for exchange_name in self.exchanges:
-            exchange = self.exchanges[exchange_name]
-            self.exchange_bundles[exchange_name] = ExchangeBundle(exchange)
+        for name in self.exchange_names:
+            self.exchange_bundles[name] = ExchangeBundle(name)
 
     def _get_first_trading_day(self, assets):
         first_date = None
@@ -276,7 +294,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         return first_date
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -287,57 +305,98 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         """
         Fetching price history window from the exchange bundle.
 
-        Using a try... except approach to minimize reads most of the time,
-        when the data exists.
+        Parameters
+        ----------
+        exchange: Exchange
+        assets: list[TradingPair]
+        end_dt: datetime
+        bar_count: int
+        frequency: str
+        field: str
+        data_frequency: str
+        ffill: bool
+
+        Returns
+        -------
+        DataFrame
 
-        :param exchange:
-        :param assets:
-        :param end_dt:
-        :param bar_count:
-        :param frequency:
-        :param field:
-        :param data_frequency:
-        :param ffill:
-        :return:
         """
+        bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
+
+        freq, candle_size, unit, adj_data_frequency = get_frequency(
+            frequency, data_frequency
+        )
+        adj_bar_count = candle_size * bar_count
+        trailing_bar_count = candle_size - 1
+
+        if data_frequency == 'minute' and adj_data_frequency == 'daily':
+            end_dt = end_dt.floor('1D')
 
-        bundle = self.exchange_bundles[exchange.name]
         series = bundle.get_history_window_series_and_load(
             assets=assets,
             end_dt=end_dt,
-            bar_count=bar_count,
+            bar_count=adj_bar_count,
             field=field,
-            data_frequency=data_frequency
+            data_frequency=adj_data_frequency,
+            algo_end_dt=self._last_available_session,
+            trailing_bar_count=trailing_bar_count
         )
-        return pd.DataFrame(series)
 
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
-                                data_frequency):
-        bundle = self.exchange_bundles[exchange.name]
+        df = resample_history_df(pd.DataFrame(series), freq, field)
+        return df
 
+    def get_exchange_spot_value(self,
+                                exchange_name,
+                                assets,
+                                field,
+                                dt,
+                                data_frequency
+                                ):
+        """
+        A spot value for the exchange bundle. Try to ingest data if not in
+        the bundle.
+
+        Parameters
+        ----------
+        exchange_name: str
+        assets: list[TradingPair]
+        field: str
+        dt: datetime
+        data_frequency: str
+
+        Returns
+        -------
+        float
+
+        """
+        bundle = self.exchange_bundles[exchange_name]
         if data_frequency == 'daily':
             dt = dt.floor('1D')
         else:
             dt = dt.floor('1 min')
 
-        try:
-            return bundle.get_spot_values(assets, field, dt, data_frequency)
-
-        except PricingDataNotLoadedError:
-            log.info(
-                'pricing data for {symbol} not found on {dt}'
-                ', updating the bundles.'.format(
-                    symbol=[asset.symbol for asset in assets],
-                    dt=dt
+        if AUTO_INGEST:
+            try:
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency
                 )
-            )
-            bundle.ingest_assets(
-                assets=assets,
-                start_dt=self._first_trading_day,
-                end_dt=self._last_available_session,
-                data_frequency=data_frequency,
-                show_progress=True
-            )
-            return bundle.get_spot_values(
-                assets, field, dt, data_frequency, True
-            )
+            except PricingDataNotLoadedError:
+                log.info(
+                    'pricing data for {symbol} not found on {dt}'
+                    ', updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        dt=dt
+                    )
+                )
+                bundle.ingest_assets(
+                    assets=assets,
+                    start_dt=self._first_trading_day,
+                    end_dt=self._last_available_session,
+                    data_frequency=data_frequency,
+                    show_progress=True
+                )
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency, True
+                )
+        else:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -6,12 +6,12 @@ from catalyst.errors import ZiplineError
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, 
-                   ExchangeAuthEmpty ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange,
+                   ExchangeAuthEmpty]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
-        print "Error traceback: {1} (line {2})\n" \
-              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue)
+        print("Error traceback: {1} (line {2})\n"
+              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
     else:
         sys.__excepthook__(exctype, excvalue, exctraceback)
 
@@ -86,6 +86,14 @@ class AlgoPickleNotFound(ZiplineError):
     ).strip()
 
 
+class InvalidHistoryFrequencyAlias(ZiplineError):
+    msg = (
+        'Invalid frequency alias {freq}. Valid suffixes are M (minute) '
+        'and D (day). For example, these aliases would be valid '
+        '1M, 5M, 1D.'
+    ).strip()
+
+
 class InvalidHistoryFrequencyError(ZiplineError):
     msg = (
         'Frequency {frequency} not supported by the exchange.'
@@ -135,7 +143,8 @@ class OrphanOrderError(ZiplineError):
 
 class OrphanOrderReverseError(ZiplineError):
     msg = (
-        'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
+        'Order {order_id} tracked by algorithm, but not found in exchange '
+        '{exchange}.'
     ).strip()
 
 
@@ -198,23 +207,65 @@ class EmptyValuesInBundleError(ZiplineError):
 
 class PricingDataBeforeTradingError(ZiplineError):
     msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
-           'starts on {first_trading_day}, but you are either trying to trade or '
-           'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
+           'starts on {first_trading_day}, but you are either trying to trade '
+           'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
+           ).strip()
 
 
 class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Pricing data {field} for trading pairs {symbols} trading on '
-           'exchange {exchange} since {first_trading_day} is unavailable. '
-           'The bundle data is either out-of-date or has not been loaded yet. '
-           'Please ingest data using the command '
-           '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. '
-           'See catalyst documentation for details.').strip()
+    msg = ('Missing data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]'
+           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
+           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
+           'for details.').strip()
+
+
+class PricingDataValueError(ZiplineError):
+    msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
+           '[{start_dt} - {end_dt}]: {error}').strip()
+
+
+class DataCorruptionError(ZiplineError):
+    msg = ('Unable to validate data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]. The data is either corrupted or '
+           'unavailable. Please try deleting this bundle:'
+           '\n`catalyst clean-exchange -x {exchange}\n'
+           'Then, ingest the data again. Please contact the Catalyst team if '
+           'the issue persists.').strip()
 
 
 class ApiCandlesError(ZiplineError):
     msg = ('Unable to fetch candles from the remote API: {error}.').strip()
 
+
 class NoDataAvailableOnExchange(ZiplineError):
-    msg = ('Requested data for trading pair {symbol} is not available on exchange {exchange} '
-           'in `{data_frequency}` frequency at this time. '
-           'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+    msg = (
+        'Requested data for trading pair {symbol} is not available on '
+        'exchange {exchange} '
+        'in `{data_frequency}` frequency at this time. '
+        'Check `http://enigma.co/catalyst/status` for market coverage.'
+    ).strip()
+
+
+class NoValueForField(ZiplineError):
+    msg = ('Value not found for field: {field}.').strip()
+
+
+class OrderTypeNotSupported(ZiplineError):
+    msg = (
+        'Order type `{order_type}` not currencly supported by Catalyst. '
+        'Please use `limit` or `market` orders only.').strip()
+
+
+class NotEnoughCapitalError(ZiplineError):
+    msg = (
+        'Not enough capital on exchange {exchange} for trading. Each '
+        'exchange should contain at least as much {base_currency} '
+        'as the specified `capital_base`. The current balance {balance} is '
+        'lower than the `capital_base`: {capital_base}').strip()
+
+class LastCandleTooEarlyError(ZiplineError):
+    msg = (
+        'The trade date of the last candle {last_traded} is before the '
+        'specified end date minus one candle {end_dt}. Please verify how '
+        '{exchange} calculates the start date of OHLCV candles.').strip()

catalyst/exchange/exchange_execution.py

@@ -4,9 +4,16 @@ from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
 class ExchangeLimitOrder(LimitOrder):
     def get_limit_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.limit_price
 
@@ -14,9 +21,16 @@ class ExchangeLimitOrder(LimitOrder):
 class ExchangeStopOrder(StopOrder):
     def get_stop_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.stop_price
 
@@ -24,16 +38,30 @@ class ExchangeStopOrder(StopOrder):
 class ExchangeStopLimitOrder(StopLimitOrder):
     def get_limit_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.limit_price
 
     def get_stop_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.stop_price

catalyst/exchange/exchange_portfolio.py

@@ -10,7 +10,8 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 class ExchangePortfolio(Portfolio):
     """
     Since the goal is to support multiple exchanges, it makes sense to
-    include additional stats in the portfolio object.
+    include additional stats in the portfolio object. This fills the role
+    of Blotter and Portfolio in live mode.
 
     Instead of relying on the performance tracker, each exchange portfolio
     tracks its own holding. This offers a separation between tracking an
@@ -29,12 +30,23 @@ class ExchangePortfolio(Portfolio):
         self.positions_value = 0.0
         self.open_orders = dict()
 
-    def calculate_pnl(self):
-        log.debug('calculating pnl')
-
     def create_order(self, order):
+        """
+        Create an open order and store in memory.
+
+        Parameters
+        ----------
+        order: Order
+
+        """
         log.debug('creating order {}'.format(order.id))
-        self.open_orders[order.id] = order
+
+        open_orders = self.open_orders[order.asset] \
+            if order.asset is self.open_orders else []
+
+        open_orders.append(order)
+
+        self.open_orders[order.asset] = open_orders
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
@@ -46,16 +58,40 @@ class ExchangePortfolio(Portfolio):
         order_position.amount += order.amount
         log.debug('open order added to portfolio')
 
+    def _remove_open_order(self, order):
+        try:
+            open_orders = self.open_orders[order.asset]
+            if order in open_orders:
+                open_orders.remove(order)
+
+        except Exception:
+            raise ValueError(
+                'unable to clear order not found in open order list.'
+            )
+
     def execute_order(self, order, transaction):
+        """
+        Update the open orders and positions to apply an executed order.
+
+        Unlike with backtesting, we do not need to add slippage and fees.
+        The executed price includes transaction fees.
+
+        Parameters
+        ----------
+        order: Order
+        transaction: Transaction
+
+        """
         log.debug('executing order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
 
         if order_position is None:
             raise ValueError(
-                'Trying to execute order for a position not held: %s' % order.id
+                'Trying to execute order for a position not held:'
+                ' {}'.format(order.id)
             )
 
         self.capital_used += order.amount * transaction.price
@@ -71,33 +107,17 @@ class ExchangePortfolio(Portfolio):
 
         log.debug('updated portfolio with executed order')
 
-    def execute_transaction(self, transaction):
-        log.debug('executing transaction {}'.format(transaction.order_id))
-
-        order_position = self.positions[transaction.asset] \
-            if transaction.asset in self.positions else None
-
-        if order_position is None:
-            raise ValueError(
-                'Trying to execute transaction for a position not held: %s' % transaction.order_id
-            )
-
-        self.capital_used += transaction.amount * transaction.price
-
-        if transaction.amount > 0:
-            if order_position.cost_basis > 0:
-                order_position.cost_basis = np.average(
-                    [order_position.cost_basis, transaction.price],
-                    weights=[order_position.amount, transaction.amount]
-                )
-            else:
-                order_position.cost_basis = transaction.price
-
-        log.debug('updated portfolio with executed order')
-
     def remove_order(self, order):
+        """
+        Removing an open order.
+
+        Parameters
+        ----------
+        order: Order
+
+        """
         log.info('removing cancelled order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None

catalyst/exchange/exchange_utils.py

@@ -1,21 +1,57 @@
+import hashlib
 import json
 import os
 import pickle
-import urllib
+import re
+import shutil
 from datetime import date, datetime
 
 import pandas as pd
+from catalyst.assets._assets import TradingPair
+from six import string_types
+from six.moves.urllib import request
 
-from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
-    ExchangeSymbolsNotFound
+from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
+    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
-SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
-              '{exchange}/symbols.json'
+
+def get_sid(symbol):
+    """
+    Create a sid by hashing the symbol of a currency pair.
+
+    Parameters
+    ----------
+    symbol: str
+
+    Returns
+    -------
+    int
+        The resulting sid.
+
+    """
+    sid = int(
+        hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+    ) % 10 ** 6
+    return sid
 
 
 def get_exchange_folder(exchange_name, environ=None):
+    """
+    The root path of an exchange folder.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     if not environ:
         environ = os.environ
 
@@ -26,29 +62,89 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
-def get_exchange_symbols_filename(exchange_name, environ=None):
+def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
+    """
+    The absolute path of the exchange's symbol.json file.
+
+    Parameters
+    ----------
+    exchange_name:
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    name = 'symbols.json' if not is_local else 'symbols_local.json'
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    return os.path.join(exchange_folder, 'symbols.json')
+    return os.path.join(exchange_folder, name)
 
 
 def download_exchange_symbols(exchange_name, environ=None):
+    """
+    Downloads the exchange's symbols.json from the repository.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     filename = get_exchange_symbols_filename(exchange_name)
     url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = urllib.urlretrieve(url=url, filename=filename)
+    response = request.urlretrieve(url=url, filename=filename)
     return response
 
 
-def get_exchange_symbols(exchange_name, environ=None):
-    filename = get_exchange_symbols_filename(exchange_name)
+def symbols_parser(asset_def):
+    for key, value in asset_def.items():
+        match = isinstance(value, string_types) \
+                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
 
-    if not os.path.isfile(filename) or \
-            pd.Timedelta(pd.Timestamp('now', tz='UTC') - last_modified_time(filename)).days > 1:
+        if match:
+            try:
+                asset_def[key] = pd.to_datetime(value, utc=True)
+            except ValueError:
+                pass
+
+    return asset_def
+
+
+def get_exchange_symbols(exchange_name, is_local=False, environ=None):
+    """
+    The de-serialized content of the exchange's symbols.json.
+
+    Parameters
+    ----------
+    exchange_name: str
+    is_local: bool
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
+    filename = get_exchange_symbols_filename(exchange_name, is_local)
+
+    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
+                pd.Timestamp('now', tz='UTC') - last_modified_time(
+                    filename)).days > 1):
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
+            try:
+                data = json.load(data_file, object_hook=symbols_parser)
+                return data
+
+            except ValueError:
+                return dict()
     else:
         raise ExchangeSymbolsNotFound(
             exchange=exchange_name,
@@ -56,7 +152,63 @@ def get_exchange_symbols(exchange_name, environ=None):
         )
 
 
+def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
+    """
+    Save assets into an exchange_symbols file.
+
+    Parameters
+    ----------
+    exchange_name: str
+    assets: list[dict[str, object]]
+    is_local: bool
+    environ
+
+    Returns
+    -------
+
+    """
+    asset_dicts = dict()
+    for symbol in assets:
+        asset_dicts[symbol] = assets[symbol].to_dict()
+
+    filename = get_exchange_symbols_filename(
+        exchange_name, is_local, environ
+    )
+    with open(filename, 'wt') as handle:
+        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
+
+
+def get_symbols_string(assets):
+    """
+    A concatenated string of symbols from a list of assets.
+
+    Parameters
+    ----------
+    assets: list[TradingPair]
+
+    Returns
+    -------
+    str
+
+    """
+    array = [assets] if isinstance(assets, TradingPair) else assets
+    return ', '.join([asset.symbol for asset in array])
+
+
 def get_exchange_auth(exchange_name, environ=None):
+    """
+    The de-serialized contend of the exchange's auth.json file.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
     filename = os.path.join(exchange_folder, 'auth.json')
 
@@ -67,10 +219,43 @@ def get_exchange_auth(exchange_name, environ=None):
     else:
         data = dict(name=exchange_name, key='', secret='')
         with open(filename, 'w') as f:
-            json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':'))
+            json.dump(data, f, sort_keys=False, indent=2,
+                      separators=(',', ':'))
             return data
 
+
+def delete_algo_folder(algo_name, environ=None):
+    """
+    Delete the folder containing the algo state.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
+    shutil.rmtree(folder)
+
+
 def get_algo_folder(algo_name, environ=None):
+    """
+    The algorithm root folder of the algorithm.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     if not environ:
         environ = os.environ
 
@@ -82,6 +267,21 @@ def get_algo_folder(algo_name, environ=None):
 
 
 def get_algo_object(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized object of the algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    environ:
+    rel_path: str
+
+    Returns
+    -------
+    Object
+
+    """
     if algo_name is None:
         return None
 
@@ -96,13 +296,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
         try:
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception as e:
+        except Exception:
             return None
     else:
         return None
 
 
 def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+    """
+    Serialize and save an object by algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    obj: Object
+    environ:
+    rel_path: str
+
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -115,16 +327,22 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
-def append_algo_object(algo_name, key, obj, environ=None):
-    algo_folder = get_algo_folder(algo_name, environ)
-    filename = os.path.join(algo_folder, key + '.p')
-
-    mode = 'a+b' if os.path.isfile(filename) else 'wb'
-    with open(filename, mode) as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized DataFrame of an algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    environ:
+    rel_path: str
+
+    Returns
+    -------
+    DataFrame
+
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -143,19 +361,43 @@ def get_algo_df(algo_name, key, environ=None, rel_path=None):
 
 
 def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
-    folder = get_algo_folder(algo_name, environ)
+    """
+    Serialize to csv and save a DataFrame by algo name and key.
 
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    df: pd.DataFrame
+    environ:
+    rel_path: str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
     filename = os.path.join(folder, key + '.csv')
 
-    with open(filename, 'wb') as handle:
-        df.to_csv(handle)
+    with open(filename, 'wt') as handle:
+        df.to_csv(handle, encoding='UTF_8')
 
 
 def get_exchange_minute_writer_root(exchange_name, environ=None):
+    """
+    The minute writer folder for the exchange.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    BcolzExchangeBarWriter
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     minute_data_folder = os.path.join(exchange_folder, 'minute_data')
@@ -163,7 +405,21 @@ def get_exchange_minute_writer_root(exchange_name, environ=None):
 
     return minute_data_folder
 
+
 def get_exchange_bundles_folder(exchange_name, environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     temp_bundles = os.path.join(exchange_folder, 'temp_bundles')
@@ -172,9 +428,221 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
-def perf_serial(obj):
-    """JSON serializer for objects not serializable by default json code"""
+def symbols_serial(obj):
+    """
+    JSON serializer for objects not serializable by default json code
 
+    Parameters
+    ----------
+    obj: Object
+
+    Returns
+    -------
+    str
+
+    """
+    if isinstance(obj, (datetime, date)):
+        return obj.floor('1D').strftime(DATE_FORMAT)
+
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
+def perf_serial(obj):
+    """
+    JSON serializer for objects not serializable by default json code
+
+    Parameters
+    ----------
+    obj: Object
+
+    Returns
+    -------
+    str
+
+    """
     if isinstance(obj, (datetime, date)):
         return obj.isoformat()
+
     raise TypeError("Type %s not serializable" % type(obj))
+
+
+def get_common_assets(exchanges):
+    """
+    The assets available in all specified exchanges.
+
+    Parameters
+    ----------
+    exchanges: list[Exchange]
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    symbols = []
+    for exchange_name in exchanges:
+        s = [asset.symbol for asset in exchanges[exchange_name].get_assets()]
+        symbols.append(s)
+
+    inter_symbols = set.intersection(*map(set, symbols))
+
+    assets = []
+    for symbol in inter_symbols:
+        for exchange_name in exchanges:
+            asset = exchanges[exchange_name].get_asset(symbol)
+            assets.append(asset)
+
+    return assets
+
+
+def get_frequency(freq, data_frequency):
+    """
+    Get the frequency parameters.
+
+    Notes
+    -----
+    We're trying to use Pandas convention for frequency aliases.
+
+    Parameters
+    ----------
+    freq: str
+    data_frequency: str
+
+    Returns
+    -------
+    str, int, str, str
+
+    """
+    if freq == 'minute':
+        unit = 'T'
+        candle_size = 1
+
+    elif freq == 'daily':
+        unit = 'D'
+        candle_size = 1
+
+    else:
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
+                else 1
+            unit = freq_match.group(2)
+
+        else:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+    if unit.lower() == 'd':
+        alias = '{}D'.format(candle_size)
+
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm' or unit == 'T':
+        alias = '{}T'.format(candle_size)
+
+        if data_frequency == 'daily':
+            data_frequency = 'minute'
+
+    # elif unit.lower() == 'h':
+    #     candle_size = candle_size * 60
+    #
+    #     alias = '{}T'.format(candle_size)
+    #     if data_frequency == 'daily':
+    #         data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyAlias(freq=freq)
+
+    return alias, candle_size, unit, data_frequency
+
+
+def resample_history_df(df, freq, field):
+    """
+    Resample the OHCLV DataFrame using the specified frequency.
+
+    Parameters
+    ----------
+    df: DataFrame
+    freq: str
+    field: str
+
+    Returns
+    -------
+    DataFrame
+
+    """
+    if field == 'open':
+        agg = 'first'
+    elif field == 'high':
+        agg = 'max'
+    elif field == 'low':
+        agg = 'min'
+    elif field == 'close':
+        agg = 'last'
+    elif field == 'volume':
+        agg = 'sum'
+    else:
+        raise ValueError('Invalid field.')
+
+    resampled_df = df.resample(freq).agg(agg)
+    return resampled_df
+
+
+def mixin_market_params(exchange_name, params, market):
+    """
+    Applies a CCXT market dict to parameters of TradingPair init.
+
+    Parameters
+    ----------
+    params: dict[Object]
+    market: dict[Object]
+
+    Returns
+    -------
+
+    """
+    # TODO: make this more externalized / configurable
+    if 'lot' in market:
+        params['min_trade_size'] = market['lot']
+        params['lot'] = market['lot']
+
+    if exchange_name == 'bitfinex':
+        params['maker'] = 0.001
+        params['taker'] = 0.002
+
+    elif 'maker' in market and 'taker' in market \
+            and market['maker'] is not None and market['taker'] is not None:
+        params['maker'] = market['maker']
+        params['taker'] = market['taker']
+
+    else:
+        # TODO: default commission, make configurable
+        params['maker'] = 0.0015
+        params['taker'] = 0.0025
+
+    info = market['info'] if 'info' in market else None
+    if info:
+        if 'minimum_order_size' in info:
+            params['min_trade_size'] = float(info['minimum_order_size'])
+
+            if 'lot' not in params:
+                params['lot'] = params['min_trade_size']
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
+
+
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)
+
+
+def group_assets_by_exchange(assets):
+    exchange_assets = dict()
+    for asset in assets:
+        if asset.exchange not in exchange_assets:
+            exchange_assets[asset.exchange] = list()
+
+        exchange_assets[asset.exchange].append(asset)
+
+    return exchange_assets

catalyst/exchange/factory.py

@@ -0,0 +1,34 @@
+import os
+
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_exchange_folder
+
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
+    exchange_auth = get_exchange_auth(exchange_name)
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
+        )
+
+    return CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+    )
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges

catalyst/exchange/init_utils.py

@@ -1,32 +0,0 @@
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.exchange_errors import ExchangeNotFoundError
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.exchange.poloniex.poloniex import Poloniex
-
-
-def get_exchange(exchange_name):
-    exchange_auth = get_exchange_auth(exchange_name)
-    if exchange_name == 'bitfinex':
-        return Bitfinex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=None,  # TODO: make optional at the exchange
-            portfolio=None
-        )
-    elif exchange_name == 'bittrex':
-        return Bittrex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=None,
-            portfolio=None
-        )
-    elif exchange_name == 'poloniex':
-        return Poloniex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=None,
-            portfolio=None
-        )
-    else:
-        raise ExchangeNotFoundError(exchange_name=exchange_name)

catalyst/exchange/live_graph_clock.py

@@ -1,16 +1,3 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
@@ -33,8 +20,8 @@ class LiveGraphClock(object):
 
     This mixes the clock with a live graph.
 
-    Note
-    ----
+    Notes
+    -----
     This seemingly awkward approach allows us to run the program using a single
     thread. This is important because Matplotlib does not play nice with
     multi-threaded environments. Zipline probably does not either.
@@ -53,7 +40,7 @@ class LiveGraphClock(object):
 
     def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
 
-        global mdates, plt                  #TODO: Could be cleaner
+        global mdates, plt  # TODO: Could be cleaner
         import matplotlib.dates as mdates
         from matplotlib import pyplot as plt
         from matplotlib import style
@@ -95,11 +82,12 @@ class LiveGraphClock(object):
         """
         Trying to assign reasonable parameters to the time axis.
 
-        TODO: room for improvement
+        Parameters
+        ----------
+        ax:
 
-        :param ax:
-        :return:
         """
+        # TODO: room for improvement
         ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
         ax.xaxis.set_major_formatter(self.fmt)
 
@@ -113,9 +101,21 @@ class LiveGraphClock(object):
         ax.grid(True)
 
     def set_legend(self, ax):
+        """
+        Set legend on the chart.
+
+        Parameters
+        ----------
+        ax
+
+        """
         ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
 
     def draw_pnl(self):
+        """
+        Draw p&l line on the chart.
+
+        """
         ax = self.ax_pnl
         df = self.context.pnl_stats
 
@@ -136,6 +136,10 @@ class LiveGraphClock(object):
         self.format_ax(ax)
 
     def draw_custom_signals(self):
+        """
+        Draw custom signals on the chart.
+
+        """
         ax = self.ax_custom_signals
         df = self.context.custom_signals_stats
 
@@ -154,6 +158,10 @@ class LiveGraphClock(object):
         self.format_ax(ax)
 
     def draw_exposure(self):
+        """
+        Draw exposure line on the chart.
+
+        """
         ax = self.ax_exposure
         context = self.context
         df = context.exposure_stats

catalyst/exchange/poloniex/poloniex.py

@@ -1,5 +1,4 @@
 import json
-import json
 import time
 from collections import defaultdict
 
@@ -18,25 +17,34 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrphanOrderReverseError)
+    InvalidOrderStyle,
+    OrphanOrderError,
+    OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
+    download_exchange_symbols, get_symbols_string
 from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 from catalyst.protocol import Account
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('Poloniex', level=LOG_LEVEL)
 
 
+@deprecated
 class Poloniex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Poloniex_api(key=key, secret=secret)
         self.name = 'poloniex'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -47,7 +55,7 @@ class Poloniex(Exchange):
         self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def sanitize_curency_symbol(self, exchange_symbol):
         """
@@ -82,7 +90,6 @@ class Poloniex(Exchange):
             # filled = -filled
 
         price = float(order_status['rate'])
-        order_type = order_status['type']
 
         stop_price = None
         limit_price = None
@@ -96,11 +103,11 @@ class Poloniex(Exchange):
         # executed_price = float(order_status['avg_execution_price'])
         executed_price = price
 
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
+        # TODO: Set Poloniex comission
         commission = None
 
-        # date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date = pytz.utc.localize(date)
+        # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
+        # date=pytz.utc.localize(date)
         date = None
 
         order = Order(
@@ -119,9 +126,9 @@ class Poloniex(Exchange):
         return order, executed_price
 
     def get_balances(self):
-        log.debug('retrieving wallets balances')
+        balances = self.api.returnbalances()
         try:
-            balances = self.api.returnbalances()
+            log.debug('retrieving wallets balances')
         except Exception as e:
             log.debug(e)
             raise ExchangeRequestError(error=e)
@@ -171,12 +178,12 @@ class Poloniex(Exchange):
         # TODO: fetch account data and keep in cache
         return None
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count=None,
                     start_dt=None, end_dt=None):
         """
         Retrieve OHLVC candles from Poloniex
 
-        :param data_frequency:
+        :param freq:
         :param assets:
         :param bar_count:
         :return:
@@ -186,41 +193,58 @@ class Poloniex(Exchange):
         '5m', '15m', '30m', '2h', '4h', '1D'
         """
 
-        # TODO: implement end_dt and start_dt filters
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
 
-        if (
-                        data_frequency == '5m' or data_frequency == 'minute'):  # TODO: Polo does not have '1m'
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        if freq == '1T' and (bar_count == 1 or bar_count is None):
+            # TODO: use the order book instead
+            # We use the 5m to fetch the last bar
             frequency = 300
-        elif (data_frequency == '15m'):
+        elif freq == '5T':
+            frequency = 300
+        elif freq == '15T':
             frequency = 900
-        elif (data_frequency == '30m'):
+        elif freq == '30T':
             frequency = 1800
-        elif (data_frequency == '2h'):
+        elif freq == '120T':
             frequency = 7200
-        elif (data_frequency == '4h'):
+        elif freq == '240T':
             frequency = 14400
-        elif (data_frequency == '1D' or data_frequency == 'daily'):
+        elif freq == '1D':
             frequency = 86400
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            # Poloniex does not offer 1m data candles
+            # It is likely to error out there frequently
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
         ohlc_map = dict()
 
         for asset in asset_list:
+            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
+            end = int(delta.total_seconds())
 
-            end = int(time.time())
-            if (bar_count is None):
+            if bar_count is None:
                 start = end - 2 * frequency
             else:
                 start = end - bar_count * frequency
 
             try:
-                response = self.api.returnchartdata(self.get_symbol(asset),
-                                                    frequency, start, end)
+                response = self.api.returnchartdata(
+                    self.get_symbol(asset), frequency, start, end
+                )
             except Exception as e:
                 raise ExchangeRequestError(error=e)
 
@@ -270,8 +294,8 @@ class Poloniex(Exchange):
         """
         exchange_symbol = self.get_symbol(asset)
 
-        if isinstance(style, ExchangeLimitOrder) or isinstance(style,
-                                                               ExchangeStopLimitOrder):
+        if (isinstance(style, ExchangeLimitOrder)
+                or isinstance(style, ExchangeStopLimitOrder)):
             if isinstance(style, ExchangeStopLimitOrder):
                 log.warn('{} will ignore the stop price'.format(self.name))
 
@@ -328,8 +352,8 @@ class Poloniex(Exchange):
         return self.portfolio.open_orders
 
         """
-            TODO: Why going to the exchange if we already have this info locally?
-                  And why creating all these Orders if we later discard them?
+        TODO: Why going to the exchange if we already have this info locally?
+              And why creating all these Orders if we later discard them?
         """
 
         try:
@@ -343,7 +367,7 @@ class Poloniex(Exchange):
         if 'error' in response:
             raise ExchangeRequestError(
                 error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
+                    response['message'])
             )
 
         print(self.portfolio.open_orders)
@@ -351,8 +375,8 @@ class Poloniex(Exchange):
         # TODO: Need to handle openOrders for 'all'
         orders = list()
         for order_status in response:
-            order, executed_price = self._create_order(
-                order_status)  # will Throw error b/c Polo doesn't track order['symbol']
+            # will Throw error b/c Polo doesn't track order['symbol']
+            order, executed_price = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 
@@ -415,7 +439,8 @@ class Poloniex(Exchange):
 
         if 'error' in response:
             log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
+                'Unable to cancel order {order_id} on exchange {exchange} '
+                '{error}.'.format(
                     order_id=order.id,
                     exchange=self.name,
                     error=response['error']
@@ -490,17 +515,17 @@ class Poloniex(Exchange):
             else:
                 try:
                     start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError as e:
+                except KeyError:
                     start_date = time.strftime('%Y-%m-%d')
 
             try:
                 end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[exchange_symbol] = dict(
@@ -571,19 +596,21 @@ class Poloniex(Exchange):
                 else:
                     for tx in response:
                         """
-                            We maintain a list of dictionaries of transactions that correspond to
-                            partially filled orders, indexed by order_id. Every time we query 
-                            executed transactions from the exchange, we check if we had that 
-                            transaction for that order already. If not, we process it.
+                        We maintain a list of dictionaries of transactions that
+                        correspond to partially filled orders, indexed by
+                        order_id. Every time we query executed transactions
+                        from the exchange, we check if we had that transaction
+                        for that order already. If not, we process it.
 
-                            When an order if fully filled, we flush the dict of transactions  
-                            associated with that order.
+                        When an order if fully filled, we flush the dict of
+                        transactions associated with that order.
                         """
                         if (not filter(
                                 lambda item: item['order_id'] == tx['tradeID'],
                                 self.transactions[order_id])):
                             log.debug(
-                                'Got new transaction for order {}: amount {}, price {}'.format(
+                                'Got new transaction for order {}: amount {}, '
+                                'price {}'.format(
                                     order_id, tx['amount'], tx['rate']))
                             tx['amount'] = float(tx['amount'])
                             if (tx['type'] == 'sell'):
@@ -594,7 +621,7 @@ class Poloniex(Exchange):
                                 dt=pd.to_datetime(tx['date'], utc=True),
                                 price=float(tx['rate']),
                                 order_id=tx['tradeID'],
-                                # it's a misnomer, but keeping it for compatibility
+                                # it's a misnomer, but keep for compatibility
                                 commission=float(tx['fee'])
                             )
                             self.transactions[order_id].append(transaction)
@@ -604,7 +631,8 @@ class Poloniex(Exchange):
                     if (not order_open):
                         """
                             Since transactions have been executed individually
-                            the only thing left to do is remove them from list of open_orders
+                            the only thing left to do is remove them from list
+                            of open_orders
                         """
                         del self.portfolio.open_orders[order_id]
                         del self.transactions[order_id]

catalyst/exchange/poloniex/poloniex_api.py

@@ -3,6 +3,7 @@ import json
 import time
 import hmac
 import hashlib
+import ssl
 
 from six.moves import urllib
 
@@ -19,19 +20,25 @@ class Poloniex_api(object):
         self.max_requests_per_second = 6
         self.request_cpt = dict()
 
-        self.public  = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                        'returnTradeHistory', 'returnChartData',
-                        'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses',
-                        'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders',
-                        'returnTradeHistory','returnOrderTrades',
+        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
+                       'returnTradeHistory', 'returnChartData',
+                       'returnCurrencies', 'returnLoanOrders']
+        self.trading = ['returnBalances', 'returnCompleteBalances',
+                        'returnDepositAddresses',
+                        'generateNewAddress', 'returnDepositsWithdrawals',
+                        'returnOpenOrders',
+                        'returnTradeHistory', 'returnOrderTrades',
                         'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo','returnAvailableAccountBalances',
+                        'withdraw', 'returnFeeInfo',
+                        'returnAvailableAccountBalances',
                         'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary','marginBuy','marginSell',
-                        'getMarginPosition', 'closeMarginPosition','createLoanOffer',
-                        'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans',
-                        'returnLendingHistory','toggleAutoRenew']
+                        'returnMarginAccountSummary', 'marginBuy',
+                        'marginSell',
+                        'getMarginPosition', 'closeMarginPosition',
+                        'createLoanOffer',
+                        'cancelLoanOffer', 'returnOpenLoanOffers',
+                        'returnActiveLoans',
+                        'returnLendingHistory', 'toggleAutoRenew']
 
     def ask_request(self):
         """
@@ -50,7 +57,7 @@ class Poloniex_api(object):
             self.request_cpt[now] = 0
             return True
 
-        cpt_date = self.request_cpt.keys()[0]
+        cpt_date = list(self.request_cpt.keys())[0]
         cpt = self.request_cpt[cpt_date]
 
         if now > cpt_date + 1:
@@ -59,9 +66,8 @@ class Poloniex_api(object):
             return True
 
         if cpt >= self.max_requests_per_second:
-            
-            log.debug('max requests 6 reached, sleeping for 1 seconds')
-            sleep(1)
+
+            time.sleep(1)
 
             now = time.time()
             self.request_cpt = dict()
@@ -73,22 +79,37 @@ class Poloniex_api(object):
     def query(self, method, req={}):
 
         if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req)
+            url = 'https://poloniex.com/public?command=' + method + '&' + \
+                  urllib.parse.urlencode(req)
             headers = {}
             post_data = None
         elif method in self.trading:
             url = 'https://poloniex.com/tradingApi'
             req['command'] = method
-            req['nonce']   = int(time.time()*1000)
-            post_data      = urllib.parse.urlencode(req)
-            signature      = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
-            headers        = { 'Sign': signature, 'Key': self.key}
+            req['nonce'] = int(time.time() * 1000)
+            post_data = urllib.parse.urlencode(req)
+
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 post_data.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
+            headers = {'Sign': signature, 'Key': self.key}
+
+            post_data = post_data.encode('utf-8')
         else:
-            raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints')
+            raise ValueError(
+                'Method "' + method + '" not found in neither the Public API '
+                                      'or Trading API endpoints'
+            )
 
         self.ask_request()
-        req = urllib.request.Request(url, data=post_data, headers=headers)
-        return json.loads(urlopen(req).read())
+        req = urllib.request.Request(
+            url,
+            data=post_data,
+            headers=headers,
+        )
+        resource = urlopen(req, context=ssl._create_unverified_context())
+        content = resource.read().decode('utf-8')
+        return json.loads(content)
 
     def returnticker(self):
         return self.query('returnTicker', {})
@@ -100,15 +121,17 @@ class Poloniex_api(object):
         return self.query('returnOrderBook', {'currencyPair': market})
 
     def returntradehistory(self, market, start=None, end=None):
-        if(start is not None and end is not None):
-            return self.query('returntradehistory', 
-                              {'currencyPair': market, 'start': start, 'end': end })
+        if (start is not None and end is not None):
+            return self.query('returntradehistory',
+                              {'currencyPair': market, 'start': start,
+                               'end': end})
         else:
-            return self.query('returntradehistory', {'currencyPair': market })
+            return self.query('returntradehistory', {'currencyPair': market})
 
     def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData', {'currencyPair': market, 'period': period,
-                          'start': start, 'end': end})
+        return self.query('returnChartData',
+                          {'currencyPair': market, 'period': period,
+                           'start': start, 'end': end})
 
     def returncurrencies(self):
         return self.query('returnCurrencies', {})
@@ -120,7 +143,7 @@ class Poloniex_api(object):
         return self.query('returnBalances')
 
     def returncompletebalances(self, account):
-        if(account):
+        if (account):
             return self.query('returnCompleteBalances', {'account': account})
         else:
             return self.query('returnCompleteBalances')
@@ -132,43 +155,50 @@ class Poloniex_api(object):
         return self.query('generateNewAddress', {'currency': currency})
 
     def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals', {'start': start, 'end': end})
+        return self.query('returnDepositsWithdrawals',
+                          {'start': start, 'end': end})
 
     def returnopenorders(self, market):
         return self.query('returnOpenOrders', {'currencyPair': market})
 
-    def returntradehistory(self, market):
-        #TODO: optional start and/or end and limit
-        return self.query('returnTradeHistory', {'currencyPair': market})
-
     def returnordertrades(self, ordernumber):
         return self.query('returnOrderTrades', {'orderNumber': ordernumber})
 
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
-        if(fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
+            postonly=0):
+        if (fillorkill):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'fillOrKill': fillorkill, })
-        elif(immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'immediateOrCancel': immediateorcancel, })
-        elif(postonly):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+        elif (immediateorcancel):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
+                                      'immediateOrCancel': immediateorcancel})
+        elif (postonly):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'postOnly': postonly, })
         else:
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount, })
 
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
-        if(fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif(immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'immediateOrCancel': immediateorcancel, })
-        elif(postonly):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'postOnly': postonly, })
+    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
+             postonly=0):
+        if (fillorkill):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'fillOrKill': fillorkill, })
+        elif (immediateorcancel):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'immediateOrCancel': immediateorcancel})
+        elif (postonly):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'postOnly': postonly, })
         else:
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount, })
 
     def cancelorder(self, ordernumber):
         return self.query('cancelOrder', {'orderNumber': ordernumber})
@@ -180,4 +210,3 @@ class Poloniex_api(object):
 
     def returnfeeinfo(self):
         return self.query('returnFeeInfo')
-

catalyst/exchange/simple_clock.py

@@ -31,7 +31,8 @@ class SimpleClock(object):
     This class is a drop-in replacement for
     :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
 
-    This is a stripped down version because crypto exchanges run around the clock.
+    This is a stripped down version because crypto exchanges run
+    around the clock.
 
     The :param:`time_skew` parameter represents the time difference between
     the Broker and the live trading machine's clock.

catalyst/exchange/stats_utils.py

@@ -1,51 +1,426 @@
+import csv
+import numbers
+
+import copy
+import numpy as np
+import os
 import pandas as pd
+import boto3
+import time
+
+from catalyst.assets._assets import TradingPair
+
+from catalyst.exchange.exchange_utils import get_algo_folder
+
+s3 = boto3.resource('s3')
 
 
-def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
+def trend_direction(series):
+    if series[-1] is np.nan or series[-1] is np.nan:
+        return None
+
+    if series[-1] > series[-2]:
+        return 'up'
+    else:
+        return 'down'
+
+
+def crossover(source, target):
+    """
+    The `x`-series is defined as having crossed over `y`-series if the value
+    of `x` is greater than the value of `y` and the value of `x` was less than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] >= target > source[-2]:
+            return True
+        else:
+            return False
+
+    else:
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] > target[-1] and source[-2] < target[-2]:
+            return True
+        else:
+            return False
+
+
+def crossunder(source, target):
+    """
+    The `x`-series is defined as having crossed under `y`-series if the value
+    of `x` is less than the value of `y` and the value of `x` was greater than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] < target <= source[-2]:
+            return True
+        else:
+            return False
+    else:
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] < target[-1] and source[-2] >= target[-2]:
+            return True
+        else:
+            return False
+
+
+def vwap(df):
+    """
+    Volume-weighted average price (VWAP) is a ratio generally used by
+    institutional investors and mutual funds to make buys and sells so as not
+    to disturb the market prices with large orders. It is the average share
+    price of a stock weighted against its trading volume within a particular
+    time frame, generally one day.
+
+    Read more: Volume Weighted Average Price - VWAP
+    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+
+    Returns
+    -------
+
+    """
+    if 'close' not in df.columns or 'volume' not in df.columns:
+        raise ValueError('price data must include `volume` and `close`')
+
+    vol_sum = np.nansum(df['volume'].values)
+
+    try:
+        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
+    except ZeroDivisionError:
+        ret = np.nan
+
+    return ret
+
+
+def set_position_row(row, asset, asset_values=list()):
+    """
+    Apply the position data as individual columns.
+
+    Parameters
+    ----------
+    row: dict[str, Object]
+    asset: TradingPair
+    asset_values: list[str]
+        If a recorded_col contains a tuple which first value is an asset
+        matching a position, its value will be displayed with the
+        position and not in the index.
+
+    Returns
+    -------
+
+    """
+    asset_cols = ['symbol']
+    row['symbol'] = asset.symbol
+
+    position = next((p for p in row['positions'] if p['sid'] == asset), None)
+
+    columns = ['amount', 'cost_basis', 'last_sale_price']
+    for column in columns:
+        if position is not None:
+            row[column] = position[column]
+
+        else:
+            row[column] = 0
+
+        asset_cols.append(column)
+
+    values = asset_values[asset] if asset in asset_values else list()
+    for column in values:
+        row[column] = values[column]
+
+        asset_cols.append(column)
+
+    return asset_cols
+
+
+def prepare_stats(stats, recorded_cols=list()):
+    """
+    Prepare the stats DataFrame for user-friendly output.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    asset_cols = list()
+
+    stats = copy.deepcopy(stats)
+    # Using a copy since we are adding rows inside the loop.
+    for row_index, row_data in enumerate(list(stats)):
+        assets = [p['sid'] for p in row_data['positions']]
+
+        asset_values = dict()
+        if recorded_cols is not None:
+            for column in recorded_cols[:]:
+                value = row_data[column]
+                if type(value) is dict:
+                    for asset in value:
+                        if not isinstance(asset, TradingPair):
+                            break
+
+                        if asset not in assets:
+                            assets.append(asset)
+
+                        if asset not in asset_values:
+                            asset_values[asset] = dict()
+
+                        asset_values[asset][column] = value[asset]
+
+        if len(assets) == 1:
+            row = stats[row_index]
+            asset_cols = set_position_row(row, assets[0], asset_values)
+
+        elif len(assets) > 1:
+            for asset_index, asset in enumerate(assets):
+                if asset_index > 0:
+                    row = copy.deepcopy(row_data)
+                    stats.append(row)
+
+                else:
+                    row = stats[row_index]
+
+                asset_cols = set_position_row(row, assets[asset_index],
+                                              asset_values)
+
+    df = pd.DataFrame(stats)
+
+    index_cols = [
+        'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
+        'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
+    ]
+
+    # Removing the asset specific entries
+    if recorded_cols is not None:
+        recorded_cols = [x for x in recorded_cols if x not in asset_cols]
+        for column in recorded_cols:
+            index_cols.append(column)
+
+    df['orders'] = df['orders'].apply(lambda orders: len(orders))
+    df['transactions'] = df['transactions'].apply(
+        lambda transactions: len(transactions)
+    )
+
+    if asset_cols:
+        columns = asset_cols
+        df.set_index(index_cols, drop=True, inplace=True)
+
+    else:
+        columns = index_cols
+        columns.remove('period_close')
+        df.set_index('period_close', drop=False, inplace=True)
+
+    df.dropna(axis=1, how='all', inplace=True)
+    df.sort_index(axis=0, level=0, inplace=True)
+
+    return df, columns
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
 
-    :param stats_df:
-    :param num_rows:
-    :return:
+    Parameters
+    ----------
+    stats: list[Object]
+        An array of statistics for the period.
+
+    num_rows: int
+        The number of rows to display on the screen.
+
+    Returns
+    -------
+    str
+
     """
-    stats_df.set_index('period_close', drop=True, inplace=True)
-    stats_df.dropna(axis=1, how='all', inplace=True)
+    if isinstance(stats, pd.DataFrame):
+        stats = stats.T.to_dict().values()
+
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
 
     pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 3)
+    pd.set_option('precision', 8)
     pd.set_option('display.width', 1000)
     pd.set_option('display.max_colwidth', 1000)
 
-    columns = ['starting_cash', 'ending_cash', 'portfolio_value',
-               'pnl', 'long_exposure', 'short_exposure', 'orders',
-               'transactions', 'positions']
-
-    if recorded_cols is not None:
-        for column in recorded_cols:
-            columns.append(column)
-
-    def format_positions(positions):
-        parts = []
-        for position in positions:
-            msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
-                amount=position['amount'],
-                market=position['sid'].market_currency,
-                cost_basis=position['cost_basis'],
-                base=position['sid'].base_currency
-            )
-            parts.append(msg)
-        return ', '.join(parts)
-
     formatters = {
-        'orders': lambda orders: len(orders),
-        'transactions': lambda transactions: len(transactions),
         'returns': lambda returns: "{0:.4f}".format(returns),
-        'positions': format_positions
     }
 
-    return stats_df.tail(num_rows).to_string(
+    return df.tail(num_rows).to_string(
         columns=columns,
         formatters=formatters
     )
+
+
+def get_csv_stats(stats, recorded_cols=None):
+    """
+    Create a CSV buffer from the stats DataFrame.
+
+    Parameters
+    ----------
+    path: str
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+
+    return df.to_csv(
+        None,
+        columns=columns,
+        # encoding='utf-8',
+        quoting=csv.QUOTE_NONNUMERIC
+    ).encode()
+
+
+def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
+                folder='catalyst/stats', bytes_to_write=None):
+    """
+    Uploads the performance stats to a S3 bucket.
+
+    Parameters
+    ----------
+    uri: str
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+    folder: str
+    bytes_to_write: str
+        Option to reuse bytes instead of re-computing the csv
+
+    Returns
+    -------
+
+    """
+    if bytes_to_write is None:
+        bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    now = pd.Timestamp.utcnow()
+    timestr = now.strftime('%Y%m%d')
+    pid = os.getpid()
+
+    parts = uri.split('//')
+    obj = s3.Object(parts[1], '{}/{}-{}-{}.csv'.format(
+        folder, timestr, algo_namespace, pid
+    ))
+    obj.put(Body=bytes_to_write)
+
+
+def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
+    """
+    Saves the performance stats to the algo local folder.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+
+    Returns
+    -------
+    str
+
+    """
+    bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    timestr = time.strftime('%Y%m%d')
+    folder = get_algo_folder(algo_namespace)
+
+    filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
+
+    with open(filename, 'wb') as handle:
+        handle.write(bytes_to_write)
+
+    return bytes_to_write
+
+
+def df_to_string(df):
+    """
+    Create a formatted str representation of the DataFrame.
+
+    Parameters
+    ----------
+    df: DataFrame
+
+    Returns
+    -------
+    str
+
+    """
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+    return df.to_string()
+
+
+def extract_transactions(perf):
+    """
+    Compute indexes for buy and sell transactions
+
+    Parameters
+    ----------
+    perf: DataFrame
+        The algo performance DataFrame.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of transactions.
+
+    """
+    trans_list = perf.transactions.values
+    all_trans = [t for sublist in trans_list for t in sublist]
+    all_trans.sort(key=lambda t: t['dt'])
+
+    transactions = pd.DataFrame(all_trans)
+    if not transactions.empty:
+        transactions.set_index('dt', inplace=True, drop=True)
+    return transactions

catalyst/exchange/validator.py

@@ -0,0 +1,142 @@
+import os
+import tempfile
+
+import pandas as pd
+import six
+from catalyst.assets._assets import TradingPair, get_calendar
+from logbook import Logger
+from pandas.util.testing import assert_frame_equal
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
+from catalyst.exchange.factory import get_exchanges
+from catalyst.utils.paths import ensure_directory
+
+log = Logger('Validator', level=LOG_LEVEL)
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path
+
+
+class Validator(object):
+    def __init__(self, data_portal):
+        self.data_portal = data_portal
+
+    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
+                                     sample_minutes):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        sample_minutes
+
+        Returns
+        -------
+
+        """
+        freq = '{}T'.format(sample_minutes)
+
+        log.info('creating data sample from bundle')
+        df1 = self.data_portal.get_history_window(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            frequency=freq,
+            field='close',
+            data_frequency='minute'
+        )
+        path = output_df(df1, assets, '{}_resampled'.format(freq))
+        log.info('saved resampled bundle candles: {}\n{}'.format(
+            path, df1.tail(10))
+        )
+
+        log.info('creating data sample from exchange api')
+        candles = exchange.get_candles(
+            end_dt=end_dt,
+            freq='{}T'.format(sample_minutes),
+            assets=assets,
+            bar_count=bar_count
+        )
+
+        series = dict()
+        for asset in assets:
+            series[asset] = pd.Series(
+                data=[candle['close'] for candle in candles[asset]],
+                index=[candle['last_traded'] for candle in candles[asset]]
+            )
+
+        df2 = pd.DataFrame(series)
+        path = output_df(df2, assets, '{}_api'.format(freq))
+        log.info('saved exchange api candles: {}\n{}'.format(
+            path, df2.tail(10))
+        )
+
+        try:
+            assert_frame_equal(df1, df2)
+            return True
+        except:
+            log.warn('differences found in dataframes')
+            return False
+
+
+if __name__ == '__main__':
+    exchanges = get_exchanges(['poloniex'])
+    exchange = six.next(six.itervalues(exchanges))
+    assets = exchange.get_assets(symbols=['eth_btc'])
+
+    open_calendar = get_calendar('OPEN')
+    asset_finder = AssetFinderExchange()
+    data_portal = DataPortalExchangeBacktest(
+        exchanges=exchanges,
+        asset_finder=asset_finder,
+        trading_calendar=open_calendar,
+        first_trading_day=None  # will set dynamically based on assets
+    )
+    validator = Validator(data_portal=data_portal)
+
+    validator.compare_bundle_with_exchange(
+        exchange=exchange,
+        assets=assets,
+        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
+        bar_count=200,
+        sample_minutes=30
+    )

catalyst/finance/execution.py

@@ -15,13 +15,8 @@
 
 import abc
 
-from sys import float_info
-
-from six import with_metaclass
-
-import catalyst.utils.math_utils as zp_math
-
 from numpy import isfinite
+from six import with_metaclass
 
 from catalyst.errors import BadOrderParameters
 
@@ -77,6 +72,7 @@ class LimitOrder(ExecutionStyle):
     Execution style representing an order to be executed at a price equal to or
     better than a specified limit price.
     """
+
     def __init__(self, limit_price, exchange=None):
         """
         Store the given price.
@@ -99,6 +95,7 @@ class StopOrder(ExecutionStyle):
     Execution style representing an order to be placed once the market price
     reaches a specified stop price.
     """
+
     def __init__(self, stop_price, exchange=None):
         """
         Store the given price.
@@ -121,6 +118,7 @@ class StopLimitOrder(ExecutionStyle):
     Execution style representing a limit order to be placed with a specified
     limit price once the market reaches a specified stop price.
     """
+
     def __init__(self, limit_price, stop_price, exchange=None):
         """
         Store the given prices
@@ -144,31 +142,20 @@ class StopLimitOrder(ExecutionStyle):
 def asymmetric_round_price_to_penny(price, prefer_round_down,
                                     diff=(0.0095 - .005)):
     """
-    Asymmetric rounding function for adjusting prices to two places in a way
-    that "improves" the price.  For limit prices, this means preferring to
-    round down on buys and preferring to round up on sells.  For stop prices,
-    it means the reverse.
+    Modified the original function because we do not want to round
+    prices on crypto exchange.
 
-    If prefer_round_down == True:
-        When .05 below to .95 above a penny, use that penny.
-    If prefer_round_down == False:
-        When .95 below to .05 above a penny, use that penny.
+    Parameters
+    ----------
+    price: float
+
+    Returns
+    -------
+    float
 
-    In math-speak:
-    If prefer_round_down: [<X-1>.0095, X.0195) -> round to X.01.
-    If not prefer_round_down: (<X-1>.0005, X.0105] -> round to X.01.
     """
-    # Subtracting an epsilon from diff to enforce the open-ness of the upper
-    # bound on buys and the lower bound on sells.  Using the actual system
-    # epsilon doesn't quite get there, so use a slightly less epsilon-ey value.
-    epsilon = float_info.epsilon * 10
-    diff = diff - epsilon
-
-    # relies on rounding half away from zero, unlike numpy's bankers' rounding
-    rounded = round(price - (diff if prefer_round_down else -diff), 2)
-    if zp_math.tolerant_equals(rounded, 0.0):
-        return 0.0
-    return rounded
+    # TODO: consider overriding outside of the original function
+    return price
 
 
 def check_stoplimit_prices(price, label):

catalyst/finance/risk/cumulative.py

@@ -22,7 +22,7 @@ from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
-from . risk import (
+from .risk import (
     check_entry,
     choose_treasury
 )
@@ -37,12 +37,11 @@ from empyrical import (
     sharpe_ratio,
     sortino_ratio,
 )
-
+import warnings
 from catalyst.constants import LOG_LEVEL
 
 log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
-
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
 
@@ -145,6 +144,8 @@ class RiskMetricsCumulative(object):
         self.num_trading_days = 0
 
     def update(self, dt, algorithm_returns, benchmark_returns, leverage):
+        warnings.filterwarnings('error')
+
         # Keep track of latest dt for use in to_dict and other methods
         # that report current state.
         self.latest_dt = dt
@@ -191,9 +192,12 @@ class RiskMetricsCumulative(object):
             if len(self.benchmark_returns) == 1:
                 self.benchmark_returns = np.append(0.0, self.benchmark_returns)
 
-        self.benchmark_cumulative_returns[dt_loc] = cum_returns(
-            self.benchmark_returns
-        )[-1]
+        try:
+            self.benchmark_cumulative_returns[dt_loc] = cum_returns(
+                self.benchmark_returns
+            )[-1]
+        except Exception:
+            self.benchmark_cumulative_returns[dt_loc] = 0
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -268,10 +272,17 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.downside_risk[dt_loc] = downside_risk(
             self.algorithm_returns
         )
-        self.sortino[dt_loc] = sortino_ratio(
-            self.algorithm_returns,
-            _downside_risk=self.downside_risk[dt_loc]
-        )
+
+        try:
+            risk = self.downside_risk[dt_loc]
+            self.sortino[dt_loc] = sortino_ratio(
+                self.algorithm_returns,
+                _downside_risk=risk
+            )
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino[dt_loc] = 0
+
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
@@ -283,6 +294,8 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.
@@ -294,18 +307,18 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         rval = {
             'trading_days': self.num_trading_days,
             'benchmark_volatility':
-            self.benchmark_volatility[dt_loc],
+                self.benchmark_volatility[dt_loc],
             'algo_volatility':
-            self.algorithm_volatility[dt_loc],
+                self.algorithm_volatility[dt_loc],
             'treasury_period_return': self.treasury_period_return,
             # Though the two following keys say period return,
             # they would be more accurately called the cumulative return.
             # However, the keys need to stay the same, for now, for backwards
             # compatibility with existing consumers.
             'algorithm_period_return':
-            self.algorithm_cumulative_returns[dt_loc],
+                self.algorithm_cumulative_returns[dt_loc],
             'benchmark_period_return':
-            self.benchmark_cumulative_returns[dt_loc],
+                self.benchmark_cumulative_returns[dt_loc],
             'beta': self.beta[dt_loc],
             'alpha': self.alpha[dt_loc],
             'sharpe': self.sharpe[dt_loc],

catalyst/finance/risk/period.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import functools
+import warnings
 
 import logbook
 
@@ -23,7 +24,7 @@ import numpy as np
 import pandas as pd
 
 from . import risk
-from . risk import check_entry
+from .risk import check_entry
 
 from empyrical import (
     alpha_beta_aligned,
@@ -78,14 +79,20 @@ class RiskMetricsPeriod(object):
         self.calculate_metrics()
 
     def calculate_metrics(self):
-        self.benchmark_period_returns = \
-            cum_returns(self.benchmark_returns).iloc[-1]
+        warnings.filterwarnings('error')
+
+        try:
+            self.benchmark_period_returns = \
+                cum_returns(self.benchmark_returns).iloc[-1]
+        except Exception:
+            # TODO: why is there an error
+            self.benchmark_period_returns = 0
 
         self.algorithm_period_returns = \
             cum_returns(self.algorithm_returns).iloc[-1]
 
         if not self.algorithm_returns.index.equals(
-            self.benchmark_returns.index
+                self.benchmark_returns.index
         ):
             message = "Mismatch between benchmark_returns ({bm_count}) and \
             algorithm_returns ({algo_count}) in range {start} : {end}"
@@ -128,10 +135,17 @@ class RiskMetricsPeriod(object):
         self.downside_risk = downside_risk(
             self.algorithm_returns.values
         )
-        self.sortino = sortino_ratio(
-            self.algorithm_returns.values,
-            _downside_risk=self.downside_risk,
-        )
+
+        try:
+            risk = self.downside_risk
+            self.sortino = sortino_ratio(
+                self.algorithm_returns.values,
+                _downside_risk=risk,
+            )
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino = 0
+
         self.information = information_ratio(
             self.algorithm_returns.values,
             self.benchmark_returns.values,
@@ -140,11 +154,13 @@ class RiskMetricsPeriod(object):
             self.algorithm_returns.values,
             self.benchmark_returns.values,
         )
-        self.excess_return = self.algorithm_period_returns - \
-            self.treasury_period_return
+        self.excess_return = self.algorithm_period_returns \
+            - self.treasury_period_return
         self.max_drawdown = max_drawdown(self.algorithm_returns.values)
         self.max_leverage = self.calculate_max_leverage()
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/finance/risk/risk.py

@@ -160,7 +160,8 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
                 )
                 break
 
-        if search_day and trading_calendar.name != 'OPEN':          # Supress warning for 'OPEN' calendar
+        # Supress warning for 'OPEN' calendar
+        if search_day and trading_calendar.name != 'OPEN':
             if (search_dist is None or search_dist > 1) and \
                     search_days[0] <= end_session <= search_days[-1]:
                 message = "No rate within 1 trading day of end date = \

catalyst/finance/slippage.py

@@ -41,7 +41,6 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
 
 
-
 class LiquidityExceeded(Exception):
     pass
 

catalyst/pipeline/factors/equity/__init__.py

@@ -1,9 +1,6 @@
 from .statistical import (
-    RollingPearson,
-    RollingLinearRegression,
     RollingLinearRegressionOfReturns,
     RollingPearsonOfReturns,
-    RollingSpearman,
     RollingSpearmanOfReturns,
 )
 from .technical import (

catalyst/pipeline/loaders/equity_pricing_loader.py

@@ -38,9 +38,11 @@ class USEquityPricingLoader(PipelineLoader):
 
     def __init__(self, bundle, data_frequency, dataset):
 
-        if data_frequency == 'daily':
-            reader = bundle.daily_bar_reader
-        elif daily_bar_reader == 'minute':
+        # TODO: This is currently broken, No Pipeline support for Catalyst
+        # if data_frequency == 'daily':
+        #    reader = bundle.daily_bar_reader
+        # elif daily_bar_reader == 'minute':
+        if data_frequency == 'minute':
             reader = bundle.minute_bar_reader
         else:
             raise ValueError(
@@ -51,7 +53,9 @@ class USEquityPricingLoader(PipelineLoader):
 
         if data_frequency == 'daily':
             all_sessions = cal.all_sessions
-        elif daily_bar_reader == 'minute':
+        # TODO: this cannot be right, but no pipeline support at the moment
+        # elif daily_bar_reader == 'minute':
+        elif data_frequency == 'minute':
             reader = bundle.minute_bar_reader
             all_sessions = cal.all_minutes
 

catalyst/pipeline/loaders/events.py

@@ -231,7 +231,7 @@ class EventsLoader(PipelineLoader):
             self.load_next_events(n, dates, sids, mask),
             self.load_previous_events(p, dates, sids, mask),
         )
-    
+
     @property
     def columns(self):
         return self._columns

catalyst/pipeline/loaders/frame.py

@@ -180,4 +180,3 @@ class DataFrameLoader(PipelineLoader):
     @property
     def columns(self):
         return self._columns
-

catalyst/pipeline/loaders/synthetic.py

@@ -163,7 +163,7 @@ class SeededRandomLoader(PrecomputedLoader):
             bool_dtype: self._bool_values,
             object_dtype: self._object_values,
         }[dtype](shape)
-    
+
     @property
     def columns(self):
         return self._columns

catalyst/support/issue_55.py

@@ -0,0 +1,41 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('xcp_btc')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=1,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2015-3-2', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='poloniex',
+    algo_namespace='issue_55',
+    base_currency='btc'
+)

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,5 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(
+            start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/cli.py

@@ -9,6 +9,7 @@ DEFAULT_BAR_TEMPLATE = '    [%(bar)s]  %(label)s:  %(info)s'
 DEFAULT_EMPTY_CHAR = ' '
 DEFAULT_FILL_CHAR = '='
 
+
 def item_show_count(total=None):
     def maybe_show_total(index):
         if total is not None:
@@ -17,12 +18,13 @@ def item_show_count(total=None):
 
     def item_show_func(item, _it=iter(count())):
         if item is not None:
-            starting = False
+            # starting = False
             return maybe_show_total(next(_it))
         return 'DONE'
 
     return item_show_func
 
+
 def maybe_show_progress(it,
                         show_progress,
                         empty_char=DEFAULT_EMPTY_CHAR,

catalyst/utils/math_utils.py

@@ -17,9 +17,11 @@ import math
 
 from numpy import isnan
 
+
 def round_nearest(x, a):
     return round(round(x / a) * a, -int(math.floor(math.log10(a))))
 
+
 def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False):
     """Check if a and b are equal with some tolerance.
 

catalyst/utils/paths.py

@@ -126,7 +126,7 @@ def catalyst_root(environ=None):
 
     root = environ.get('ZIPLINE_ROOT', None)
     if root is None:
-        root = expanduser('~/.catalyst')
+        root = os.path.join(expanduser('~'), '.catalyst')
 
     return root
 

catalyst/utils/run_algo.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 import warnings
 from datetime import timedelta
@@ -7,10 +8,11 @@ from time import sleep
 
 import click
 import pandas as pd
+from logbook import Logger
 
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.poloniex.poloniex import Poloniex
+from catalyst.data.bundles import load
+from catalyst.data.data_portal import DataPortal
+from catalyst.exchange.factory import get_exchange
 
 try:
     from pygments import highlight
@@ -29,19 +31,16 @@ from catalyst.utils.factory import create_simulation_parameters
 from catalyst.data.loader import load_crypto_market_data
 import catalyst.utils.paths as pth
 
-from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
-    ExchangeTradingAlgorithmBacktest
-from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
+from catalyst.exchange.exchange_algorithm import (
+    ExchangeTradingAlgorithmLive,
+    ExchangeTradingAlgorithmBacktest,
+)
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeAuthEmpty,
-    ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, ExchangeNotFoundError)
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object, get_exchange_folder
-from logbook import Logger
+    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
+    BaseCurrencyNotFoundError, NotEnoughCapitalError)
 
 from catalyst.constants import LOG_LEVEL
 
@@ -91,7 +90,9 @@ def _run(handle_data,
          exchange,
          algo_namespace,
          base_currency,
-         live_graph):
+         live_graph,
+         simulate_orders,
+         stats_output):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
@@ -140,7 +141,8 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
-    mode = 'live' if live else 'backtest'
+    mode = 'paper-trading' if simulate_orders else 'live-trading' \
+        if live else 'backtest'
     log.info('running algo in {mode} mode'.format(mode=mode))
 
     exchange_name = exchange
@@ -151,51 +153,12 @@ def _run(handle_data,
 
     exchanges = dict()
     for exchange_name in exchange_list:
-
-        # Looking for the portfolio from the cache first
-        portfolio = get_algo_object(
-            algo_name=algo_namespace,
-            key='portfolio_{}'.format(exchange_name),
-            environ=environ
+        exchanges[exchange_name] = get_exchange(
+            exchange_name=exchange_name,
+            base_currency=base_currency,
+            must_authenticate=(live and not simulate_orders),
         )
 
-        if portfolio is None:
-            portfolio = ExchangePortfolio(
-                start_date=pd.Timestamp.utcnow()
-            )
-
-        # This corresponds to the json file containing api token info
-        exchange_auth = get_exchange_auth(exchange_name)
-
-        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
-            raise ExchangeAuthEmpty(
-                    exchange=exchange_name.title(), 
-                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
-
-        if exchange_name == 'bitfinex':
-            exchanges[exchange_name] = Bitfinex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'bittrex':
-            exchanges[exchange_name] = Bittrex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'poloniex':
-            exchanges[exchange_name] = Poloniex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        else:
-            raise ExchangeNotFoundError(exchange_name=exchange_name)
-
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
@@ -210,7 +173,7 @@ def _run(handle_data,
         asset_db_path=None  # We don't need an asset db, we have exchanges
     )
     env.asset_finder = AssetFinderExchange()
-    choose_loader = None  # TODO: use the DataPortal for in the algorithm class for this
+    choose_loader = None  # TODO: use the DataPortal in the algo class for this
 
     if live:
         start = pd.Timestamp.utcnow()
@@ -258,17 +221,32 @@ def _run(handle_data,
                     )
 
             if base_currency in balances:
-                return balances[base_currency]
+                base_currency_available = balances[base_currency]['free']
+                log.info(
+                    'base currency available in the account: {} {}'.format(
+                        base_currency_available, base_currency
+                    )
+                )
+
+                return base_currency_available
             else:
                 raise BaseCurrencyNotFoundError(
                     base_currency=base_currency,
                     exchange=exchange_name
                 )
 
-        capital_base = 0
-        for exchange_name in exchanges:
-            exchange = exchanges[exchange_name]
-            capital_base += fetch_capital_base(exchange)
+        if not simulate_orders:
+            for exchange_name in exchanges:
+                exchange = exchanges[exchange_name]
+                balance = fetch_capital_base(exchange)
+
+                if balance < capital_base:
+                    raise NotEnoughCapitalError(
+                        exchange=exchange_name,
+                        base_currency=base_currency,
+                        balance=balance,
+                        capital_base=capital_base,
+                    )
 
         sim_params = create_simulation_parameters(
             start=start,
@@ -285,9 +263,11 @@ def _run(handle_data,
             ExchangeTradingAlgorithmLive,
             exchanges=exchanges,
             algo_namespace=algo_namespace,
-            live_graph=live_graph
+            live_graph=live_graph,
+            simulate_orders=simulate_orders,
+            stats_output=stats_output,
         )
-    else:
+    elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
         # We can add back the complexity if required.
 
@@ -297,7 +277,7 @@ def _run(handle_data,
         # can handle this later.
 
         data = DataPortalExchangeBacktest(
-            exchanges=exchanges,
+            exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,
             trading_calendar=open_calendar,
             first_trading_day=start,
@@ -317,6 +297,36 @@ def _run(handle_data,
             exchanges=exchanges
         )
 
+    elif bundle is not None:
+        bundle_data = load(
+            bundle,
+            environ,
+            bundle_timestamp,
+        )
+
+        prefix, connstr = re.split(
+            r'sqlite:///',
+            str(bundle_data.asset_finder.engine.url),
+            maxsplit=1,
+        )
+        if prefix:
+            raise ValueError(
+                "invalid url %r, must begin with 'sqlite:///'" %
+                str(bundle_data.asset_finder.engine.url),
+            )
+
+        env = TradingEnvironment(asset_db_path=connstr, environ=environ)
+        first_trading_day = \
+            bundle_data.equity_minute_bar_reader.first_trading_day
+
+        data = DataPortal(
+            env.asset_finder, open_calendar,
+            first_trading_day=first_trading_day,
+            equity_minute_reader=bundle_data.equity_minute_bar_reader,
+            equity_daily_reader=bundle_data.equity_daily_bar_reader,
+            adjustment_reader=bundle_data.adjustment_reader,
+        )
+
     perf = algorithm_class(
         namespace=namespace,
         env=env,
@@ -416,7 +426,10 @@ def run_algorithm(initialize,
                   exchange_name=None,
                   base_currency=None,
                   algo_namespace=None,
-                  live_graph=False):
+                  live_graph=False,
+                  simulate_orders=True,
+                  stats_output=None,
+                  output=os.devnull):
     """Run a trading algorithm.
 
     Parameters
@@ -486,8 +499,18 @@ def run_algorithm(initialize,
     --------
     catalyst.data.bundles.bundles : The available data bundles.
     """
-    load_extensions(default_extension, extensions, strict_extensions, environ)
+    load_extensions(
+        default_extension, extensions, strict_extensions, environ
+    )
 
+    if capital_base is None:
+        raise ValueError(
+            'Please specify a `capital_base` parameter which is the maximum '
+            'amount of base currency available for trading. For example, '
+            'if the `capital_base` is 5ETH, the '
+            '`order_target_percent(asset, 1)` command will order 5ETH worth '
+            'of the specified asset.'
+        )
     # I'm not sure that we need this since the modified DataPortal
     # does not require extensions to be explicitly loaded.
 
@@ -527,7 +550,7 @@ def run_algorithm(initialize,
         bundle_timestamp=bundle_timestamp,
         start=start,
         end=end,
-        output=os.devnull,
+        output=output,
         print_algo=False,
         local_namespace=False,
         environ=environ,
@@ -535,5 +558,7 @@ def run_algorithm(initialize,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        simulate_orders=simulate_orders,
+        stats_output=stats_output
     )

docs/live-trading-wiki.md

@@ -1,105 +0,0 @@
-<h1>Live Trading</h1>
-This document explains how to get started with live trading.
-
-<h2>Supported Exchanges</h2>
-Catalyst can trade against these exchanges:
-
-* Bitfinex, id=`bitfinex`
-* Bittrex, id=`bittrex`
-
-<h3>Authentication</h3>
-Most exchanges require key/token combination for authentication. By
-convention, Catalyst uses an "auth.json" file to hold this data.
-
-This example illustrates the convention using the Bitfinex exchange.
-Here is how to generate key and secret values for bitfinex:
-https://docs.bitfinex.com/v1/docs/api-access. Most exchanges follow
-a similar process.
-
-The auth.json file:
-```json
-{
-  "name": "bitfinex",
-  "key": "my-key",
-  "secret": "my-secret"
-}
-```
-
-The file goes here:
-```
-~/.catalyst/data/exchanges/bitfinex/auth.json
-```
-
-Note that the 'bitfinex' directory corresponds to the id of the Bitfinex
-exchange as defined in the "Supported Exchanges" section above.
-Attempting to run an algorithm where the targeted exchange is missing
-its "auth.json" file will create the directory structure but result
-in an error.
-
-<h3>Currency Symbols</h3>
-Catalyst introduces a universal convention to reference
-trading pairs and individual currencies. This
-is required to ensure that the `symbol()` api predictably
-returns the correct asset regardless of the targeted exchange.
-
-Exchanges tend to use their own convention to represent currencies
-(e.g. XBT and BTC both represent Bitcoin on different exchanges).
-Trading pairs are also inconsistent. For example, Bitfinex
-puts the market currency before the base currency without a
-separator, Bittrex puts the base currency first and uses a dash
-seperator.
-
-Here is the Catalyst convention:
-
-*[Market Currency]_[Base Currency]* all lowercase.
-
-Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
-
-Here are some examples:
-```python
-# With Bitfinex
-bitcoin_usd_asset = symbol('btc_usd')
-ethereum_bitcoin_asset = symbol('eth_btc')
-
-# With Bittrex
-ethereum_bitcoin_asset = symbol('eth_btc')
-neo_ethereum_asset = symbol('neo_eth)
-```
-
-Note that the trading pairs are always referenced in the same manner.
-However, not all trading pairs are available on all exchanges. An
-error will occur if the specified trading pair is not trading
-on the exchange.
-
-<h2>Trading an Algorithm</h2>
-There is no special convention to follow when writing an
-algorithm for live trading. The same algorithm should work in
-backtest and live execution mode without modification.
-
-What differs are the arguments provided to the catalyst client or
-`run_algorithm()` interface. Here is example:
-
-```python
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace='my_algo_trading_xrp',
-    base_currency='btc'
-)
-```
-
-Here is the breakdown of the new arguments:
-* live: Boolean flag which enables live trading.
-* exchange_name: The name of the targeted exchange
-    (supported values: *bitfinex*, *bittrex*).
-* algo_namespace: A arbitrary label assigned to your algorithm for
-    data storage purposes.
-* base_currency: The base currency used to calculate the
-    statistics of your algorithm. Currently, the base currency of all
-    trading pairs of your algorithm must match this value.
-
-Here is a complete algorithm for reference:
-[Buy Low and Sell High](../catalyst/examples/buy_low_sell_high_live.py)

docs/source/development-guidelines.rst

@@ -1,21 +1,17 @@
 Development Guidelines
 ======================
-This page is intended for developers of Zipline, people who want to contribute to the Zipline codebase or documentation, or people who want to install from source and make local changes to their copy of Zipline.
+This page is intended for developers of Catalyst, people who want to contribute to the Catalyst codebase or documentation, or people who want to install from source and make local changes to their copy of Catalyst.
 
-All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We `track issues`__ on `GitHub`__ and also have a `mailing list`__ where you can ask questions.
-
-__ https://github.com/quantopian/zipline/issues
-__ https://github.com/
-__ https://groups.google.com/forum/#!forum/zipline
+All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We `track issues <https://github.com/enigmampc/catalyst/issues>`_ on `GitHub <https://github.com/enigmampc/catalyst>`_ and also have a `discord group <https://discord.gg/SJK32GY>`_ where you can ask questions.
 
 Creating a Development Environment
 ----------------------------------
 
-First, you'll need to clone Zipline by running:
+First, you'll need to clone Catalyst by running:
 
 .. code-block:: bash
 
-   $ git clone git@github.com:your-github-username/zipline.git
+   $ git clone git@github.com:enigmampc/catalyst.git
 
 Then check out to a new branch where you can make your changes:
 
@@ -23,15 +19,13 @@ Then check out to a new branch where you can make your changes:
 		
    $ git checkout -b some-short-descriptive-name
 
-If you don't already have them, you'll need some C library dependencies. You can follow the `install guide`__ to get the appropriate dependencies.
-
-__ install.html
+If you don't already have them, you'll need some C library dependencies. You can follow the `install guide <install.html>`_ to get the appropriate dependencies.
 
 The following section assumes you already have virtualenvwrapper and pip installed on your system. Suggested installation of Python library dependencies used for development:
 
 .. code-block:: bash
 
-   $ mkvirtualenv zipline
+   $ mkvirtualenv catalyst
    $ ./etc/ordered_pip.sh ./etc/requirements.txt
    $ pip install -r ./etc/requirements_dev.txt
    $ pip install -r ./etc/requirements_blaze.txt 
@@ -42,104 +36,39 @@ Finally, you can build the C extensions by running:
 
    $ python setup.py build_ext --inplace
 
-To finish, make sure `tests`__ pass.
+.. To finish, make sure `tests`__ pass.
 
-__ #style-guide-running-tests
+.. __ #style-guide-running-tests
 
-If you get an error running nosetests after setting up a fresh virtualenv, please try running
+.. If you get an error running nosetests after setting up a fresh virtualenv, please try running
 
-.. code-block:: bash
+.. code-block
 
-   # where zipline is the name of your virtualenv
-   $ deactivate zipline
-   $ workon zipline
+..   # where zipline is the name of your virtualenv
+..   $ deactivate zipline
+..   $ workon zipline
 
 
-Development with Docker
+.. Development with Docker
+.. -----------------------
+
+..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
+
+.. __ https://docs.docker.com/get-started/
+   
+Git Branching Structure
 -----------------------
 
-If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
+If you want to contribute to the codebase of Catalyst, familiarize yourself with our branching structure, a fairly standardized one for that matter, that follows what is documented in the following article: `A successful Git branching model <http://nvie.com/posts/a-successful-git-branching-model/>`_. To contribute, create your local branch and submit a Pull Request (PR) to the **develop** branch.
 
-__ https://docs.docker.com/get-started/
+.. image:: https://camo.githubusercontent.com/9bde6fb64a9542a572e0e2017cbb58d9d2c440ac/687474703a2f2f6e7669652e636f6d2f696d672f6769742d6d6f64656c4032782e706e67
 
-
-Style Guide & Running Tests
----------------------------
-
-We use `flake8`__ for checking style requirements and `nosetests`__ to run Zipline tests. Our `continuous integration`__ tools will run these commands.
-
-__ http://flake8.pycqa.org/en/latest/
-__ http://nose.readthedocs.io/en/latest/
-__ https://en.wikipedia.org/wiki/Continuous_integration
-
-Before submitting patches or pull requests, please ensure that your changes pass when running:
-
-.. code-block:: bash
-
-   $ flake8 zipline tests
-
-In order to run tests locally, you'll need `TA-lib`__, which you can install on Linux by running:
-
-__ https://mrjbq7.github.io/ta-lib/install.html
-
-.. code-block:: bash
-
-   $ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
-   $ tar -xvzf ta-lib-0.4.0-src.tar.gz
-   $ cd ta-lib/
-   $ ./configure --prefix=/usr
-   $ make
-   $ sudo make install
-
-And for ``TA-lib`` on OS X you can just run:
-
-.. code-block:: bash
-
-   $ brew install ta-lib
-
-Then run ``pip install`` TA-lib:
-
-.. code-block:: bash
-
-   $ pip install -r ./etc/requirements_talib.txt
-
-You should now be free to run tests:
-
-.. code-block:: bash
-		
-   $ nosetests
-
-
-Continuous Integration
-----------------------
-
-We use `Travis CI`__ for Linux-64 bit builds and `AppVeyor`__ for Windows-64 bit builds.
-
-.. note::
-
-   We do not currently have CI for OSX-64 bit builds. 32-bit builds may work but are not included in our integration tests.
-
-__ https://travis-ci.org/quantopian/zipline
-__ https://ci.appveyor.com/project/quantopian/zipline
-
-
-Packaging
----------
-To learn about how we build Zipline conda packages, you can read `this`__ section in our release process notes.
-
-__ release-process.html#uploading-conda-packages
-   
 Contributing to the Docs
 ------------------------
 
-If you'd like to contribute to the documentation on zipline.io, you can navigate to ``docs/source/`` where each `reStructuredText`__ (``.rst``) file is a separate section there. To add a section, create a new file called ``some-descriptive-name.rst`` and add ``some-descriptive-name`` to ``appendix.rst``. To edit a section, simply open up one of the existing files, make your changes, and save them.
-
-__ https://en.wikipedia.org/wiki/ReStructuredText
-
-We use `Sphinx`__ to generate documentation for Zipline, which you will need to install by running:
-
-__ http://www.sphinx-doc.org/en/stable/
+If you'd like to contribute to the documentation on enigmampc.github.io, you can navigate to ``docs/source/`` where each `reStructuredText <https://en.wikipedia.org/wiki/ReStructuredText>`_ file is a separate section there. To add a section, create a new file called ``some-descriptive-name.rst`` and add ``some-descriptive-name`` to ``index.rst``. To edit a section, simply open up one of the existing files, make your changes, and save them.
 
+We use `Sphinx <http://www.sphinx-doc.org/en/stable/>`_ to generate documentation for Catalyst, which you will need to install by running:
 
 .. code-block:: bash
 
@@ -149,7 +78,7 @@ To build and view the docs locally, run:
 
 .. code-block:: bash
 
-   # assuming you're in the Zipline root directory
+   # assuming you're in the Catalyst root directory
    $ cd docs
    $ make html
    $ {BROWSER} build/html/index.html
@@ -162,7 +91,7 @@ Standard prefixes to start a commit message:
 
 .. code-block:: text
 
-   BLD: change related to building Zipline
+   BLD: change related to building Catalyst
    BUG: bug fix
    DEP: deprecate something, or remove a deprecated object
    DEV: development tool or utility
@@ -172,15 +101,13 @@ Standard prefixes to start a commit message:
    REV: revert an earlier commit
    STY: style fix (whitespace, PEP8, flake8, etc)
    TST: addition or modification of tests
-   REL: related to releasing Zipline
+   REL: related to releasing Catalyst
    PERF: performance enhancements
 
 
 Some commit style guidelines:
 
-Commit lines should be no longer than `72 characters`__. The first line of the commit should include one of the above prefixes. There should be an empty line between the commit subject and the body of the commit. In general, the message should be in the imperative tense. Best practice is to include not only what the change is, but why the change was made.
-
-__ https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project
+Commit lines should be no longer than `72 characters <https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project>`_. The first line of the commit should include one of the above prefixes. There should be an empty line between the commit subject and the body of the commit. In general, the message should be in the imperative tense. Best practice is to include not only what the change is, but why the change was made.
 
 **Example:**
 
@@ -203,8 +130,6 @@ __ https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project
 Formatting Docstrings
 ---------------------
 
-When adding or editing docstrings for classes, functions, etc, we use `numpy`__ as the canonical reference.
-
-__ https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
+When adding or editing docstrings for classes, functions, etc, we use `numpy <https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_ as the canonical reference.
 
 

docs/source/features.rst

@@ -1,5 +1,61 @@
+Features
+========
+
+This page describes the features that Catalyst provides in the current version,
+and what is planned for future releases.
+
+Current Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+* Backtesting and live-trading modes to run your trading algorithms, with a 
+  seamless transition between the two.
+* Paper trading simulates order in live-trading mode.
+* Support for 3 exchanges: Bitfinex, Bittrex and Poloniex in both modes 
+  (backtesting and live-trading). Historical data for backtesting is provided 
+  with daily resolution for all three exchanges, and minute resolution for 
+  Bitfinex and Poloniex. No minute-resolution data is currently available for 
+  Bittrex. Refer to 
+  `Catalyst Market Coverage <https://www.enigma.co/catalyst/status>`_ for 
+  details.
+* Interface with over 90 exchanges available in live and paper trading modes.
+* Granular commission models which closely simulates each exchange fee
+  structure in backtesting and paper trading.
+* Standardized naming convention for all asset pairs trading on any exchange in 
+  the form ``{market_currency}_{base_currency}``. See 
+  :ref:`naming`.
+* Output of performance statistics based on Pandas DataFrames to integrate 
+  nicely into the existing PyData ecosystem.
+* Support for accessing multiple exchanges per algorithm, which opens the door
+  to cross-exchange arbitrage opportunities.
+* Support for running multiple algorithms on the same exchange independently of
+  one another. Catalyst performance tracker stores just enough data to allow 
+  algorithms to run independently while still sharing critical data through 
+  exchanges.
+* Benchmark defaults to Bitcoin price (btc_usdt in Poloniex exchange) for the 
+  purpose of comparing performance across trading algorithms. A custom benchmark
+  can be specified through ``set_benchmark()`` (but see 
+  `issue #86 <https://github.com/enigmampc/catalyst/issues/86>`_). 
+* Support for MacOS, Linux and Windows installations.
+* Support for Python2 and Python3.
+
+For additional details on the functionality added on recent releases, see the
+:doc:`Release Notes<releases>`.
+
+Upcoming features
+~~~~~~~~~~~~~~~~~
+
+* Additional datasets beyond pricing data (Dec. 2017)
+* API documentation (Jan. 2017)
+* Support for decentralized exchanges (Jan. 2017)
+* Support for data ingestion of community-contributed data sets (Jan. 2017)
+* Pipeline support (Jan. 2018)
+* Web UI (Q2 2018)
+
+
+ .. _naming:
+
 Naming Convention
-=================
+~~~~~~~~~~~~~~~~~
 
 Catalyst introduces a standardized naming convention for all asset pairs 
 trading on any exchange in the following form:

docs/source/index.rst

@@ -1,4 +1,4 @@
-.. include:: welcome.rst
+.. include:: ../../README.rst
 |
 |
 Table of Contents
@@ -9,9 +9,15 @@ Table of Contents
 
    install
    beginner-tutorial
-   naming-convention
+   live-trading
+   features
+   example-algos
+   utilities
+   videos
+   resources
+   development-guidelines
+   releases
 ..   bundles
-..   development-guidelines
 ..   appendix
 ..   release-process
-..   releases
+

docs/source/install.rst

@@ -1,6 +1,160 @@
 Install
 =======
 
+To get started with Catalyst, you will need to install it in your computer. 
+Like any other piece of software, Catalyst has a number of dependencies 
+(other software on which it depends to run) that you will need to install, as 
+well. We recommend using a software named ``Conda`` that will manage all 
+these dependencies for you, and set up the environment needed to get you up 
+and running as easily as possible. This is the recommended installation method
+for Windows, MacOS and Linux. See :ref:`Installing with Conda <conda>`.
+
+What conda does is create a pre-configured environment, and inside that 
+environment install Catalyst using ``pip``, Python's package manager. Thus, 
+as an alternative installation method for MacOS and Linux, you can install 
+Catalyst directly with ``pip`` (we recommend in combination with a virtual 
+environemnt). See :ref:`Installing with pip <pip>`.
+
+Regardless of the method, each operating system (OS), has its own 
+prerequisites, make sure to review the corresponding sections for your system:
+:ref:`Linux <linux>`, :ref:`MacOS <macos>` and :ref:`Windows <windows>`.
+
+.. _conda:
+
+Installing with ``conda``
+-------------------------
+
+The preferred method to install Catalyst is via the ``conda`` package manager, 
+which comes as part of Continuum Analytics' `Anaconda
+<http://continuum.io/downloads>`_ distribution.
+
+The primary advantage of using Conda over ``pip`` is that conda natively
+understands the complex binary dependencies of packages like ``numpy`` and
+``scipy``.  This means that ``conda`` can install Catalyst and its 
+dependencies without requiring the use of a second tool to acquire Catalyst's 
+non-Python dependencies.
+
+  For Windows, you will first need to install the *Microsoft Visual C++ 
+  Compiler for Python 2.7*. Follow the instructions on the :ref:`Windows 
+  <windows>` section and come back here.
+
+For instructions on how to install ``conda``, see the `Conda Installation
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
+you can install MiniConda, which is a smaller footprint (fewer packages and 
+smaller size) than its big brother Anaconda, but it still contains all the 
+main packages needed. To install MiniConda, you can follow these steps:
+
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
+   for your Operating System.
+2. Install MiniConda. See the `Installation Instructions 
+   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal 
+   window, which should print the list of packages installed with Conda.
+
+  For Windows, if you accepted the default installation options, you didn't 
+  check an option to add Conda to the PATH, so trying to run ``conda`` from
+  a regular ``Command Prompt`` will result in the following error: ``'conda' 
+  is no recognized as an internal or external command, operatble program or 
+  batch file``. That's to be expected. You will nee to launch an ``Anaconda 
+  Prompt`` that was added at installation time to your list of programs 
+  available from the Start menu. 
+
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+
+     To download, simply click on the 'Raw' button and save the file locally 
+     to a folder you can remember. Make sure that the file gets saved with the
+     ``.yml`` extension, and nothing like a ``.txt`` file or anything else.
+
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
+   saved the above ``python2.7-environment.yml`` file.
+
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new 
+   session to run Catalyst):
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+5. Verify that Catalyst is install correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
+above failed for any reason, you can try setting up the environment manually 
+with the following steps:
+
+1. If the above installation failed, and you have a partially set up catalyst
+   environment, remove it first. If you are starting from scratch, proceed to 
+   step #2:
+
+   .. code-block:: bash
+
+      conda env remove --name catalyst
+
+2. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy zlib
+
+3. Activate the environment:
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+4. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+5. Verify that Catalyst is installed correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst properly installed.
+
+.. _pip:
+
 Installing with ``pip``
 -----------------------
 
@@ -9,148 +163,47 @@ Python package.
 
 There are two reasons for the additional complexity:
 
-1. Catalyst ships several C extensions that require access to the CPython C API.
-   In order to build the C extensions, ``pip`` needs access to the CPython
-   header files for your Python installation.
+1. Catalyst ships several C extensions that require access to the CPython C 
+   API. In order to build the C extensions, ``pip`` needs access to the 
+   CPython header files for your Python installation.
 
 2. Catalyst depends on `numpy <http://www.numpy.org/>`_, the core library for
    numerical array computing in Python.  Numpy depends on having the `LAPACK
    <http://www.netlib.org/lapack>`_ linear algebra routines available.
 
-Because LAPACK and the CPython headers are non-Python dependencies, the correct
-way to install them varies from platform to platform.  If you'd rather use a
-single tool to install Python and non-Python dependencies, or if you're already
-using `Anaconda <http://continuum.io/downloads>`_ as your Python distribution,
-you can skip to the :ref:`Installing with Conda <conda>` section.
+Because LAPACK and the CPython headers are non-Python dependencies, the 
+correctway to install them varies from platform to platform.  If you'd rather 
+use a single tool to install Python and non-Python dependencies, or if you're 
+already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
+distribution, refer to the :ref:`Installing with Conda <conda>` section.
 
-Once you've installed the necessary additional dependencies (see below for
-your particular platform), you should be able to simply run
+Once you've installed the necessary additional dependencies for your system 
+(see below for your particular platform: :ref:`Linux`, :ref:`MacOS` or 
+:ref:`Windows`), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install enigma-catalyst
+   $ pip install enigma-catalyst matplotlib
+
+Note that in the command above we install two different packages. The second 
+one, ``matplotlib`` is a visualization library. While it's not strictly 
+required to run catalyst simulations or live trading, it comes in very handy
+to visualize the performance of your algorithms, and for this reason we 
+recommend you install it, as well.
 
 If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
 Python`_ provides an `excellent tutorial on virtualenv
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a summarized
-version:
+<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a 
+summarized version:
 
 .. code-block:: bash
 
+   $ pip install virtualenv
    $ virtualenv catalyst-venv
    $ source ./catalyst-venv/bin/activate
-   $ pip install enigma-
-
-Though not required by Catalyst directly, our example algorithms use matplotlib 
-to visually display the results of the trading algorithms. If you wish to run 
-any examples or use matplotlib during development, it can be installed using:
-
-.. code-block:: bash
-
-    $ pip install matplotlib
-
-GNU/Linux
-~~~~~~~~~
-
-On `Debian-derived`_ Linux distributions, you can acquire all the necessary
-binary dependencies from ``apt`` by running:
-
-.. code-block:: bash
-
-   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
-
-On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
-following should be sufficient to acquire the necessary additional
-dependencies:
-
-.. code-block:: bash
-
-   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
-
-On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
-
-.. code-block:: bash
-
-   $ pacman -S lapack gcc gcc-fortran pkg-config
-
-.. Commenting it out until Catalyst fully supports Python 3.X
-..
-.. There are also AUR packages available for installing `Python 3.4
-.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
-.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
-.. Python 2 is also installable via:
-
-.. 
-
-..   $ pacman -S python2
-
-OSX
-~~~
-
-The version of Python shipped with OSX by default is generally out of date, and
-has a number of quirks because it's used directly by the operating system.  For
-these reasons, many developers choose to install and use a separate Python
-installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
-to `Installing Python on OSX <http://docs.python-guide.org/en/latest/>`_, which
-explains how to install Python with the `Homebrew`_ manager.
-
-Assuming you've installed Python with Homebrew, you'll also likely need the
-following brew packages:
-
-.. code-block:: bash
-
-   $ brew install freetype pkg-config gcc openssl
-
-OSX + virtualenv + matplotlib
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-A note about using matplotlib in virtual enviroments on OSX: it may be necessary to run
-
-.. code-block:: bash
-
-   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
-
-in order to override the default ``macosx`` backend for your system, which may not 
-be accessible from inside the virtual environment. This will allow Catalyst to open 
-matplotlib charts from within a virtual environment, which is useful for displaying 
-the performance of your backtests.  To learn more about matplotlib backends, please refer to the
-`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
-
-
-Windows
-~~~~~~~
-
-In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
-<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This package 
-contains the compiler and the set of system headers necessary for producing 
-binary wheels for Python 2.7 packages. If it's not already in your system, download
-it and install it before proceeding to the next step.
-
-For windows, the easiest and best supported way to install Catalyst is to use
-:ref:`Conda <conda>`.
-
-Amazon Linux AMI
-~~~~~~~~~~~~~~~~
-
-The packages ``pip`` and ``setuptools`` that come shipped by default are very outdated. 
-Thus, you first need to run:
-
-.. code-block:: bash
-
-   pip install --upgrade pip setuptools
-
-The default installation is also missing the C and C++ compilers, which you install by:
-
-.. code-block:: bash
-
-   sudo yum install gcc gcc-c++
-
-Then you should follow the regular installation instructions outlined at the beginning 
-of this page.
-
+   $ pip install enigma-catalyst matplotlib
 
 Troubleshooting ``pip`` Install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -174,17 +227,24 @@ Troubleshooting ``pip`` Install
 ----
 
 **Issue**: 
-   Package enigma-catalyst cannot still be found, even after upgrading pip (see above), with an error similar to:
+   Package enigma-catalyst cannot still be found, even after upgrading pip 
+   (see above), with an error similar to:
 
    .. code-block:: bash
 
       Downloading/unpacking enigma-catalyst
-      Could not find a version that satisfies the requirement enigma-catalyst (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 0.2.dev1, 0.1.dev8, 0.1.dev6)
+      Could not find a version that satisfies the requirement enigma-catalyst 
+      (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 
+      0.2.dev1, 0.1.dev8, 0.1.dev6)
       Cleaning up...
       No distributions matching the version for enigma-catalyst
 
 **Solution**:
-   In some systems (this error has been reported in Ubuntu), pip is configured to only find stable versions by default. Since Catalyst is in alpha version, pip cannot find a matching version that satisfies the installation requirements. The solution is to include the `--pre` flag to include pre-release and development versions:
+   In some systems (this error has been reported in Ubuntu), pip is configured 
+   to only find stable versions by default. Since Catalyst is in alpha 
+   version, pip cannot find a matching version that satisfies the installation 
+   requirements. The solution is to include the `--pre` flag to include 
+   pre-release and development versions:
 
    .. code-block:: bash
 
@@ -220,121 +280,185 @@ Troubleshooting ``pip`` Install
 ----
 
 **Issue**: 
-   Installation fails with error: ``fatal error: Python.h: No such file or directory``
+   Installation fails with error: 
+   ``fatal error: Python.h: No such file or directory``
 
 **Solution**: 
-   Some systems (this issue has been reported in Ubuntu) require `python-dev` for the proper build and installation of package dependencies. The solution is to install python-dev, which is independent of the virtual environment. In Ubuntu, you would need to run:
+   Some systems (this issue has been reported in Ubuntu) require `python-dev` 
+   for the proper build and installation of package dependencies. The solution 
+   is to install python-dev, which is independent of the virtual environment. 
+   In Ubuntu, you would need to run:
 
    .. code-block:: bash
 
       sudo apt-get install python-dev
 
 
-.. _conda:
+.. _linux:
 
-Installing with ``conda``
--------------------------
+GNU/Linux Requirements
+----------------------
 
-Another way to install Catalyst is via the ``conda`` package manager, which
-comes as part of Continuum Analytics' `Anaconda
-<http://continuum.io/downloads>`_ distribution.
+On `Debian-derived`_ Linux distributions, you can acquire all the necessary
+binary dependencies from ``apt`` by running:
 
-The primary advantage of using Conda over ``pip`` is that conda natively
-understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Catalyst and its dependencies
-without requiring the use of a second tool to acquire Catalyst's non-Python
-dependencies.
+.. code-block:: bash
 
-For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, you 
-can install MiniConda, which is a smaller footprint (fewer packages and smaller 
-size) than its big brother Anaconda, but it still contains all the main packages 
-needed. To install MiniConda, you can follow these steps:
+   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
 
-1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 for 
-   your Operating System.
-2. Install MiniConda. See the `Installation Instructions <https://conda.io/docs/user-guide/install/index.html>`_
-   if you need help.
-3. Ensure the correct installation by running ``conda list`` in a Terminal window,
-   which should print the list of packages installed with Conda.
+On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
+following should be sufficient to acquire the necessary additional
+dependencies:
 
-Once either Conda or MiniConda has been set up you can install Catalyst:
+.. code-block:: bash
 
-1. Download the file `python2.7-environment.yml <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
-2. Open a Terminal window and enter [``cd/dir``] into the directory where you saved
-   the above ``python2.7-environment.yml`` file.
-3. Install using this file. This step can take about 5-10 minutes to install.
+   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
 
-   .. code-block:: bash
+On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
-      conda env create -f python2.7-environment.yml
+.. code-block:: bash
 
-4. Activate the environment (which you need to do every time you start a new session
-   to run Catalyst):
+   $ pacman -S lapack gcc gcc-fortran pkg-config
 
-   **Linux or OSX:**
+.. Commenting it out until Catalyst fully supports Python 3.X
+..
+.. There are also AUR packages available for installing `Python 3.4
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-   .. code-block:: bash
+.. 
 
-      source activate catalyst
+..   $ pacman -S python2
 
-   **Windows:**
+Amazon Linux AMI Notes
+~~~~~~~~~~~~~~~~~~~~~~
 
-   .. code-block:: bash
+The packages ``pip`` and ``setuptools`` that come shipped by default are very 
+outdated. Thus, you first need to run:
 
-      activate catalyst
+.. code-block:: bash
 
-Congratulations! You now have Catalyst installed.
+   pip install --upgrade pip setuptools
 
-Troubleshooting ``conda`` Install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The default installation is also missing the C and C++ compilers, which you 
+install by:
 
-If the command  ``conda env create -f python2.7-environment.yml`` in step 3 above failed 
-for any reason, you can try setting up the environment manually with the following steps:
+.. code-block:: bash
 
-1. Create the environment:
+   sudo yum install gcc gcc-c++
 
-   .. code-block:: bash
+Then you should follow the regular installation instructions outlined at the 
+beginning of this page.
 
-      conda create --name catalyst python=2.7 scipy
 
-2. Activate the environment:
+.. _MacOS:
 
-   **Linux or OSX:**
+MacOS Requirements
+------------------
 
-   .. code-block:: bash
+The version of Python shipped with MacOS by default is generally out of date, 
+and has a number of quirks because it's used directly by the operating system.
+For these reasons, many developers choose to install and use a separate Python
+installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
+to `Installing Python on MacOS <http://docs.python-guide.org/en/latest/>`_, 
+which explains how to install Python with the `Homebrew`_ manager.
 
-      source activate catalyst
+Assuming you've installed Python with Homebrew, you'll also likely need the
+following brew packages:
 
-   **Windows:**
+.. code-block:: bash
 
-   .. code-block:: bash
+   $ brew install freetype pkg-config gcc openssl
 
-      activate catalyst
+MacOS + virtualenv + matplotlib
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-3. Install the Catalyst inside the environment:
+A note about using matplotlib in virtual enviroments on MacOS: it may be 
+necessary to run
 
-   .. code-block:: bash
+.. code-block:: bash
 
-      pip install enigma-catalyst matplotlib
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+
+in order to override the default ``MacOS`` backend for your system, which 
+may not be accessible from inside the virtual environment. This will allow 
+Catalyst to open matplotlib charts from within a virtual environment, which 
+is useful for displaying the performance of your backtests.  To learn more 
+about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+.. _windows:
+
+Windows Requirements
+--------------------
+
+In Windows, you will first need to install the `Microsoft Visual C++ Compiler 
+for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
+package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python 2.7 packages. If it's not already in your 
+system, download it and install it before proceeding to the next step.
+
+Once you have the above compiler installed, the easiest and best supported way 
+to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't 
+any problems installing the compiler, jump to the :ref:`Conda <conda>` section, 
+otherwise keep on reading to troubleshoot the C++ compiler installtion.
+
+Some problems we have encountered installing the **Visual C++ Compiler** 
+mentioned above are as follows:
+
+- **The system administrator has set policies to prevent this installation**.
+  
+  In some systems, there is a default *Windows Software Restriction* policy 
+  that prevents the installation of some software packages like this one. 
+  You'll have to change the Registry to circumvent this:
+
+  - Click ``Start``, and search for ``regedit`` and launch the 
+    ``Registry Editor``
+  - Navigate to the following folder:
+    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
+  - If the last folder does not exist, create it by right-clicking on the 
+    parent folder and choosing -> ``New`` -> ``Key`` and typing ``Installer``
+  - If there is an entry for ``DisableMSI``, set the Value data to 0.
+  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
+    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
+    default you get 0 as the Value Data)
+
+|
+- **The installer has encountered an unexpected error installing this package. 
+  This may indicate a problem with this package. The error code is 2503.**
+
+  We have observed this when trying to install a package without enough 
+  administrator permissions. Even when you are logged in as an Administrator, 
+  you have to explictily install this package with administrator privileges:
+
+  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
+  - Right click on it and choose ``Run as administrator``
+  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
+  - Run ``msiexec /i VCForPython27.msi``
 
 Getting Help
 ------------
 
-If after following the instructions above, and going through the *Troubleshooting* sections, 
-you still experience problems installing Catalyst, you can seek additional help through the 
-following channels:
+If after following the instructions above, and going through the 
+*Troubleshooting* sections, you still experience problems installing Catalyst,
+you can seek additional help through the following channels:
 
-- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over the #catalyst_dev 
-  channel where many other users (as well as the project developers) hang out, and can assist 
-  you with your particular issue. The more descriptive and the more information you can provide, 
-  the easiest will be for others to help you out.
+- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
+  the #catalyst_dev channel where many other users (as well as the project 
+  developers) hang out, and can assist you with your particular issue. The 
+  more descriptive and the more information you can provide, the easiest will 
+  be for others to help you out.
 
 - Report the problem you are experiencing on our 
-  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ following the guidelines 
-  provided therein. Before you do so, take a moment to browse through all `previous reported issues
-  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ in the likely case
-  that someone else experienced that same issue before, and you get a hint on how to solve it.
+  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ 
+  following the guidelines provided therein. Before you do so, take a moment 
+  to browse through all `previous reported issues
+  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ 
+  in the likely case that someone else experienced that same issue before, 
+  and you get a hint on how to solve it.
 
 
 .. _`Debian-derived`: https://www.debian.org/misc/children-distros

docs/source/live-trading.rst

@@ -0,0 +1,124 @@
+Live Trading
+============
+This document explains how to get started with live trading.
+
+Supported Exchanges
+^^^^^^^^^^^^^^^^^^^
+Catalyst can trade against these exchanges:
+
+- Bitfinex, id= ``bitfinex``
+- Bittrex, id= ``bittrex``
+- Poloniex, id= ``poloniex``
+
+Authentication
+^^^^^^^^^^^^^^
+Most exchanges require token key/secret combination for authentication. By
+convention, Catalyst uses an ``auth.json`` file to hold this data.
+
+This example illustrates the convention using the *Bitfinex* exchange.
+Here is how to generate key and secret values for the Bitfinex exchange:
+https://docs.bitfinex.com/v1/docs/api-access. Most exchanges follow
+a similar process.
+
+The auth.json file:
+
+.. code-block:: json
+
+  {
+    "name": "bitfinex",
+    "key": "my-key",
+    "secret": "my-secret"
+  }
+
+
+The file goes here: ``~/.catalyst/data/exchanges/bitfinex/auth.json``
+
+Note that the `bitfinex` part in the directory above corresponds to the id of the Bitfinex
+exchange as defined in the "Supported Exchanges" section above.
+Attempting to run an algorithm where the targeted exchange is missing
+its ``auth.json`` file will create the directory structure and create an empty
+auth.json file, but will result in an error.
+
+Currency Symbols
+^^^^^^^^^^^^^^^^
+Catalyst introduces a universal convention to reference
+trading pairs and individual currencies. This
+is required to ensure that the ``symbol()`` api predictably
+returns the correct asset regardless of the targeted exchange.
+
+Exchanges tend to use their own convention to represent currencies
+(e.g. XBT and BTC both represent Bitcoin on different exchanges).
+Trading pairs are also inconsistent. For example, Bitfinex
+puts the market currency before the base currency without a
+separator, Bittrex puts the base currency first and uses a dash
+seperator.
+
+Here is the Catalyst convention:
+
+*[Market Currency]_[Base Currency]* all lowercase.
+
+Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
+
+Here are some examples:
+
+.. code-block:: json
+
+  # With Bitfinex
+  bitcoin_usd_asset = symbol('btc_usd')
+  ethereum_bitcoin_asset = symbol('eth_btc')
+
+  # With Bittrex
+  ethereum_bitcoin_asset = symbol('eth_btc')
+  neo_ethereum_asset = symbol('neo_eth)
+
+Note that the trading pairs are always referenced in the same manner.
+However, not all trading pairs are available on all exchanges. An
+error will occur if the specified trading pair is not trading
+on the exchange. To check which currency pairs are available on each 
+of the supported exchanges, see `Catalyst Market Coverage <https://www.enigma.co/catalyst/status`_.
+
+Trading an Algorithm
+^^^^^^^^^^^^^^^^^^^^
+There is no special convention to follow when writing an
+algorithm for live trading. The same algorithm should work in
+backtest and live execution mode without modification.
+
+What differs are the arguments provided to the catalyst client or
+`run_algorithm()` interface. Here is the same example in both interfaces:
+
+.. code-block:: bash
+
+  catalyst live -f my_algo_code -x bitfinex -c btc -n my_algo_name 
+
+.. code-block:: python
+
+  run_algorithm(
+      initialize=initialize,
+      handle_data=handle_data,
+      analyze=analyze,
+      exchange_name='bitfinex',
+      live=True,
+      algo_namespace='my_algo_name',
+      base_currency='btc'
+  )
+
+
+Here is the breakdown of the new arguments:
+
+- ``live``: Boolean flag which enables live trading.
+- ``capital_base``: The amount of base_currency assigned to the strategy.
+  It has to be lower or equal to the amount of base currency available for
+  trading on the exchange. For illustration, order_target_percent(asset, 1)
+  will order the capital_base amount specified here of the specified asset.
+- ``exchange_name``: The name of the targeted exchange
+  (supported values: *bitfinex*, *bittrex*).
+- ``algo_namespace``: A arbitrary label assigned to your algorithm for
+  data storage purposes.
+- ``base_currency``: The base currency used to calculate the
+  statistics of your algorithm. Currently, the base currency of all
+  trading pairs of your algorithm must match this value.
+- ``simulate_orders``: Enables the paper trading mode, in which orders are
+  simulated in Catalyst instead of processed on the exchange.
+
+Here is a complete algorithm for reference:
+`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_

docs/source/releases.rst

@@ -2,24 +2,287 @@
 Release Notes
 =============
 
-.. include:: whatsnew/1.1.1.txt
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-12-12
 
-.. include:: whatsnew/1.1.0.txt
+Bug Fixes
+~~~~~~~~~
 
-.. include:: whatsnew/1.0.2.txt
+- Fixed issue with fetching assets with daily frequency
 
-.. include:: whatsnew/1.0.1.txt
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
 
-.. include:: whatsnew/1.0.0.txt
+Bug Fixes
+~~~~~~~~~
 
-.. include:: whatsnew/0.9.0.txt
+- Fixed issue with fetching assets with daily frequency
+- Changed Poloniex interface (should solve :issue:`95` and :issue:`94`)
+- Solved issue with overriding commission and slippage (:issue:`87`)
+- Fixed inefficiency with Bittrex current prices (:issue:`76`)
 
-.. include:: whatsnew/0.8.4.txt
+Build
+~~~~~
+- Integrated with CCXT
+- Added paper trading capability (`simulate_orders=True` param in live mode)
+- More granular commissions (:issue:`82`)
+- Added market orders in live mode (:issue:`81`)
 
-.. include:: whatsnew/0.8.3.txt
+Version 0.3.9
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
 
-.. include:: whatsnew/0.8.0.txt
+Bug Fixes
+~~~~~~~~~
 
-.. include:: whatsnew/0.7.0.txt
+- Fixed sortino warning issues (:issue:`77`)
+- Adjusted computation of last candle of data.history (:issue:`71`)
 
-.. include:: whatsnew/0.6.1.txt
+Build
+~~~~~
+- Added capital_base parameter to live mode to limit cash (:issue:`79`)
+- Added support for csv ingestion (:issue:`65`)
+- Improved cash display in running stats (:issue:`80`)
+
+
+Version 0.3.8
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed a warning filter issue introduced with the latest release
+
+Version 0.3.7
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an SSL cert issue (:issue:`64`)
+- Fixed cumulative stats warnings (:issue:`63`)
+- Disabled auto-ingestion because of unresolved caching issues (:issue:`47`)
+- Standardized live-trading stats (:issue:`61`)
+
+Build
+~~~~~
+
+- Added a mean-reversion sample algo
+- Added minutely stats in the analyze() function (:issue:`62`)
+- Added specificity to some error messages
+
+Version 0.3.6
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an issue with single bar data.history() (:issue:`55`)
+
+Version 0.3.5
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
+
+Bug Fixes
+~~~~~~~~~
+
+- Added workaround for: KeyError: Timestamp error (:issue:`53`)
+
+Version 0.3.4
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-2
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with auto-ingestion of minute data (:issue:`47`)
+- Fixed issue with sell orders in backtesting
+- Fixed data frequency issues with data.history() in backtesting
+- Fixed an issue with can_trade()
+- Reduced the commission and slippage values to account for lower volume
+  transactions
+
+Build
+~~~~~
+
+- Added more unit tests
+
+Documentation
+~~~~~~~~~~~~~
+
+- Improved installation notes for Windows C++ compiler and Conda
+- Addition of
+  `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
+- Addition of
+  `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
+- Addition of
+  `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
+- Addition of
+  `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
+- Addition of `Development Guidelines
+  <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
+- Addition of
+  `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
+- Updated code docstrings
+
+
+Version 0.3.3
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-26
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix missing -x in ingest-exchange
+- Fix issue with daily chunks end date (data bundles)
+- Fix issue in the prepare_chunk logic (data bundles)
+
+Build
+~~~~~
+
+- Added data validation unit tests
+
+
+Version 0.3.2
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-25
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix to work with empty data bundles
+- Fix Windows path of ``$HOME/.catalyst`` folder
+- Fix ``etc/python2.7-environment.yml`` for Windows Conda install
+- Fix hash method to create sid numbers compatible across platforms
+- Fix an issue with asset date in chunks
+
+Build
+~~~~~
+
+- Python3 adjustments
+- Added method to clean bundle folders, and remove symbols.json
+- Implemented and improved unit tests
+
+
+Version 0.3.1
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-22
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed OS-dependent path issue in data bundle
+- Changed handling of empty ``auth.json``, instead of throwing an error for
+  missing file
+- Updated ``etc/python2.7-environment.yml`` to work with Catalyst version 0.3
+- Updated ``catalyst/examples/buy_and_hodl.py``  and
+  ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
+
+
+Version 0.3
+^^^^^^^^^^^
+**Release Date**: 2017-10-20
+
+- Standardized live and backtesting syntax
+- Added a repository for historical data
+- Added supported for multiple exchanges per algorithm
+- Added a standardized dictionary of symbols for each exchange
+- Added auto-ingestion of bundle data while backtesting
+- Bug fixes
+
+
+Version 0.2.dev5
+^^^^^^^^^^^^^^^^
+**Release Date**: 2017-10-03
+
+- Fixes bug in data.history function that was formatting 'volume' data as
+  integers, now they are returned as floats with up to 9 decimals of precision.
+  Data bundles redone.
+
+Version 0.2.dev4
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-20
+
+- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal
+  places. Pricing resolution of daily data remains set to 9 decimal places.
+- The current data bundle takes 340MB compressed for download, and 460MB
+  uncompressed on disk for Catalyst to use.
+
+Version 0.2.dev3
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-20
+
+- 1-minute resolution OHLCV data bundle for backtesting from Poloniex exchange
+- Implementation of trading of fractional crypto assets (i.e. 0.01 BTC)
+- Minimum trade size of a coin can be configured on a per-coin basis, defaults
+  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger
+  amounts, which will impact live trading)
+- Increased pricing resolution from 3 to 9 decimal places
+- The current data bundle takes 40MB compressed for download, and 99MB
+  uncompressed on disk for Catalyst to use.
+
+Version 0.2.dev2
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-07
+
+- Fix path issue
+
+Version 0.2.dev1
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-03
+
+- Implementation of live trading:
+
+  - Comprehensive trading functionality against exchanges Bitfinex and Bittrex.
+  - Support for all trading pairs available on each exchange.
+  - Multiple algorithms can trade simultaneously against a single exchange
+    using the same account.
+  - Each algorithm has a persisted state (i.e. algorithm can be stopped and
+    restarted preserving the state without data loss) that tracks all open
+    orders, executed transactions and portfolio positions.
+
+- Minute by minute portfolio performance metrics.
+
+  - Daily summary performance statistics compatible with pyfolio, a Python
+    library for performance and risk analysis of financial portfolios
+
+Version 0.1.dev9
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-08-28
+
+- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex
+  exchange directly
+- Change of bundle storage provider from Dropbox to AWS
+- Fix issue with 1/1000 scaling issue of prices in bundle
+
+Version 0.1.dev8
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-08-18
+
+- Fixes issue in the creation of bundles (:issue:`27`)
+
+
+Version 0.1.dev7
+^^^^^^^^^^^^^^^^
+- Fixes issues in empty benchmark (:issue:`16`)
+- Fixes issue of normalizing timestamps before comparison (:issue:`24`)
+- Generic data bundles
+- CLI UI improvements
+
+Version 0.1.dev6
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-07-13
+
+- Initial public release

docs/source/resources.rst

@@ -0,0 +1,26 @@
+Resources
+=========
+
+-  `Catalyst Whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ 
+
+
+Related 3rd Party APIs
+^^^^^^^^^^^^^^^^^^^^^^
+
+- `Zipline <http://www.zipline.io/appendix.html>`_ is a Pythonic Algorithmic 
+  Trading Library, and the project Catalyst forked off in the spring of 2017.
+- `Quantopian <https://www.quantopian.com/help>`_ provides a platform for 
+  freelance quantitative analysts develop, test, and use trading algorithms to 
+  buy and sell securities. They aim to create a crowd-sourced hedge fund by
+  fostering their community of freelance traders. Quantopian's backtesting and 
+  live-trading engine is powered by *Zipline*.
+- `Pandas <https://pandas.pydata.org/pandas-docs/stable/api.html>`_ is a Python
+  library providing high-performance, easy-to-use data structures and data 
+  analysis tools. Catalyst relies heavily on pandas, and many API functions 
+  return data as Pandas dataframes.
+- `Numpy <https://docs.scipy.org/doc/numpy/reference/>`_ is the fundamental 
+  package for scientific computing with Python. Some of the data computation 
+  that your algorithms will need, will be optimized leveraging Numpy.
+- `Matplotlib <https://matplotlib.org/1.5.3/api/index.html>`_ is a Python 2D 
+  plotting library that many of examples rely on to plot the performance of 
+  trading algorithms

docs/source/utilities.rst

@@ -0,0 +1,149 @@
+Utilities
+=========
+
+This section covers a variety of utilites that provide complimentary 
+functionality to your trading algorithms. These are code snippets that you can
+add to any algorithm to add the desired functionality.
+
+If you are looking for example trading algorithms, see the corresponding section.
+
+Output to CSV file
+~~~~~~~~~~~~~~~~~~
+
+Add this script to the analyze method to create and save a CSV file with the 
+results from the trading algorithm. This file will include the default 
+parameters of the results DataFrame plus any recorded variables and will be 
+saved in the same location where your trading algorithm is saved. The exact 
+script that you need to use depends on the interface that you are using to run 
+your trading algorithm, which could be the CLI or a Python Interpreter.
+
+1. Script to use with CLI:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import sys
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(basename(sys.argv[3]))[0]
+          results.to_csv(filename + '.csv')
+
+2. Script to use with Python Interpreter:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(os.path.basename(__file__))[0]
+          results.to_csv(filename + '.csv')
+
+Extracting market data
+~~~~~~~~~~~~~~~~~~~~~~
+
+Use this script to save the price and volume data of one cryptoasset in a CSV 
+file, which will be saved in the same location and with the same name as your 
+Python file. To get custom data, simply modify the asset's symbol and the dates. 
+Run this script directly from your development environment: python scriptname.py, 
+where the contents of 'scriptname.py' are as follows. Two different version are 
+provided as an example for daily- and minute-resolution data respectively:
+
+Simpler case for daily data
+
+.. code-block:: python
+
+   import os
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+   def initialize(context):
+      # Portfolio assets list
+      context.asset = symbol('btc_usdt') # Bitcoin on Poloniex
+                          
+   def handle_data(context, data):
+      # Variables to record for a given asset: price and volume
+      price = data.current(context.asset, 'price')
+      volume = data.current(context.asset, 'volume')
+      record(price=price, volume=volume)
+      
+   def analyze(context=None, results=None):
+      
+      # Generate DataFrame with Price and Volume only
+      data = results[['price','volume']]
+      
+      # Save results in CSV file
+      filename = os.path.splitext(os.path.basename(__file__))[0]
+      data.to_csv(filename + '.csv')
+      
+   ''' Bitcoin data is available on Poloniex since 2015-3-1. 
+       Dates vary for other tokens. In the example below, we choose the 
+       full month of July of 2017.
+   '''
+   start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data, 
+                           analyze=analyze, 
+                           start=start, 
+                           end=end, 
+                           exchange_name='poloniex', 
+                           capital_base=10000, 
+                           base_currency = 'usdt')
+
+More versatile case for minute data
+
+.. code-block:: python
+
+   import os
+   import csv
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+
+   def initialize(context):
+   # Portfolio assets list
+   context.asset     = symbol('btc_usdt') # Bitcoin on Poloniex
+
+   # Creates a .CSV file with the same name as this script to store results
+   context.csvfile   = open(os.path.splitext(
+                            os.path.basename(__file__))[0]+'.csv', 'w+')
+   context.csvwriter = csv.writer(context.csvfile)
+
+   def handle_data(context, data):
+   # Variables to record for a given asset: price and volume
+   # Other options include 'open', 'high', 'open', 'close'
+   # Please note that 'price' equals 'close'
+   date   = context.blotter.current_dt     # current time in each iteration
+   price  = data.current(context.asset, 'price')
+   volume = data.current(context.asset, 'volume')
+
+   # Writes one line to CSV on each iteration with the chosen variables    
+   context.csvwriter.writerow([date,price,volume])
+
+   def analyze(context=None, results=None):
+   # Close open file properly at the end
+   context.csvfile.close()
+
+
+   # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
+   start = datetime(2017, 7, 30, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data,
+                           analyze=analyze, 
+                           start=start, 
+                           end=end,
+                           exchange_name='poloniex', 
+                           data_frequency='minute',
+                           base_currency ='usdt', 
+                           capital_base=10000 )

docs/source/videos.rst

@@ -0,0 +1,58 @@
+Videos
+======
+
+
+Installation: MacOS
+-------------------
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/ZnsslmHljvw" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Installation: Windows
+---------------------
+
+Where things go smoothly:
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/H8HqcEbZmkk" frameborder="0" allowfullscreen></iframe>
+
+|
+Where things don't:
+
+.. raw:: html
+   
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/qLkQcWlUBy8" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Backtesting a Strategy
+----------------------
+
+This is the first video of a two-part series on using Catalyst for algorithmic 
+trading. This video implements a simple momentum strategy based on 
+`mean reversion <example-algos.html#mean-reversion>`_: when the cryptoasset
+goes up quickly, we’re going to buy; when it goes down quickly, we’re going to 
+sell. Hopefully, we’ll ride the waves.
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/JOBRwst9jUY" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Live Trading a Strategy
+-----------------------
+
+This is the second part of the two-part series on using Catalyst for algorithmic
+trading. Having backtested `our strategy <example-algos.html#mean-reversion>`_ 
+in the previous video, we now take it to trade live against the Bittrex exchange.
+
+.. raw:: html
+
+	<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
+|
+|

docs/source/welcome.rst

@@ -1,28 +0,0 @@
-.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
-|
-Catalyst is a data-driven crypto investment platform. It supports both 
-backtesting and live-trading in a number of different crypto-exchanges. 
-Catalyst empowers users to share and curate data and build profitable, 
-data-driven investment strategies.
-
-Features
-========
-
--  Ease of use: Catalyst tries to get out of your way so that you can 
-   focus on algorithm development. See 
-   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
-   provided.
--  Support for several of the top crypto-exchanges by trading volume:
-   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
-   and `Poloniex <https://www.poloniex.com>`_. 
--  Secure: You and only you have access to each exchange API keys for your accounts.
--  Input of historical pricing data of all crypto-assets by exchange, 
-   with daily and minute resolution. See 
-   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
--  Backtesting and live-trading functionality, with a seamless transition
-   between the two modes.
--  Output of performance statistics are based on Pandas DataFrames to 
-   integrate nicely into the existing PyData eco-system.
--  Statistic and machine learning libraries like matplotlib, scipy, 
-   statsmodels, and sklearn support development, analysis, and 
-   visualization of state-of-the-art trading systems.

etc/python2.7-environment.yml

@@ -3,25 +3,24 @@ channels:
 - defaults
 dependencies:
 - certifi=2016.2.28=py27_0
-- libgfortran=3.0.0=1
-- mkl=2017.0.3=0
+- mkl=2017.0.3
 - numpy=1.13.1=py27_0
-- openssl=1.0.2l=0
+- openssl=1.0.2l
 - pip=9.0.1=py27_1
-- python=2.7.13=0
-- readline=6.2=2
+- python=2.7.13
 - scipy=0.19.1=np113py27_0
 - setuptools=36.4.0=py27_1
-- sqlite=3.13.0=0
-- tk=8.5.18=0
+- sqlite=3.13.0
+- tk=8.5.18
 - wheel=0.29.0=py27_0
-- zlib=1.2.11=0
+- zlib=1.2.11
 - pip:
   - alembic==0.9.6
   - backports.functools-lru-cache==1.4
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
+  - ccxt==1.10.319
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0

etc/requirements.txt

@@ -80,3 +80,6 @@ empyrical==0.2.1
 
 tables==3.3.0
 
+#Catalyst dependencies
+ccxt==1.10.283
+boto3==1.4.8

tests/exchange/base.py

@@ -1,4 +1,3 @@
-import unittest
 from abc import ABCMeta, abstractmethod
 
 

tests/exchange/test_bcolz.py

@@ -0,0 +1,150 @@
+import shutil
+import random
+import tempfile
+import pandas as pd
+
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
+    BcolzExchangeBarReader
+
+from catalyst.exchange.bundle_utils import get_df_from_arrays
+
+from nose.tools import assert_equals
+
+
+class TestBcolzWriter(object):
+    @classmethod
+    def setup_class(cls):
+        cls.columns = ['open', 'high', 'low', 'close', 'volume']
+
+    def setUp(self):
+        self.root_dir = tempfile.mkdtemp()  # Create a temporary directory
+
+    def tearDown(self):
+        shutil.rmtree(self.root_dir)  # Remove the directory after the test
+
+    def generate_df(self, exchange_name, freq, start, end):
+        bundle = ExchangeBundle(exchange_name)
+        index = bundle.get_calendar_periods_range(start, end, freq)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(random.random(), inplace=True)
+        return df
+
+    def test_bcolz_write_daily_past(self):
+        start = pd.to_datetime('2016-01-01')
+        end = pd.to_datetime('2016-12-31')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_daily_present(self):
+        start = pd.to_datetime('2017-01-01')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_minute_past(self):
+        start = pd.to_datetime('2015-04-01 00:00')
+        end = pd.to_datetime('2015-04-30 23:59')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        pass
+
+    def test_bcolz_write_minute_present(self):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def bcolz_exchange_daily_write_read(self, exchange_name):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        bundle = ExchangeBundle(exchange_name)
+
+        df = self.generate_df(exchange_name, freq, start, end)
+
+        print(df.index[0], df.index[-1])
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=df.index[0],
+            end_session=df.index[-1],
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        reader = BcolzExchangeBarReader(rootdir=self.root_dir,
+                                        data_frequency=freq)
+
+        arrays = reader.load_raw_arrays(self.columns, start, end, [1, ])
+
+        periods = bundle.get_calendar_periods_range(
+            start, end, freq
+        )
+
+        dx = get_df_from_arrays(arrays, periods)
+
+        assert_equals(df.equals(dx), True)
+        pass
+
+    def test_bcolz_bitfinex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('bitfinex')
+
+    def test_bcolz_poloniex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('poloniex')

tests/exchange/test_bitfinex.py

@@ -4,11 +4,13 @@ from base import BaseExchangeTestCase
 from catalyst.exchange.bitfinex.bitfinex import Bitfinex
 from catalyst.exchange.exchange_utils import get_exchange_auth
 from catalyst.finance.execution import (LimitOrder)
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('test_bitfinex')
 
 
-class BitfinexTestCase(BaseExchangeTestCase):
+@deprecated
+class TestBitfinex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
         log.info('creating bitfinex object')
@@ -34,7 +36,7 @@ class BitfinexTestCase(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        orders = self.exchange.get_open_orders()
+        # orders = self.exchange.get_open_orders()
         pass
 
     def test_get_order(self):
@@ -47,18 +49,17 @@ class BitfinexTestCase(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
-            data_frequency='1m',
-            assets=self.exchange.get_asset('neo_btc')
-        )
+        # ohlcv_neo = self.exchange.get_candles(
+        #    freq='1T',
+        #    assets=self.exchange.get_asset('neo_btc'))
         pass
 
     def test_tickers(self):
         log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
+        # tickers = self.exchange.tickers([
+        #     self.exchange.get_asset('eth_btc'),
+        #     self.exchange.get_asset('etc_btc')
+        # ])
         pass
 
     def test_get_account(self):
@@ -67,11 +68,11 @@ class BitfinexTestCase(BaseExchangeTestCase):
 
     def test_get_balances(self):
         log.info('testing exchange balances')
-        balances = self.exchange.get_balances()
+        # balances = self.exchange.get_balances()
         pass
 
     def test_orderbook(self):
         log.info('testing order book for bitfinex')
-        asset = self.exchange.get_asset('eth_btc')
-        orderbook = self.exchange.get_orderbook(asset)
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset)
         pass

tests/exchange/test_bittrex.py

@@ -1,21 +1,24 @@
+# import pandas as pd
 from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('test_bittrex')
 
 
-class BittrexTestCase(BaseExchangeTestCase):
+@deprecated
+class TestBittrex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        print ('creating bittrex object')
         auth = get_exchange_auth('bittrex')
         self.exchange = Bittrex(
             key=auth['key'],
             secret=auth['secret'],
-            base_currency='btc'
+            base_currency=None,
+            portfolio=None
         )
 
     def test_order(self):
@@ -32,8 +35,8 @@ class BittrexTestCase(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        asset = self.exchange.get_asset('neo_btc')
-        orders = self.exchange.get_open_orders(asset)
+        # asset = self.exchange.get_asset('neo_btc')
+        # orders = self.exchange.get_open_orders(asset)
         pass
 
     def test_get_order(self):
@@ -50,18 +53,21 @@ class BittrexTestCase(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
-            data_frequency='5m',
-            assets=self.exchange.get_asset('neo_btc')
-        )
-        ohlcv_neo_ubq = self.exchange.get_candles(
-            data_frequency='5m',
-            assets=[
-                self.exchange.get_asset('neo_btc'),
-                self.exchange.get_asset('ubq_btc')
-            ],
-            bar_count=14
-        )
+        # ohlcv_neo = self.exchange.get_candles(
+        #     freq='5T',
+        #     assets=self.exchange.get_asset('neo_btc'),
+        #     bar_count=20,
+        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
+        # )
+        # ohlcv_neo_ubq = self.exchange.get_candles(
+        #     freq='1D',
+        #     assets=[
+        #         self.exchange.get_asset('neo_btc'),
+        #         self.exchange.get_asset('ubq_btc')
+        #     ],
+        #     bar_count=14,
+        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
+        # )
         pass
 
     def test_tickers(self):
@@ -75,7 +81,7 @@ class BittrexTestCase(BaseExchangeTestCase):
 
     def test_get_balances(self):
         log.info('testing wallet balances')
-        balances = self.exchange.get_balances()
+        # balances = self.exchange.get_balances()
         pass
 
     def test_get_account(self):
@@ -84,6 +90,6 @@ class BittrexTestCase(BaseExchangeTestCase):
 
     def test_orderbook(self):
         log.info('testing order book for bittrex')
-        asset = self.exchange.get_asset('eth_btc')
-        orderbook = self.exchange.get_orderbook(asset)
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset)
         pass

tests/exchange/test_bundle.py

@@ -1,54 +1,56 @@
-from logging import Logger
+# import hashlib
+import os
+import tempfile
+from logging import getLogger
 
 import pandas as pd
 
-from catalyst import get_calendar
-from catalyst.exchange.bundle_utils import get_bcolz_chunk, get_periods, \
-    get_periods_range
+from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
+    get_start_dt, get_df_from_arrays
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.exchange_utils import get_exchange_folder
-from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.stats_utils import df_to_string
 from catalyst.utils.paths import ensure_directory
 
-log = Logger('test_exchange_bundle')
+log = getLogger('test_exchange_bundle')
 
 
-class ExchangeBundleTestCase:
+class TestExchangeBundle:
     def test_spot_value(self):
-        data_frequency = 'daily'
+        # data_frequency = 'daily'
+        # exchange_name = 'poloniex'
+
+        # exchange = get_exchange(exchange_name)
+        # exchange_bundle = ExchangeBundle(exchange)
+        # assets = [
+        #     exchange.get_asset('btc_usdt')
+        # ]
+        # dt = pd.to_datetime('2017-10-14', utc=True)
+
+        # values = exchange_bundle.get_spot_values(
+        #     assets=assets,
+        #     field='close',
+        #     dt=dt,
+        #     data_frequency=data_frequency
+        # )
+        pass
+
+    def test_ingest_minute(self):
+        data_frequency = 'minute'
         exchange_name = 'poloniex'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
         assets = [
-            exchange.get_asset('btc_usdt')
-        ]
-        dt = pd.to_datetime('2017-10-14', utc=True)
-
-        values = exchange_bundle.get_spot_values(
-            assets=assets,
-            field='close',
-            dt=dt,
-            data_frequency=data_frequency
-        )
-        pass
-
-    def test_ingest_minute(self):
-        data_frequency = 'minute'
-        exchange_name = 'bitfinex'
-
-        exchange = get_exchange(exchange_name)
-        exchange_bundle = ExchangeBundle(exchange)
-        assets = [
-            exchange.get_asset('neo_eth')
+            exchange.get_asset('eth_btc')
         ]
 
-        # start = pd.to_datetime('2017-09-01', utc=True)
-        start = pd.to_datetime('2017-9-15', utc=True)
-        end = pd.to_datetime('2017-9-30', utc=True)
+        start = pd.to_datetime('2016-03-01', utc=True)
+        end = pd.to_datetime('2017-11-1', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -93,19 +95,44 @@ class ExchangeBundleTestCase:
         )
         pass
 
-    def test_ingest_daily(self):
+    def test_ingest_exchange(self):
         # exchange_name = 'bitfinex'
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'poloniex'
-        data_frequency = 'daily'
-        include_symbols = 'btc_usdt'
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
 
-        start = pd.to_datetime('2016-1-1', utc=True)
-        end = pd.to_datetime('2017-10-16', utc=True)
-        periods = get_periods_range(start, end, data_frequency)
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
 
+        log.info('ingesting exchange bundle {}'.format(exchange_name))
+        exchange_bundle.ingest(
+            data_frequency=data_frequency,
+            include_symbols=None,
+            exclude_symbols=None,
+            start=None,
+            end=None,
+            show_progress=True
+        )
+
+        pass
+
+    def test_ingest_daily(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+        include_symbols = 'neo_btc'
+
+        # exchange_name = 'poloniex'
+        # data_frequency = 'daily'
+        # include_symbols = 'eth_btc'
+
+        # start = pd.to_datetime('2017-1-1', utc=True)
+        # end = pd.to_datetime('2017-10-16', utc=True)
+        # periods = get_periods_range(start, end, data_frequency)
+
+        start = None
+        end = None
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
 
@@ -125,12 +152,18 @@ class ExchangeBundleTestCase:
             assets.append(exchange.get_asset(pair_symbol))
 
         reader = exchange_bundle.get_reader(data_frequency)
+        start_dt = reader.first_trading_day
+        end_dt = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
+
         for asset in assets:
             arrays = reader.load_raw_arrays(
                 sids=[asset.sid],
                 fields=['close'],
-                start_dt=start,
-                end_dt=end
+                start_dt=start_dt,
+                end_dt=end_dt
             )
             print('found {} rows for {} ingestion\n{}'.format(
                 len(arrays[0]), asset.symbol, arrays[0])
@@ -181,7 +214,7 @@ class ExchangeBundleTestCase:
         # encounter these problems as I have been focusing on minute data.
         reader = exchange_bundle.get_reader(data_frequency)
         for asset in assets:
-            # Since this pair was loaded last. It should be there in daily mode.
+            # Since this pair was loaded last. It should be here in daily mode.
             arrays = reader.load_raw_arrays(
                 sids=[asset.sid],
                 fields=['close'],
@@ -218,7 +251,6 @@ class ExchangeBundleTestCase:
         ensure_directory(path)
 
         exchange_bundle = ExchangeBundle(exchange)
-        calendar = get_calendar('OPEN')
 
         # We are using a BcolzMinuteBarWriter even though the data is daily
         # Each day has a maximum of one bar
@@ -270,17 +302,248 @@ class ExchangeBundleTestCase:
         pass
 
     def test_minute_bundle(self):
+        # exchange_name = 'poloniex'
+        # data_frequency = 'minute'
+
+        # exchange = get_exchange(exchange_name)
+        # asset = exchange.get_asset('neos_btc')
+
+        # path = get_bcolz_chunk(
+        #     exchange_name=exchange_name,
+        #     symbol=asset.symbol,
+        #     data_frequency=data_frequency,
+        #     period='2017-5',
+        # )
+        pass
+
+    def test_hash_symbol(self):
+        # symbol = 'etc_btc'
+        # sid = int(
+        #     hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+        # ) % 10 ** 6
+        pass
+
+    def test_validate_data(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-9-2 1:00', utc=True)
+        bar_count = 60
+
+        bundle_series = exchange_bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count * 5,
+            field='close',
+            data_frequency='minute',
+        )
+        candles = exchange.get_candles(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            freq='1T'
+        )
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+
+        frames = []
+        for asset in assets:
+            bundle_df = pd.DataFrame(
+                data=dict(bundle_price=bundle_series[asset]),
+                index=bundle_series[asset].index
+            )
+            exchange_series = exchange.get_series_from_candles(
+                candles=candles[asset],
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                field='close'
+            )
+            exchange_df = pd.DataFrame(
+                data=dict(exchange_price=exchange_series),
+                index=exchange_series.index
+            )
+
+            df = exchange_df.join(bundle_df, how='left')
+            df['last_traded'] = df.index
+            df['asset'] = asset.symbol
+            df.set_index(['asset', 'last_traded'], inplace=True)
+
+            frames.append(df)
+
+        df = pd.concat(frames)
+        print('\n' + df_to_string(df))
+        pass
+
+    def test_ingest_candles(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-10-20', utc=True)
+        bar_count = 100
+
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        candles = exchange.get_candles(
+            assets=assets,
+            start_dt=start_dt,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            freq='1T'
+        )
+
+        writer = bundle.get_writer(start_dt, end_dt, data_frequency)
+        for asset in assets:
+            dates = [candle['last_traded'] for candle in candles[asset]]
+
+            values = dict()
+            for field in ['open', 'high', 'low', 'close', 'volume']:
+                values[field] = [candle[field] for candle in candles[asset]]
+
+            periods = bundle.get_calendar_periods_range(
+                start_dt, end_dt, data_frequency
+            )
+            df = pd.DataFrame(values, index=dates)
+            df = df.loc[periods].fillna(method='ffill')
+
+            # TODO: why do I get an extra bar?
+            bundle.ingest_df(
+                ohlcv_df=df,
+                data_frequency=data_frequency,
+                asset=asset,
+                writer=writer,
+                empty_rows_behavior='raise',
+                duplicates_behavior='raise'
+            )
+
+        bundle_series = bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            field='close',
+            data_frequency=data_frequency,
+            reset_reader=True
+        )
+        df = pd.DataFrame(bundle_series)
+        print('\n' + df_to_string(df))
+        pass
+
+    def main_bundle_to_csv(self):
         exchange_name = 'poloniex'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neo_btc')
+        asset = exchange.get_asset('eth_btc')
 
-        path = get_bcolz_chunk(
-            exchange_name=exchange_name,
-            symbol=asset.symbol,
+        start_dt = pd.to_datetime('2016-5-31', utc=True)
+        end_dt = pd.to_datetime('2016-6-1', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange_name=exchange.name,
             data_frequency=data_frequency,
-            period='2017-5',
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
         )
 
+    def bundle_to_csv(self):
+        exchange_name = 'poloniex'
+        data_frequency = 'minute'
+        period = '2017-01'
+        symbol = 'eth_btc'
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset(symbol)
+
+        path = get_bcolz_chunk(
+            exchange_name=exchange.name,
+            symbol=asset.symbol,
+            data_frequency=data_frequency,
+            period=period
+        )
+        self._bundle_to_csv(
+            asset=asset,
+            exchange_name=exchange.name,
+            data_frequency=data_frequency,
+            path=path,
+            filename=period
+        )
+        pass
+
+    def _bundle_to_csv(self, asset, exchange_name, data_frequency, filename,
+                       path=None, start_dt=None, end_dt=None):
+        bundle = ExchangeBundle(exchange_name)
+        reader = bundle.get_reader(data_frequency, path=path)
+
+        if start_dt is None:
+            start_dt = reader.first_trading_day
+
+        if end_dt is None:
+            end_dt = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
+
+        arrays = None
+        try:
+            arrays = reader.load_raw_arrays(
+                sids=[asset.sid],
+                fields=['open', 'high', 'low', 'close', 'volume'],
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+        except Exception as e:
+            log.warn('skipping ctable for {} from {} to {}: {}'.format(
+                asset.symbol, start_dt, end_dt, e
+            ))
+
+        periods = bundle.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+        df = get_df_from_arrays(arrays, periods)
+
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', exchange_name, asset.symbol
+        )
+        ensure_directory(folder)
+
+        path = os.path.join(folder, filename + '.csv')
+
+        log.info('creating csv file: {}'.format(path))
+        print('HEAD\n{}'.format(df.head(100)))
+        print('TAIL\n{}'.format(df.tail(100)))
+        df.to_csv(path)
+        pass
+
+    def test_ingest_csv(self):
+        data_frequency = 'minute'
+        exchange_name = 'bittrex'
+        path = '/Users/fredfortier/Dropbox/Enigma/Data/bittrex_bat_eth.csv'
+
+        exchange_bundle = ExchangeBundle(exchange_name)
+        exchange_bundle.ingest_csv(path, data_frequency)
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset('bat_eth')
+
+        start_dt = pd.to_datetime('2017-6-3', utc=True)
+        end_dt = pd.to_datetime('2017-8-3 19:24', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange_name=exchange.name,
+            data_frequency=data_frequency,
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
         pass

tests/exchange/test_ccxt.py

@@ -0,0 +1,93 @@
+import pandas as pd
+from logbook import Logger
+from base import BaseExchangeTestCase
+
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.finance.order import Order
+from catalyst.exchange.exchange_utils import get_exchange_auth
+
+log = Logger('test_ccxt')
+
+
+class TestCCXT(BaseExchangeTestCase):
+    @classmethod
+    def setup(self):
+        exchange_name = 'gdax'
+        auth = get_exchange_auth(exchange_name)
+        self.exchange = CCXT(
+            exchange_name=exchange_name,
+            key=auth['key'],
+            secret=auth['secret'],
+            base_currency='eth',
+            portfolio=None
+        )
+
+    def test_order(self):
+        log.info('creating order')
+        asset = self.exchange.get_asset('neo_eth')
+        order_id = self.exchange.order(
+            asset=asset,
+            limit_price=0.07,
+            amount=1,
+        )
+        log.info('order created {}'.format(order_id))
+        assert order_id is not None
+        pass
+
+    def test_open_orders(self):
+        # log.info('retrieving open orders')
+        # asset = self.exchange.get_asset('neo_eth')
+        # orders = self.exchange.get_open_orders(asset)
+        pass
+
+    def test_get_order(self):
+        log.info('retrieving order')
+        order = self.exchange.get_order('2631386', 'neo_eth')
+        # order = self.exchange.get_order('2631386')
+        assert isinstance(order, Order)
+        pass
+
+    def test_cancel_order(self, ):
+        log.info('cancel order')
+        self.exchange.cancel_order('2631386', 'neo_eth')
+        pass
+
+    def test_get_candles(self):
+        log.info('retrieving candles')
+        candles = self.exchange.get_candles(
+            freq='5T',
+            assets=[self.exchange.get_asset('eth_btc')],
+            bar_count=200,
+            start_dt=pd.to_datetime('2017-01-01', utc=True)
+        )
+
+        for asset in candles:
+            df = pd.DataFrame(candles[asset])
+            df.set_index('last_traded', drop=True, inplace=True)
+        pass
+
+    def test_tickers(self):
+        log.info('retrieving tickers')
+        tickers = self.exchange.tickers([
+            self.exchange.get_asset('eth_btc'),
+        ])
+        assert len(tickers) == 1
+        pass
+
+    def test_get_balances(self):
+        log.info('testing wallet balances')
+        # balances = self.exchange.get_balances()
+        pass
+
+    def test_get_account(self):
+        log.info('testing account data')
+        pass
+
+    def test_orderbook(self):
+        log.info('testing order book for bittrex')
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
+        pass
+
+    def test_get_fees(self):
+        pass