merged from develop

Example: Simple Universe

README.rst

@@ -1,3 +1,72 @@
-All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_
-can be found in the 
-`documentation website <https://enigmampc.github.io/catalyst>`_.
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+    :target: https://enigmampc.github.io/catalyst
+    :align: center
+    :alt: Enigma | Catalyst
+
+|version tag|
+|version status|
+|discord|
+|twitter|
+
+|
+
+Catalyst is an algorithmic trading library for crypto-assets written in Python.
+It allows trading strategies to be easily expressed and backtested against 
+historical data (with daily and minute resolution), providing analytics and 
+insights regarding a particular strategy's performance. Catalyst also supports
+live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
+and Poloniex) with more being added over time. Catalyst empowers users to share 
+and curate data and build profitable, data-driven investment strategies. Please 
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
+refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
+further technical details.
+
+Catalyst builds on top of the well-established 
+`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
+minimize structural changes to the general API to maximize compatibility with 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
+for questions around Catalyst, algorithmic trading and technical support.
+
+Overview
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.
+-  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
+   performance across trading algorithms.
+
+Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
+
+
+
+
+.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
+   :target: https://discordapp.com/invite/SJK32GY
+
+.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
+   :target: https://twitter.com/enigmampc
+
+

catalyst/__init__.py

@@ -29,11 +29,14 @@ from ._version import get_versions
 from . algorithm import TradingAlgorithm
 from . import api
 
+from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
+
+__version__ = get_versions()['version']
+del get_versions
 
 # PERF: Fire a warning if calendars were instantiated during catalyst import.
 # Having calendars doesn't break anything per-se, but it makes catalyst imports
 # noticeably slower, which becomes particularly noticeable in the Zipline CLI.
-from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
 if global_calendar_dispatcher._calendars:
     import warnings
     warnings.warn(
@@ -44,10 +47,6 @@ if global_calendar_dispatcher._calendars:
 del global_calendar_dispatcher
 
 
-__version__ = get_versions()['version']
-del get_versions
-
-
 def load_ipython_extension(ipython):
     from .__main__ import catalyst_magic
     ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -69,7 +68,6 @@ if os.name == 'nt':
     _()
     del _
 
-
 __all__ = [
     'TradingAlgorithm',
     'api',
@@ -80,7 +78,3 @@ __all__ = [
     'run_algorithm',
     'utils',
 ]
-
-from ._version import get_versions
-__version__ = get_versions()['version']
-del get_versions

catalyst/__main__.py

@@ -10,7 +10,6 @@ from six import text_type
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_utils import delete_algo_folder
-from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -30,16 +29,17 @@ except NameError:
 @click.option(
     '--strict-extensions/--non-strict-extensions',
     is_flag=True,
-    help='If --strict-extensions is passed then catalyst will not run if it'
-         ' cannot load all of the specified extensions. If this is not passed or'
-         ' --non-strict-extensions is passed then the failure will be logged but'
-         ' execution will continue.',
+    help='If --strict-extensions is passed then catalyst will not run '
+         'if it cannot load all of the specified extensions. If this is '
+         'not passed or --non-strict-extensions is passed then the '
+         'failure will be logged but execution will continue.',
 )
 @click.option(
     '--default-extension/--no-default-extension',
     is_flag=True,
     default=True,
-    help="Don't load the default catalyst extension.py file in $CATALYST_HOME.",
+    help="Don't load the default catalyst extension.py file "
+         "in $CATALYST_HOME.",
 )
 @click.version_option()
 def main(extension, strict_extensions, default_extension):
@@ -124,9 +124,9 @@ def ipython_only(option):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '--data-frequency',
@@ -138,7 +138,6 @@ def ipython_only(option):
 @click.option(
     '--capital-base',
     type=float,
-    default=10e6,
     show_default=True,
     help='The starting capital for the simulation.',
 )
@@ -176,8 +175,8 @@ def ipython_only(option):
     default='-',
     metavar='FILENAME',
     show_default=True,
-    help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
+    help="The location to write the perf data. If this is '-' the perf"
+         " will be written to stdout.",
 )
 @click.option(
     '--print-algo/--no-print-algo',
@@ -194,8 +193,7 @@ def ipython_only(option):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange.',
 )
 @click.option(
     '-n',
@@ -240,16 +238,27 @@ def run(ctx,
         # does not pass either of these and then passes the first only
         # to be told they need to pass the second argument also
         ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'",
+            "must specify dates with '-s' / '--start' and '-e' / '--end'"
+            " in backtest mode",
         )
     if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'")
+        ctx.fail("must specify a start date with '-s' / '--start'"
+                 " in backtest mode")
     if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'")
+        ctx.fail("must specify an end date with '-e' / '--end'"
+                 " in backtest mode")
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
+    if base_currency is None:
+        ctx.fail("must specify a base currency with '-c' in backtest mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    click.echo('Running in backtesting mode.')
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -273,7 +282,9 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=False
+        live_graph=False,
+        simulate_orders=True,
+        stats_output=None,
     )
 
     if output == '-':
@@ -301,11 +312,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                     # these options are set when running in line magic mode
-                     # set a non None algo text to use the ipython user_ns
-                     '--algotext', '',
-                     '--local-namespace',
-                 ] if cell is None else []) + line.split(),
+                # these options are set when running in line magic mode
+                # set a non None algo text to use the ipython user_ns
+                '--algotext', '',
+                '--local-namespace',
+            ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,
@@ -325,6 +336,12 @@ def catalyst_magic(line, cell=None):
     type=click.File('r'),
     help='The file that contains the algorithm to run.',
 )
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
 @click.option(
     '-t',
     '--algotext',
@@ -335,9 +352,9 @@ def catalyst_magic(line, cell=None):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '-o',
@@ -363,8 +380,7 @@ def catalyst_magic(line, cell=None):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange.',
 )
 @click.option(
     '-n',
@@ -383,9 +399,17 @@ def catalyst_magic(line, cell=None):
     default=False,
     help='Display live graph.',
 )
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
 @click.pass_context
 def live(ctx,
          algofile,
+         capital_base,
          algotext,
          define,
          output,
@@ -394,7 +418,8 @@ def live(ctx,
          exchange_name,
          algo_namespace,
          base_currency,
-         live_graph):
+         live_graph,
+         simulate_orders):
     """Trade live with the given algorithm.
     """
     if (algotext is not None) == (algofile is not None):
@@ -405,11 +430,22 @@ def live(ctx,
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
+
     if algo_namespace is None:
         ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
     if base_currency is None:
         ctx.fail("must specify a base currency '-c' in live execution mode")
 
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.')
+
+    else:
+        click.echo('Running in live trading mode.')
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -419,7 +455,7 @@ def live(ctx,
         algotext=algotext,
         defines=define,
         data_frequency=None,
-        capital_base=None,
+        capital_base=capital_base,
         data=None,
         bundle=None,
         bundle_timestamp=None,
@@ -433,7 +469,9 @@ def live(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        simulate_orders=simulate_orders,
+        stats_output=None,
     )
 
     if output == '-':
@@ -448,9 +486,7 @@ def live(ctx,
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
+    help='The name of the exchange bundle to ingest.',
 )
 @click.option(
     '-f',
@@ -486,6 +522,13 @@ def live(ctx,
     help='A list of symbols to exclude from the ingestion '
          '(optional comma separated list)',
 )
+@click.option(
+    '--csv',
+    default=None,
+    help='The path of a CSV file containing the data. If specified, start, '
+         'end, include-symbols and exclude-symbols will be ignored. Instead,'
+         'all data in the file will be ingested.',
+)
 @click.option(
     '--show-progress/--no-show-progress',
     default=True,
@@ -501,9 +544,10 @@ def live(ctx,
     default=False,
     help='Report potential anomalies found in data bundles.'
 )
-def ingest_exchange(exchange_name, data_frequency, start, end,
-                    include_symbols, exclude_symbols, show_progress, verbose,
-                    validate):
+@click.pass_context
+def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
+                    include_symbols, exclude_symbols, csv, show_progress,
+                    verbose, validate):
     """
     Ingest data for the given exchange.
     """
@@ -511,8 +555,7 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
-    exchange = get_exchange(exchange_name)
-    exchange_bundle = ExchangeBundle(exchange)
+    exchange_bundle = ExchangeBundle(exchange_name)
 
     click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
@@ -523,7 +566,8 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
         end=end,
         show_progress=show_progress,
         show_breakdown=verbose,
-        show_report=validate
+        show_report=validate,
+        csv=csv
     )
 
 
@@ -536,18 +580,17 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
 @click.pass_context
 def clean_algo(ctx, algo_namespace):
     click.echo(
-        'Deleting the state folder of algo: {}...'.format(algo_namespace)
+        'Cleaning algo state: {}'.format(algo_namespace)
     )
     delete_algo_folder(algo_namespace)
+    click.echo('Done')
 
 
 @main.command(name='clean-exchange')
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
+    help='The name of the exchange bundle to ingest.',
 )
 @click.option(
     '-f',
@@ -565,8 +608,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
-    exchange = get_exchange(exchange_name)
-    exchange_bundle = ExchangeBundle(exchange)
+    exchange_bundle = ExchangeBundle(exchange_name)
 
     click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
     exchange_bundle.clean(
@@ -587,9 +629,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
+    help='The name of the exchange bundle to ingest.',
 )
 @click.option(
     '-c',

catalyst/algorithm.py

@@ -124,7 +124,6 @@ from catalyst.utils.events import (
 from catalyst.utils.factory import create_simulation_parameters
 from catalyst.utils.math_utils import (
     tolerant_equals,
-    round_if_near_integer,
     round_nearest
 )
 from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
@@ -1485,7 +1484,6 @@ class TradingAlgorithm(object):
         """
         Converts the number of shares to the smallest tradable lot size for
         the asset being ordered.
-        
         """
         return round_nearest(amount, asset.min_trade_size)
 
@@ -1523,6 +1521,7 @@ class TradingAlgorithm(object):
                              self.updated_portfolio(),
                              self.get_datetime(),
                              self.trading_client.current_data)
+
     @staticmethod
     def __convert_order_params_for_blotter(limit_price, stop_price, style):
         """

catalyst/assets/_assets.pyx

@@ -38,6 +38,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
+from catalyst.exchange.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
@@ -395,11 +396,18 @@ cdef class Future(Asset):
 
 cdef class TradingPair(Asset):
     cdef readonly float leverage
-    cdef readonly object market_currency
+    cdef readonly object quote_currency
     cdef readonly object base_currency
     cdef readonly object end_daily
     cdef readonly object end_minute
     cdef readonly object exchange_symbol
+    cdef readonly float maker
+    cdef readonly float taker
+    cdef readonly int trading_state
+    cdef readonly object data_source
+    cdef readonly float max_trade_size
+    cdef readonly float lot
+    cdef readonly int decimals
 
     _kwargnames = frozenset({
         'sid',
@@ -412,12 +420,19 @@ cdef class TradingPair(Asset):
         'exchange',
         'exchange_full',
         'leverage',
-        'market_currency',
+        'quote_currency',
         'base_currency',
         'end_daily',
         'end_minute',
         'exchange_symbol',
-        'min_trade_size'
+        'min_trade_size',
+        'max_trade_size',
+        'lot',
+        'maker',
+        'taker',
+        'trading_state',
+        'data_source',
+        'decimals'
     })
     def __init__(self,
                  object symbol,
@@ -433,10 +448,17 @@ cdef class TradingPair(Asset):
                  object first_traded=None,
                  object auto_close_date=None,
                  object exchange_full=None,
-                 object min_trade_size=None):
+                 float min_trade_size=0.0001,
+                 float max_trade_size=1000000,
+                 float maker=0.0015,
+                 float taker=0.0025,
+                 float lot=0,
+                 int decimals = 8,
+                 int trading_state=0,
+                 object data_source='catalyst'):
         """
         Replicates the Asset constructor with some built-in conventions
-        and a new 'leverage' attribute.
+        and adds properties for leverage and fees.
 
         Symbol
         ------
@@ -468,8 +490,6 @@ cdef class TradingPair(Asset):
         highest volume and market cap generally benefit from high leverage.
         New currencies from ICO generally cannot be leveraged.
 
-        The leverage value is either None or and integer.
-
         Leverage allows you to open a larger position with a smaller amount
         of funds. For example, if you open a $5,000 position in BTC/USD
         with 5:1 leverage, only one-fifth of this amount, or $1000, will be
@@ -479,6 +499,11 @@ cdef class TradingPair(Asset):
         the position. If you open with 1:1 leverage, $5,000 of your balance
         will be tied to the position.
 
+        Fees
+        ----
+        Exchanges generally charge a taker (taking from the order book) or
+        maker (adding to the order book) fee.
+
         :param symbol:
         :param exchange:
         :param start_date:
@@ -493,21 +518,23 @@ cdef class TradingPair(Asset):
         :param auto_close_date:
         :param exchange_full:
         :param min_trade_size:
+        :param max_trade_size:
+        :param maker:
+        :param taker:
+        :param data_source
+        :param decimals
+        :param lot
         """
 
         symbol = symbol.lower()
         try:
-            self.market_currency, self.base_currency = symbol.split('_')
+            self.base_currency, self.quote_currency = symbol.split('_')
         except Exception as e:
             raise InvalidSymbolError(symbol=symbol, error=e)
 
         if sid == 0 or sid is None:
             try:
-                # sid = abs(hash(symbol)) % (10 ** 4)
-                # TODO: try to encode the symbol in the main scope
-                sid = int(
-                    hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
-                ) % 10 ** 6
+                sid = get_sid(symbol)
             except Exception as e:
                 raise SidHashError(symbol=symbol)
 
@@ -515,11 +542,14 @@ cdef class TradingPair(Asset):
             asset_name = ' / '.join(symbol.split('_')).upper()
 
         if start_date is None:
-            start_date = pd.Timestamp.utcnow()
+            start_date = pd.to_datetime('2009-1-1', utc=True)
 
         if end_date is None:
             end_date = pd.Timestamp.utcnow() + timedelta(days=365)
 
+        if lot == 0 and min_trade_size > 0:
+            lot = min_trade_size
+
         super().__init__(
             sid,
             exchange,
@@ -530,19 +560,26 @@ cdef class TradingPair(Asset):
             first_traded=first_traded,
             auto_close_date=auto_close_date,
             exchange_full=exchange_full,
-            min_trade_size=min_trade_size
+            min_trade_size=min_trade_size,
         )
 
+        self.maker = maker
+        self.taker = taker
         self.leverage = leverage
         self.end_daily = end_daily
         self.end_minute = end_minute
         self.exchange_symbol = exchange_symbol
+        self.trading_state = trading_state
+        self.data_source = data_source
+        self.max_trade_size = max_trade_size
+        self.lot = lot
+        self.decimals = decimals
 
     def __repr__(self):
         return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
                'Introduced On: {start_date}, ' \
-               'Market Currency: {market_currency}, ' \
                'Base Currency: {base_currency}, ' \
+               'Quote Currency: {quote_currency}, ' \
                'Exchange Leverage: {leverage}, ' \
                'Minimum Trade Size: {min_trade_size} ' \
                'Last daily ingestion: {end_daily} ' \
@@ -551,7 +588,7 @@ cdef class TradingPair(Asset):
             sid=self.sid,
             exchange=self.exchange,
             start_date=self.start_date,
-            market_currency=self.market_currency,
+            quote_currency=self.quote_currency,
             base_currency=self.base_currency,
             leverage=self.leverage,
             min_trade_size=self.min_trade_size,
@@ -559,6 +596,18 @@ cdef class TradingPair(Asset):
             end_minute=self.end_minute
         )
 
+    cpdef to_dict(self):
+        """
+        Convert to a python dict.
+        """
+        #TODO: missing fields
+        super_dict = super(TradingPair, self).to_dict()
+        super_dict['end_daily'] = self.end_daily
+        super_dict['end_minute'] = self.end_minute
+        super_dict['leverage'] = self.leverage
+        super_dict['min_trade_size'] = self.min_trade_size
+        return super_dict
+
     def is_exchange_open(self, dt_minute):
         """
         Parameters
@@ -570,7 +619,7 @@ cdef class TradingPair(Asset):
         -------
         boolean: whether the asset's exchange is open at the given minute.
         """
-        #TODO: consider implementing to spot holds
+        #TODO: make more dymanic to catch holds
         return True
 
     cpdef __reduce__(self):
@@ -580,6 +629,7 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
+        #TODO: make sure that all fields set there
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
@@ -590,7 +640,12 @@ cdef class TradingPair(Asset):
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
-                                 self.min_trade_size))
+                                 self.min_trade_size,
+                                 self.max_trade_size,
+                                 self.lot,
+                                 self.decimals,
+                                 self.taker,
+                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -1,9 +1,18 @@
 # -*- coding: utf-8 -*-
 
+import os
 import logbook
 
-LOG_LEVEL = logbook.INFO
+''' You can override the LOG level from your environment.
+    For example, if you want to see the DEBUG messages, run:
+    $ export CATALYST_LOG_LEVEL=10
+'''
+LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
+
+SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
+              '{exchange}/symbols.json'
 
 DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+DATE_FORMAT = '%Y-%m-%d'
 
-AUTO_INGEST = False
+AUTO_INGEST = False

catalyst/curate/poloniex.py

@@ -1,25 +1,33 @@
-import json, time, csv
+import os
+import time
+import shutil
+import json
+import csv
 from datetime import datetime
+
 import pandas as pd
-import os, time, shutil, requests, logbook
+import requests
+import logbook
+
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 
-DT_START        = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END          = pd.to_datetime('today').value // 10 ** 9
-CSV_OUT_FOLDER  = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
-CONN_RETRIES    = 2
+DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
+DT_END = pd.to_datetime('today').value // 10 ** 9
+CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
+CONN_RETRIES = 2
 
 logbook.StderrHandler().push_application()
 log = logbook.Logger(__name__)
 
+
 class PoloniexCurator(object):
     '''
     OHLCV data feed generator for crypto data. Based on Poloniex market data
     '''
 
-    _api_path       = 'https://poloniex.com/public?'
-    currency_pairs  = []
+    _api_path = 'https://poloniex.com/public?'
+    currency_pairs = []
 
     def __init__(self):
         if not os.path.exists(CSV_OUT_FOLDER):
@@ -30,10 +38,9 @@ class PoloniexCurator(object):
                             CSV_OUT_FOLDER))
                 log.exception(e)
 
-
     def get_currency_pairs(self):
         '''
-        Retrieves and returns all currency pairs from the exchange 
+        Retrieves and returns all currency pairs from the exchange
         '''
         url = self._api_path + 'command=returnTicker'
 
@@ -45,7 +52,7 @@ class PoloniexCurator(object):
             return None
 
         data = response.json()
-        self.currency_pairs  = []
+        self.currency_pairs = []
         for ticker in data:
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
@@ -54,54 +61,60 @@ class PoloniexCurator(object):
                     len(self.currency_pairs)
                     ))
 
-
-
     def _retrieve_tradeID_date(self, row):
         '''
         Helper function that reads tradeID and date fields from CSV readline
         '''
         tId = int(row.split(',')[0])
-        d   = pd.to_datetime(row.split(',')[1], 
-                             infer_datetime_format=True).value // 10 ** 9
+        d = pd.to_datetime(row.split(',')[1],
+                           infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
-
-    def retrieve_trade_history(self, currencyPair, start=DT_START, 
+    def retrieve_trade_history(self, currencyPair, start=DT_START,
                                end=DT_END, temp=None):
         '''
-        Retrieves TradeHistory from exchange for a given currencyPair 
-        between start and end dates. If no start date is provided, uses 
+        Retrieves TradeHistory from exchange for a given currencyPair
+        between start and end dates. If no start date is provided, uses
         a system-wide one (beginning of time for cryptotrading).
         If no end date is provided, 'now' is used.
 
         Stores results in CSV file on disk.
-        
-        This function is called recursively to work around the 
+
+        This function is called recursively to work around the
         limitations imposed by the provider API.
         '''
         csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
 
         '''
-        Check what data we already have on disk, reading first and last 
+        Check what data we already have on disk, reading first and last
         lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
-            with open(csv_fn, 'ab+') as f: 
+            with open(csv_fn, 'ab+') as f:
                 f.seek(0, os.SEEK_END)
                 if(f.tell() > 2):                   # Check file size is not 0
-                    f.seek(0)                       # Go to start to read 
-                    last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
+                    f.seek(0)                       # Go to start to read
+                    last_tradeID, end_file = self._retrieve_tradeID_date(
+                                                    f.readline())
                     f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
                     while f.read(1) != b"\n":       # Until EOL is found...
-                        f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
-                    first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
+                        # ...jump back the read byte plus one more.
+                        f.seek(-2, os.SEEK_CUR)
+                    first_tradeID, start_file = self._retrieve_tradeID_date(
+                                                    f.readline())
 
-                    if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
-                        or (currencyPair == 'BTC_HUC' and first_tradeID ==     2) 
-                        or (currencyPair == 'BTC_RIC' and first_tradeID ==     2) 
-                        or (currencyPair == 'BTC_XCP' and first_tradeID ==     2)
-                        or (currencyPair == 'BTC_NAV' and first_tradeID ==  4569)
-                        or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
+                    if(end_file + 3600 * 6 > DT_END
+                        and (first_tradeID == 1
+                             or (currencyPair == 'BTC_HUC'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_RIC'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_XCP'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_NAV'
+                                 and first_tradeID == 4569)
+                             or (currencyPair == 'BTC_POT'
+                                 and first_tradeID == 23511))):
                         return
 
         except Exception as e:
@@ -109,11 +122,11 @@ class PoloniexCurator(object):
             log.exception(e)
 
         '''
-        Poloniex API limits querying TradeHistory to intervals smaller 
+        Poloniex API limits querying TradeHistory to intervals smaller
         than 1 month, so we make sure that start date is never more than
         1 month apart from end date
         '''
-        if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days     
+        if(end - start > 2419200):   # 60s/min * 60min/hr * 24hr/day * 28days
             newstart = end - 2419200
         else:
             newstart = start
@@ -124,12 +137,11 @@ class PoloniexCurator(object):
 
         url = '{path}command=returnTradeHistory&currencyPair={pair}' \
               '&start={start}&end={end}'.format(
-                    path = self._api_path,
-                    pair = currencyPair,
-                    start = str(newstart),
-                    end = str(end)
+                    path=self._api_path,
+                    pair=currencyPair,
+                    start=str(newstart),
+                    end=str(end)
                 )
-        print url
 
         attempts = 0
         success = 0
@@ -137,14 +149,14 @@ class PoloniexCurator(object):
             try:
                 response = requests.get(url)
             except Exception as e:
-                log.error('Failed to retrieve trade history data for {}'.format(
-                            currencyPair
-                            ))
+                log.error('Failed to retrieve trade history data'
+                          'for {}'.format(currencyPair))
                 log.exception(e)
                 attempts += 1
             else:
                 try:
-                    if isinstance(response.json(), dict) and response.json()['error']:
+                    if(isinstance(response.json(), dict)
+                       and response.json()['error']):
                         log.error('Failed to to retrieve trade history data '
                                   'for {}: {}'.format(
                                     currencyPair,
@@ -161,33 +173,32 @@ class PoloniexCurator(object):
         if not success:
             return None
 
-
         '''
-        If we get to transactionId == 1, and we already have that on 
+        If we get to transactionId == 1, and we already have that on
         disk, we got to the end of TradeHistory for this coin.
         '''
-        if('first_tradeID' in locals() 
-                and response.json()[-1]['tradeID'] == first_tradeID): 
+        if('first_tradeID' in locals()
+                and response.json()[-1]['tradeID'] == first_tradeID):
             return
 
         '''
             There are primarily two scenarios:
-                a) There is newer data available that we need to add at 
-                   the beginning of the file. We'll retrieve all what we 
-                   need until we get to what we already have, writing it 
-                   to a temporary file; and we will write that at the 
+                a) There is newer data available that we need to add at
+                   the beginning of the file. We'll retrieve all what we
+                   need until we get to what we already have, writing it
+                   to a temporary file; and we will write that at the
                    beginning of our existing file.
-                b) We are going back in time, appending at the end of 
-                   our existing TradeHistory until the first transaction 
+                b) We are going back in time, appending at the end of
+                   our existing TradeHistory until the first transaction
                    for this currencyPair
         '''
-        try: 
-            if( 'end_file' in locals() and end_file + 3600 < end):
+        try:
+            if('end_file' in locals() and end_file + 3600 < end):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
                 for item in response.json():
-                    if( item['tradeID'] <= last_tradeID ):
+                    if(item['tradeID'] <= last_tradeID):
                         continue
                     tempcsv.writerow([
                         item['tradeID'],
@@ -196,27 +207,28 @@ class PoloniexCurator(object):
                         item['rate'],
                         item['amount'],
                         item['total'],
-                        item['globalTradeID']        
+                        item['globalTradeID'],
                     ])
-                if( response.json()[-1]['tradeID'] > last_tradeID ):
-                    end = pd.to_datetime( response.json()[-1]['date'], 
-                                infer_datetime_format=True).value // 10 ** 9
-                    self.retrieve_trade_history(currencyPair, start, 
-                                                end, temp=temp) 
+                if(response.json()[-1]['tradeID'] > last_tradeID):
+                    end = pd.to_datetime(response.json()[-1]['date'],
+                                         infer_datetime_format=True
+                                         ).value // 10**9
+                    self.retrieve_trade_history(currencyPair, start,
+                                                end, temp=temp)
                 else:
-                    with open(csv_fn,'rb+') as f:
-                        shutil.copyfileobj(f,temp)
+                    with open(csv_fn, 'rb+') as f:
+                        shutil.copyfileobj(f, temp)
                         f.seek(0)
                         temp.seek(0)
-                        shutil.copyfileobj(temp,f)
+                        shutil.copyfileobj(temp, f)
                     temp.close()
                     end = start_file
             else:
                 with open(csv_fn, 'ab') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in response.json():
-                        if( 'first_tradeID' in locals() 
-                                and item['tradeID'] >= first_tradeID ):
+                        if('first_tradeID' in locals()
+                                and item['tradeID'] >= first_tradeID):
                             continue
                         csvwriter.writerow([
                             item['tradeID'],
@@ -227,70 +239,66 @@ class PoloniexCurator(object):
                             item['total'],
                             item['globalTradeID']
                         ])
-                end = pd.to_datetime(response.json()[-1]['date'], 
-                            infer_datetime_format=True).value // 10 ** 9
+                end = pd.to_datetime(response.json()[-1]['date'],
+                                     infer_datetime_format=True).value//10**9
 
         except Exception as e:
             log.error('Error opening {}'.format(csv_fn))
             log.exception(e)
 
         '''
-            If we got here, we aren't done yet. Call recursively with 
+            If we got here, we aren't done yet. Call recursively with
             'end' times that go sequentially back in time.
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
-
-
     def generate_ohlcv(self, df):
         '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
         by resampling with 1-minute period
         '''
-        df.set_index('date', inplace=True)               # Index by date
-        vol = df['total'].to_frame('volume')             # set Vol aside
-        df.drop('total', axis=1, inplace=True)           # Drop volume data 
-        ohlc = df.resample('T').ohlc()                   # Resample OHLC 1min
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Raname columns by dropping 'rate'
-        closes = ohlc['close'].fillna(method='pad')      # Pad fwd missing 'close'
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))    # Fill N/A with last close
-        vol = vol.resample('T').sum().fillna(0)          # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)            # Concatenate OHLC + Vol
+        df.set_index('date', inplace=True)             # Index by date
+        vol = df['total'].to_frame('volume')           # set Vol aside
+        df.drop('total', axis=1, inplace=True)         # Drop volume data
+        ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
+        ohlc.cols = ohlc.cols.map(lambda t: t[1])      # Raname cols
+        closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
+        ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
+        vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin
+        ohlcv = pd.concat([ohlc, vol], axis=1)         # Concat OHLC + Vol
         return ohlcv
 
-
-
-    def write_ohlcv_file(self, currencyPair): 
+    def write_ohlcv_file(self, currencyPair):
         '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
-        '''       
+        '''
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
-        csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
+        csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
+        if(os.path.getmtime(csv_1min) > time.time() - 7200):
             log.debug(currencyPair+': 1min data file already up to date. '
                       'Delete the file if you want to rebuild it.')
         else:
-            df = pd.read_csv(csv_trades, 
-                                names=['tradeID',
-                                       'date',
-                                       'type',
-                                       'rate',
-                                       'amount',
-                                       'total',
-                                       'globalTradeID'], 
-                                dtype = {'tradeID': int, 
-                                         'date': str, 
-                                         'type': str, 
-                                         'rate': float, 
-                                         'amount': float, 
-                                         'total': float, 
-                                         'globalTradeID': int } 
-                                )
-            df.drop(['tradeID','type','amount','globalTradeID'], 
+            df = pd.read_csv(csv_trades,
+                             names=['tradeID',
+                                    'date',
+                                    'type',
+                                    'rate',
+                                    'amount',
+                                    'total',
+                                    'globalTradeID'],
+                             dtype={'tradeID': int,
+                                    'date': str,
+                                    'type': str,
+                                    'rate': float,
+                                    'amount': float,
+                                    'total': float,
+                                    'globalTradeID': int}
+                             )
+            df.drop(['tradeID', 'type', 'amount', 'globalTradeID'],
                     axis=1, inplace=True)
-            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
+            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
             ohlcv = self.generate_ohlcv(df)
-            try: 
+            try:
                 with open(csv_1min, 'w') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in ohlcv.itertuples():
@@ -305,32 +313,28 @@ class PoloniexCurator(object):
                             item.volume,
                         ])
             except Exception as e:
-                log.error('Error opening {}'.format(csv_fn))
+                log.error('Error opening {}'.format(csv_1min))
                 log.exception(e)
             log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
 
-
-
     def onemin_to_dataframe(self, currencyPair, start, end):
         '''
         Returns a data frame for a given currencyPair from data on disk
         '''
-        csv_fn     = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df         = pd.read_csv(csv_fn, names=['date', 
-                                                'open', 
-                                                'high', 
-                                                'low', 
-                                                'close', 
-                                                'volume']
-                                )
-        df['date'] = pd.to_datetime(df['date'],unit='s')
+        csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
+        df = pd.read_csv(csv_fn, names=['date',
+                                        'open',
+                                        'high',
+                                        'low',
+                                        'close',
+                                        'volume'])
+        df['date'] = pd.to_datetime(df['date'], unit='s')
         df.set_index('date', inplace=True)
-        return df[start : end]
-
+        return df[start:end]
 
     def generate_symbols_json(self, filename=None):
         '''
-        Generates a symbols.json file with corresponding start_date 
+        Generates a symbols.json file with corresponding start_date
         for each currencyPair
         '''
         symbol_map = {}
@@ -341,36 +345,37 @@ class PoloniexCurator(object):
         with open(filename, 'w') as symbols:
             for currencyPair in self.currency_pairs:
                 start = None
-                csv_fn     = '{}crypto_trades-{}.csv'.format(
-                                CSV_OUT_FOLDER, currencyPair)
-                with open(csv_fn, 'r') as f: 
+                csv_fn = '{}crypto_trades-{}.csv'.format(
+                                                    CSV_OUT_FOLDER,
+                                                    currencyPair)
+                with open(csv_fn, 'r') as f:
                     f.seek(0, os.SEEK_END)
                     if(f.tell() > 2):               # Check file size is not 0
                         f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
                         while f.read(1) != b"\n":   # Until EOL is found...
-                            f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
-                        start = pd.to_datetime( f.readline().split(',')[1], 
-                                                infer_datetime_format=True)
+                            # ...jump back the read byte plus one more.
+                            f.seek(-2, os.SEEK_CUR)
+                        start = pd.to_datetime(f.readline().split(',')[1],
+                                               infer_datetime_format=True)
 
                 if(start is None):
                     start = time.gmtime()
                 base, market = currencyPair.lower().split('_')
-                symbol = '{market}_{base}'.format( market=market, base=base )
+                symbol = '{market}_{base}'.format(market=market, base=base)
                 symbol_map[currencyPair] = dict(
-                    symbol = symbol,
-                    start_date = start.strftime("%Y-%m-%d")
+                    symbol=symbol,
+                    start_date=start.strftime("%Y-%m-%d")
                 )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2, 
-                        separators=(',',':'))    
+            json.dump(symbol_map, symbols, sort_keys=True, indent=2,
+                      separators=(',', ':'))
 
 
 if __name__ == '__main__':
     pc = PoloniexCurator()
     pc.get_currency_pairs()
-    #pc.generate_symbols_json()
-    
+    # pc.generate_symbols_json()
+
     for currencyPair in pc.currency_pairs:
         pc.retrieve_trade_history(currencyPair)
         log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
-

catalyst/data/bundles/__init__.py

@@ -1,6 +1,5 @@
 # These imports are necessary to force module-scope register calls to happen.
 from . import quandl  # noqa
-from . import poloniex
 from .core import (
     UnknownBundle,
     bundles,

catalyst/data/bundles/base.py

@@ -13,10 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from itertools import count
 import tarfile
-from time import time, sleep
+from time import sleep
 
 from abc import abstractmethod, abstractproperty
 import logbook
@@ -37,6 +36,7 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 
+
 class BaseBundle(object):
     def __init__(self, asset_filter=[]):
         self._asset_filter = asset_filter
@@ -104,11 +104,11 @@ class BaseBundle(object):
 
     def post_process_symbol_metadata(self, metadata, data):
         return metadata
-    
+
     @abstractmethod
     def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date):
         raise NotImplementedError()
-    
+
     def ingest(self,
                environ,
                asset_db_writer,
@@ -128,7 +128,7 @@ class BaseBundle(object):
             retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
 
             if is_compile:
-                # User has instructed local compilation and ingestion of bundle.
+                # User has instructed local compilation & ingestion of bundle.
                 # Fetch raw metadata for all symbols.
                 raw_metadata = self._fetch_metadata_frame(
                     api_key,
@@ -157,9 +157,9 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # Post-process metadata using cached symbol frames, and write to
-                # disk.  This metadata must be written before any attempt to write
-                # minute data.
+                # Post-process metadata using cached symbol frames, and write
+                # to disk.  This metadata must be written before any attempt
+                # to write minute data.
                 metadata = self._post_process_metadata(
                     raw_metadata,
                     cache,
@@ -184,10 +184,11 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # For legacy purposes, this call is required to ensure the database
-                # contains an appropriately initialized file structure.  We don't
-                # forsee a usecase for adjustments at this time, but may later
-                # choose to expose this functionality in the future.
+                # For legacy purposes, this call is required to ensure the
+                # database contains an appropriately initialized file
+                # structure.  We don't forsee a usecase for adjustments at
+                # this time, but may later choose to expose this functionality
+                # in the future.
                 adjustment_writer.write(
                     splits=(
                         pd.concat(self.splits, ignore_index=True)
@@ -232,12 +233,12 @@ class BaseBundle(object):
             tar.extractall(output_dir)
 
     def _fetch_metadata_frame(self,
-                             api_key,
-                             cache,
-                             retries=DEFAULT_RETRIES,
-                             environ=None,
-                             show_progress=False):
-        
+                              api_key,
+                              cache,
+                              retries=DEFAULT_RETRIES,
+                              environ=None,
+                              show_progress=False):
+
         # Setup raw metadata iterator to fetch pages if necessary.
         raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
 
@@ -251,7 +252,7 @@ class BaseBundle(object):
             show_percent=False,
         ) as blocks:
             metadata = pd.concat(blocks, ignore_index=True)
-        
+
         return metadata
 
     def _fetch_metadata_iter(self, api_key, cache, retries, environ):
@@ -269,21 +270,20 @@ class BaseBundle(object):
                             page_number,
                         )
                         break
-                    except ValueError as e:
+                    except ValueError:
                         raw = pd.DataFrame([])
                         break
-                    except Exception as e:
+                    except Exception:
                         log.exception(
                             'Failed to load metadata from {}. '
                             'Retrying.'.format(self.name)
-                        ) 
+                        )
                 else:
                     raise ValueError(
                         'Failed to download metadata page {} after {} '
                         'attempts.'.format(page_number, retries)
                     )
 
-
             if raw.empty:
                 # Empty DataFrame signals completion.
                 break
@@ -305,7 +305,7 @@ class BaseBundle(object):
             columns=self.md_column_names,
             index=metadata.index,
         )
-        
+
         # Iterate over the available symbols, loading the asset's raw symbol
         # data from the cache.  The final metadata is computed and recorded in
         # the appropriate row depending on the asset's id.
@@ -318,22 +318,22 @@ class BaseBundle(object):
             show_percent=False,
         ) as symbols_map:
             for asset_id, symbol in symbols_map:
-                # Attempt to load data from disk, the cache should have an entry
-                # for each symbol at this point of the execution. If one does
-                # not exist, we should fail.
+                # Attempt to load data from disk, the cache should have an
+                # entry for each symbol at this point of the execution. If one
+                # does not exist, we should fail.
                 key = '{sym}.daily.frame'.format(sym=symbol)
                 try:
                     raw_data = cache[key]
                 except KeyError:
                     raise ValueError(
-                      'Unable to find cached data for symbol: {0}'.format(symbol)
-                    )
+                      'Unable to find cached data for symbol:'
+                      ' {0}'.format(symbol))
 
                 # Perform and require post-processing of metadata.
                 final_symbol_metadata = self.post_process_symbol_metadata(
                     asset_id,
                     metadata.iloc[asset_id],
-                    raw_data,        
+                    raw_data,
                 )
 
                 # Record symbol's final metadata.
@@ -363,8 +363,8 @@ class BaseBundle(object):
             # returns the cached data unaltered.  The `should_sleep` flag
             # indicates that an API call was attempted, and that we should be
             # ensure aren't exceeding our rate limit before proceeding to the
-            # next symbol. If the raw_data is updated, it is cached before being
-            # returned.
+            # next symbol. If the raw_data is updated, it is cached before
+            # being returned.
             raw_data, should_sleep = self._maybe_update_symbol_frame(
                 start_time,
                 api_key,
@@ -414,7 +414,7 @@ class BaseBundle(object):
         last = start_session
         if raw_data is not None and len(raw_data) > 0:
             last = raw_data.index[-1].tz_localize('UTC')
-        
+
         should_sleep = False
 
         # Determine time at which cached data will be considered stale.
@@ -455,7 +455,7 @@ class BaseBundle(object):
                             retries=DEFAULT_RETRIES):
 
         # Data for symbol is old enough to attempt an update or is not
-        # present in the cache.  Fetch raw data for a single symbol 
+        # present in the cache.  Fetch raw data for a single symbol
         # with requested intervals and frequency. Retry as necessary.
         for _ in range(retries):
             try:
@@ -468,7 +468,6 @@ class BaseBundle(object):
                     data_frequency,
                 )
                 raw_data.index = pd.to_datetime(raw_data.index, utc=True)
-                #raw_data.index = raw_data.index.tz_localize('UTC')
 
                 # Filter incoming data to fit start and end sessions.
                 raw_data = raw_data[
@@ -482,7 +481,7 @@ class BaseBundle(object):
 
                 return raw_data
 
-            except Exception as e:
+            except Exception:
                 log.exception(
                     'Exception raised fetching {name} data. Retrying.'
                     .format(name=self.name)

catalyst/data/bundles/base_pricing.py

@@ -16,6 +16,7 @@
 from catalyst.data.bundles.base import BaseBundle
 from catalyst.utils.memoize import lazyval
 
+
 class BasePricingBundle(BaseBundle):
     @lazyval
     def md_dtypes(self):
@@ -38,6 +39,7 @@ class BasePricingBundle(BaseBundle):
             ('volume', 'float64'),
         ]
 
+
 class BaseCryptoPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):
@@ -55,6 +57,7 @@ class BaseCryptoPricingBundle(BasePricingBundle):
     def dividends(self):
         return []
 
+
 class BaseEquityPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):

catalyst/data/bundles/core.py

@@ -37,6 +37,7 @@ from catalyst.utils.cli import maybe_show_progress
 
 ONE_MEGABYTE = 1024 * 1024
 
+
 def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
     return pth.data_path(
         asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -135,6 +136,7 @@ def ingestions_for_bundle(bundle, environ=None):
         reverse=True,
     )
 
+
 def download_with_progress(url, chunk_size, **progress_kwargs):
     """
     Download streaming data from a URL, printing progress information to the
@@ -705,4 +707,5 @@ def _make_bundle_core():
     )
 
 
-bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
+bundles, register_bundle, register, unregister, ingest, load, clean = \
+    _make_bundle_core()

catalyst/data/bundles/poloniex.py

@@ -14,19 +14,17 @@
 # limitations under the License.
 
 import sys
-
-from datetime import datetime
+from six.moves.urllib.parse import urlencode
 
 import pandas as pd
 
-from six.moves.urllib.parse import urlencode
-
 from catalyst.data.bundles.core import register_bundle
 from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
 from catalyst.utils.memoize import lazyval
 
 from catalyst.curate.poloniex import PoloniexCurator
 
+
 class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def name(self):
@@ -46,7 +44,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def tar_url(self):
         return (
-            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
+            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
+            'poloniex/poloniex-bundle.tar.gz'
         )
 
     @lazyval
@@ -67,12 +66,11 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         raw = raw.sort_index().reset_index()
         raw.rename(
-            columns={'index':'symbol'},
+            columns={'index': 'symbol'},
             inplace=True,
         )
 
         raw = raw[raw['isFrozen'] == 0]
-
         return raw
 
     def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
@@ -98,7 +96,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                                frequency):
 
         # TODO: replace this with direct exchange call
-        # The end date and frequency should be used to calculate the number of bars
+        # The end date and frequency should be used to
+        # calculate the number of bars
         if(frequency == 'minute'):
             pc = PoloniexCurator()
             raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
@@ -116,8 +115,9 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             )
             raw.set_index('date', inplace=True)
 
-        # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
-        # on disk, which we compensate here to get the right pricing amounts
+        # BcolzDailyBarReader introduces a 1/1000 factor in the way
+        # pricing is stored on disk, which we compensate here to get
+        # the right pricing amounts
         # ref: data/us_equity_pricing.py
         scale = 1
         raw.loc[:, 'open'] /= scale
@@ -139,7 +139,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         return self._format_polo_query(query_params)
 
-
     def _format_data_url(self,
                          api_key,
                          symbol,
@@ -162,27 +161,26 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             ('end', end_date.value / 10**9),
             ('period', period),
         ]
-            
+
         return self._format_polo_query(query_params)
-    
+
     def _format_polo_query(self, query_params):
         # TODO: got against the exchange object
         return 'https://poloniex.com/public?{query}'.format(
             query=urlencode(query_params),
         )
 
-''' 
-As a second parameter, you can pass an array of currency pairs 
-that will be processed as an asset_filter to only process that 
+
+'''
+As a second parameter, you can pass an array of currency pairs
+that will be processed as an asset_filter to only process that
 subset of assets in the bundle, such as:
 register_bundle(PoloniexBundle, ['USDT_BTC',])
 
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-
 if 'ingest' in sys.argv and '-c' in sys.argv:
     register_bundle(PoloniexBundle)
 else:
     register_bundle(PoloniexBundle, create_writers=False)
-

catalyst/data/bundles/quandl.py

@@ -16,7 +16,6 @@
 from datetime import datetime
 
 import pandas as pd
-
 from six.moves.urllib.parse import urlencode
 
 from catalyst.data.bundles.core import register_bundle
@@ -26,25 +25,16 @@ from catalyst.utils.memoize import lazyval
 """
 Module for building a complete daily dataset from Quandl's WIKI dataset.
 """
-from itertools import count
-import tarfile
-from time import time, sleep
-from datetime import datetime
-
 from logbook import Logger
-import pandas as pd
-from six.moves.urllib.parse import urlencode
-
-from catalyst.utils.calendars import register_calendar_alias
-from catalyst.utils.cli import maybe_show_progress
-
-from . import core as bundles
 
 from catalyst.constants import LOG_LEVEL
+from catalyst.utils.calendars import register_calendar_alias
+
 
 log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
+
 class QuandlBundle(BaseEquityPricingBundle):
     @lazyval
     def name(self):
@@ -109,8 +99,8 @@ class QuandlBundle(BaseEquityPricingBundle):
         # Filter out invalid symbols
         raw = raw[~raw.symbol.isin(self._excluded_symbols)]
 
-        # cut out all the other stuff in the name column
-        # we need to escape the paren because it is actually splitting on a regex
+        # cut out all the other stuff in the name column. We need to
+        # escape the paren because it is actually splitting on a regex
         raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
 
         return raw
@@ -175,7 +165,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['sid'] = asset_id
         self.splits.append(df)
 
-
     def _update_dividends(self, asset_id, raw_data):
         divs = raw_data.ex_dividend
         df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -186,7 +175,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
         self.dividends.append(df)
 
-
     def _format_metadata_url(self, api_key, page_number):
         """Build the query RL for the quandl WIKI metadata.
         """
@@ -200,10 +188,10 @@ class QuandlBundle(BaseEquityPricingBundle):
             query_params = [('api_key', api_key)] + query_params
 
         return (
-            'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
+            'https://www.quandl.com/api/v3/datasets.csv?'
+            + urlencode(query_params)
         )
 
-
     def _format_wiki_url(self,
                          api_key,
                          symbol,
@@ -229,5 +217,6 @@ class QuandlBundle(BaseEquityPricingBundle):
             )
         )
 
+
 register_calendar_alias('QUANDL', 'NYSE')
 register_bundle(QuandlBundle)

catalyst/data/data_portal.py

@@ -656,11 +656,11 @@ class DataPortal(object):
         return spot_value
 
     def _get_minutely_spot_value(self,
-                                asset,
-                                column,
-                                dt,
-                                data_frequency,
-                                ffill=False):
+                                 asset,
+                                 column,
+                                 dt,
+                                 data_frequency,
+                                 ffill=False):
 
         reader = self._get_pricing_reader(data_frequency)
 
@@ -706,7 +706,7 @@ class DataPortal(object):
             asset,
             column,
             dt,
-            ffill, 
+            ffill,
             'minute',
         )
 

catalyst/data/dispatch_bar_reader.py

@@ -133,11 +133,13 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
 
         return results
 
+
 class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):
         return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
 
+
 class AssetDispatchSessionBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):

catalyst/data/loader.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import datetime
 import os
 from collections import OrderedDict
 
@@ -129,11 +128,13 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # before this date.
     '''
     if(bundle_data):
-        # If we are using the bundle to retrieve the cryptobenchmark, find the last
-        # date for which there is trading data in the bundle
-        asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
+        # If we are using the bundle to retrieve the cryptobenchmark, find
+        # the last date for which there is trading data in the bundle
+        asset = bundle_data.asset_finder.lookup_symbol(
+                    symbol=bm_symbol,as_of_date=None)
         ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
-        last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
+        last_date = pd.to_datetime(
+                    bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
@@ -142,27 +143,30 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.poloniex.poloniex import Poloniex
-        exchange = Poloniex('', '', '')
+        from catalyst.exchange.factory import get_exchange
+        exchange = get_exchange(
+            exchange_name='poloniex', base_currency='usdt'
+        )
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 
     # exchange.get_history_window() already ensures that we have the right data
     # for the right dates
-    br = exchange.get_history_window(
+    br = exchange.get_history_window_with_bundle(
         assets=[benchmark_asset],
         end_dt=last_date,
         bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
-        data_frequency='daily')
+        data_frequency='daily',
+        force_auto_ingest=True)
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
     br.loc[start_dt] = 0
     br = br.sort_index()
 
-    # Override first_date for treasury data since we have it for many more years
-    # and is independent of crypto data
+    # Override first_date for treasury data since we have it for many more
+    # years and is independent of crypto data
     first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
     tc = ensure_treasury_data(
         bm_symbol,
@@ -298,14 +302,14 @@ def ensure_crypto_benchmark_data(symbol,
 
     if (bundle == 'poloniex'):
         '''
-        If we're using the Poloniex bundle, we'll get the benchmark from the bundle
-        instead of downloading it from Poloniex every time we need it.
-        Poloniex has a captcha for API queries originating from outside the US that 
-        prevents users abroad from getting Catalyst to work
+        If we're using the Poloniex bundle, we'll get the benchmark from the
+        bundle instead of downloading it from Poloniex every time we need it.
+        Poloniex has a captcha for API queries originating from outside the US
+        that prevents users abroad from getting Catalyst to work
         '''
         logger.info(
-            (
-                'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
+            ('Retrieving benchmark data from bundle for {symbol!r}'
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
@@ -327,11 +331,12 @@ def ensure_crypto_benchmark_data(symbol,
                         last_date)]
 
     else:
-        # This is how it used to be: downloading the benchmark everytime. 
-        # Leaving this code here to be repurposed in the future for other bundles.
+        # This is how it used to be: downloading the benchmark everytime.
+        # Leaving this code here to be repurposed in the future for
+        # other bundles.
         logger.info(
-            (
-                'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
+            ('Downloading benchmark data for {symbol!r}'
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         raise DeprecationWarning('poloniex bundle deprecated')
@@ -428,67 +433,6 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
     return data
 
 
-def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
-                          environ=None):
-    """
-    Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
-
-    Parameters
-    ----------
-    symbol : str
-        The symbol for the benchmark to load.
-    first_date : pd.Timestamp
-        First required date for the cache.
-    last_date : pd.Timestamp
-        Last required date for the cache.
-    now : pd.Timestamp
-        The current time.  This is used to prevent repeated attempts to
-        re-download data that isn't available due to scheduling quirks or other
-        failures.
-    trading_day : pd.CustomBusinessDay
-        A trading day delta.  Used to find the day before first_date so we can
-        get the close of the day prior to first_date.
-
-    We attempt to download data unless we already have data stored at the data
-    cache for `symbol` whose first entry is before or on `first_date` and whose
-    last entry is on or after `last_date`.
-
-    If we perform a download and the cache criteria are not satisfied, we wait
-    at least one hour before attempting a redownload.  This is determined by
-    comparing the current time to the result of os.path.getmtime on the cache
-    path.
-    """
-    filename = get_benchmark_filename(symbol)
-    data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
-                             environ)
-    if data is not None:
-        return data
-
-    # If no cached data was found or it was missing any dates then download the
-    # necessary data.
-    logger.info(
-        ('Downloading benchmark data for {symbol!r} '
-         'from {first_date} to {last_date}'),
-        symbol=symbol,
-        first_date=first_date - trading_day,
-        last_date=last_date
-    )
-
-    try:
-        data = get_benchmark_returns(
-            symbol,
-            first_date - trading_day,
-            last_date,
-        )
-        data.to_csv(get_data_filepath(filename, environ))
-    except (OSError, IOError, HTTPError):
-        logger.exception('Failed to cache the new benchmark returns')
-        raise
-    if not has_data_for_dates(data, first_date, last_date):
-        logger.warn("Still don't have expected data after redownload!")
-    return data
-
-
 def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
     """
     Ensure we have treasury data from treasury module associated with

catalyst/data/minute_bars.py

@@ -341,12 +341,10 @@ class BcolzMinuteBarMetadata(object):
             'end_session': str(self.end_session.date()),
             # Write these values for backwards compatibility
             'first_trading_day': str(self.start_session.date()),
-            'market_opens': (
-                market_opens.values.astype('datetime64[m]').
-                    astype(np.int64).tolist()),
-            'market_closes': (
-                market_closes.values.astype('datetime64[m]').
-                    astype(np.int64).tolist()),
+            'market_opens': (market_opens.values.astype('datetime64[m]').
+                             astype(np.int64).tolist()),
+            'market_closes': (market_closes.values.astype('datetime64[m]').
+                              astype(np.int64).tolist()),
         }
         with open(self.metadata_path(rootdir), 'w+') as fp:
             json.dump(metadata, fp)
@@ -1256,8 +1254,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 values = carray[start_idx:end_idx + 1]
                 if indices_to_exclude is not None:
                     for excl_start, excl_stop in indices_to_exclude[::-1]:
-                        excl_slice = np.s_[
-                                     excl_start - start_idx:excl_stop - start_idx + 1]
+                        excl_slice = np.s_[excl_start - start_idx:excl_stop
+                                           - start_idx + 1]
                         values = np.delete(values, excl_slice)
 
                 where = values != 0
@@ -1320,9 +1318,8 @@ class H5MinuteBarUpdateWriter(object):
 
     def __init__(self, path, complevel=None, complib=None):
         self._complevel = complevel if complevel \
-                                       is not None else self._COMPLEVEL
-        self._complib = complib if complib \
-                                   is not None else self._COMPLIB
+                            is not None else self._COMPLEVEL
+        self._complib = complib if complib is not None else self._COMPLIB
         self._path = path
 
     def write(self, frames):

catalyst/data/us_equity_pricing.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division     # Python2 req to have division of ints yield float
+from __future__ import division  # Python2 req for division of ints yield float
 
 from errno import ENOENT
 from functools import partial
@@ -120,7 +120,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
-PRICE_ADJUSTMENT_FACTOR = 1000000000    # Provides 9 decimals resolution. Also affects _equities.pyx L220
+# Provides 9 decimals resolution. Also affects _equities.pyx L220
+PRICE_ADJUSTMENT_FACTOR = 1000000000
 
 
 def check_uint32_safe(value, colname):
@@ -130,6 +131,7 @@ def check_uint32_safe(value, colname):
             "for uint32" % (value, colname)
         )
 
+
 def check_uint64_safe(value, colname):
     if value >= UINT64_MAX:
         raise ValueError(
@@ -322,8 +324,8 @@ class BcolzDailyBarWriter(object):
         # Maps column name -> output carray.
         columns = {
             k: carray(array([], dtype=uint64))
-               if k in OHLCV
-               else carray(array([], dtype=uint32))
+            if k in OHLCV
+            else carray(array([], dtype=uint32))
             for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
         }
 
@@ -439,11 +441,13 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed = (raw_data[list(OHLC)]
+                     * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
-        processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed['volume'] = (raw_data.volume
+                               * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         return ctable.fromdataframe(processed)
 
 
@@ -496,7 +500,7 @@ class BcolzDailyBarReader(SessionBarReader):
 
     The data in these columns is interpreted as follows:
 
-    - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted 
+    - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
       as 10^9 * as-traded dollar value.
     - Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
     - Id is the asset id of the row.

catalyst/examples/README.rst

@@ -0,0 +1,3 @@
+An overview of most of the trading strategies in this folder can be found in the 
+`Examples Algorithms <https://enigmampc.github.io/catalyst/example-algos.html>`_
+section of our documentation website.

catalyst/examples/arbitrage_with_interface.py

@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
     else:
         raise ValueError('invalid order action')
 
-    base_currency = enter_exchange.base_currency
-    base_currency_amount = enter_exchange.portfolio.cash
+    quote_currency = enter_exchange.quote_currency
+    quote_currency_amount = enter_exchange.portfolio.cash
 
     exit_balances = exit_exchange.get_balances()
     exit_currency = context.trading_pairs[
-        context.selling_exchange].market_currency
+        context.selling_exchange].quote_currency
 
     if exit_currency in exit_balances:
-        market_currency_amount = exit_balances[exit_currency]
+        quote_currency_amount = exit_balances[exit_currency]
     else:
         log.warn(
             'the selling exchange {exchange_name} does not hold '
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
         )
         return
 
-    if base_currency_amount < (amount * entry_price):
-        adj_amount = base_currency_amount / entry_price
+    if quote_currency_amount < (amount * entry_price):
+        adj_amount = quote_currency_amount / entry_price
         log.warn(
-            'not enough {base_currency} ({base_currency_amount}) to buy '
+            'not enough {quote_currency} ({quote_currency_amount}) to buy '
             '{amount}, adjusting the amount to {adj_amount}'.format(
-                base_currency=base_currency,
-                base_currency_amount=base_currency_amount,
+                quote_currency=quote_currency,
+                quote_currency_amount=quote_currency_amount,
                 amount=amount,
                 adj_amount=adj_amount
             )
         )
         amount = adj_amount
 
-    elif market_currency_amount < amount:
+    elif quote_currency_amount < amount:
         log.warn(
             'not enough {currency} ({currency_amount}) to sell '
             '{amount}, aborting'.format(
                 currency=exit_currency,
-                currency_amount=market_currency_amount,
+                currency_amount=quote_currency_amount,
                 amount=amount
             )
         )
@@ -263,13 +263,20 @@ def analyze(context, stats):
     pass
 
 
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='poloniex,bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc',
-    live_graph=False
-)
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'live'
+    if MODE == 'live':
+        run_algorithm(
+            capital_base=0.1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex,bitfinex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=True,
+            stats_output=None,
+        )

catalyst/examples/buy_and_hodl.py

@@ -14,30 +14,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (order_target_value, symbol, record,
+                          cancel_order, get_open_orders, )
 
-from catalyst.api import (
-    order_target_value,
-    symbol,
-    record,
-    cancel_order,
-    get_open_orders,
-)
 
 def initialize(context):
-    context.ASSET_NAME = 'BTC_USDT'
+    context.ASSET_NAME = 'btc_usd'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
     context.is_buying = True
     context.asset = symbol(context.ASSET_NAME)
 
     context.i = 0
 
+
 def handle_data(context, data):
     context.i += 1
 
@@ -60,12 +55,12 @@ def handle_data(context, data):
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
+        print('buying')
         # Place order to make position in asset equal to target_hodl_value
         order_target_value(
             context.asset,
             target_hodl_value,
-            limit_price=price*1.1,
-            stop_price=price*0.9,
+            limit_price=price * 1.1,
         )
 
     record(
@@ -76,28 +71,29 @@ def handle_data(context, data):
         leverage=context.account.leverage,
     )
 
+
 def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
 
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value (USD)')
+    ax1.set_ylabel('Portfolio\nValue\n(USD)')
 
     ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
+    ax2.set_ylabel('{asset}\n(USD)'.format(asset=context.ASSET_NAME))
+    results[['price']].plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     buys = trans.ix[
         [t[0]['amount'] > 0 for t in trans.transactions]
     ]
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
+    ax2.scatter(
+        buys.index.to_pydatetime(),
+        results.price[buys.index],
+        marker='^',
+        s=100,
+        c='g',
+        label=''
     )
 
     ax3 = plt.subplot(613, sharex=ax1)
@@ -124,14 +120,29 @@ def analyze(context=None, results=None):
         'algorithm',
         'benchmark',
     ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
+    ax5.set_ylabel('Percent\nChange')
 
     ax6 = plt.subplot(616, sharex=ax1)
     results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mCoins/5min)')
+    ax6.set_ylabel('Volume')
 
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
-    plt.show()
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bitfinex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_btc_simple.py

@@ -1,29 +1,49 @@
 '''
-	This is a very simple example referenced in the beginner's tutorial:
-	https://enigmampc.github.io/catalyst/beginner-tutorial.html
+    This is a very simple example referenced in the beginner's tutorial:
+    https://enigmampc.github.io/catalyst/beginner-tutorial.html
 
-	Run this example, by executing the following from your terminal:
-	catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+    Run this example, by executing the following from your terminal:
+      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
+      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
 
-	If you want to run this code using another exchange, make sure that 
-	the asset is available on that exchange. For example, if you were to run 
-	it for exchange Poloniex, you would need to edit the following line:
+    If you want to run this code using another exchange, make sure that
+    the asset is available on that exchange. For example, if you were to run
+    it for exchange Poloniex, you would need to edit the following line:
 
-		context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
+        context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
 
-	and specify exchange poloniex as follows:
+    and specify exchange poloniex as follows:
+    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
+    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
 
-	catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
-
-	To see which assets are available on each exchange, visit:
-	https://www.enigma.co/catalyst/status
+    To see which assets are available on each exchange, visit:
+    https://www.enigma.co/catalyst/status
 '''
-
+from catalyst import run_algorithm
 from catalyst.api import order, record, symbol
+import pandas as pd
+
 
 def initialize(context):
     context.asset = symbol('btc_usd')
 
+
 def handle_data(context, data):
     order(context.asset, 1)
-    record(btc = data.current(context.asset, 'price'))
+    record(btc=data.current(context.asset, 'price'))
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='bitfinex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_low_sell_high.py

@@ -1,17 +1,19 @@
 '''
-This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
-Install it first by running: 
+This algorithm requires an additional library (ta-lib) beyond those
+required by catalyst. Install it first by running:
 $ pip install TA-Lib
 
-If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and needs to be installed.
-See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install 
-the required dependencies.
+If you get build errors like:
+    "fatal error: ta-lib/ta_libc.h: No such file or directory"
+it typically means that it can't find the underlying TA-Lib library and it
+needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
+instructions on how to install the required dependencies.
 '''
 
 import talib
 from logbook import Logger
 
+from catalyst import run_algorithm
 from catalyst.api import (
     order,
     order_target_percent,
@@ -20,6 +22,7 @@ from catalyst.api import (
     get_open_orders,
 )
 from catalyst.exchange.stats_utils import get_pretty_stats
+import pandas as pd
 
 algo_namespace = 'buy_low_sell_high_xrp'
 log = Logger(algo_namespace)
@@ -100,8 +103,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
+        elif (position.amount > 0
+              and price > cost_basis * (1 + context.PROFIT_TARGET)):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -156,3 +159,18 @@ def handle_data(context, data):
 def analyze(context, stats):
     log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
     pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_low_sell_high_live.py

@@ -41,7 +41,7 @@ def _handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='1d'
+        frequency='1D'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
@@ -88,8 +88,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
+        elif (position.amount > 0
+              and price > cost_basis * (1 + context.PROFIT_TARGET)):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -146,23 +146,15 @@ def analyze(context, stats):
     pass
 
 
-run_algorithm(
-    capital_base=100000,
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='poloniex',
-    start=pd.to_datetime('2017-5-01', utc=True),
-    end=pd.to_datetime('2017-10-16', utc=True),
-    base_currency='usdt',
-    data_frequency='daily'
-)
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='poloniex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc'
-# )
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.001,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=algo_namespace,
+        base_currency='btc',
+        simulate_orders=True,
+    )

catalyst/examples/dual_moving_average.py

@@ -0,0 +1,162 @@
+import numpy as np
+import pandas as pd
+from logbook import Logger
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (record, symbol, order_target_percent,
+                          get_open_orders)
+from catalyst.exchange.stats_utils import extract_transactions
+
+NAMESPACE = 'dual_moving_average'
+log = Logger(NAMESPACE)
+
+
+def initialize(context):
+    context.i = 0
+    context.asset = symbol('ltc_usd')
+    context.base_price = None
+
+
+def handle_data(context, data):
+    # define the windows for the moving averages
+    short_window = 50
+    long_window = 200
+
+    # Skip as many bars as long_window to properly compute the average
+    context.i += 1
+    if context.i < long_window:
+        return
+
+    # Compute moving averages calling data.history() for each
+    # moving average with the appropriate parameters. We choose to use
+    # minute bars for this simulation -> freq="1m"
+    # Returns a pandas dataframe.
+    short_mavg = data.history(context.asset,
+                              'price',
+                              bar_count=short_window,
+                              frequency="1m",
+                              ).mean()
+    long_mavg = data.history(context.asset,
+                             'price',
+                             bar_count=long_window,
+                             frequency="1m",
+                             ).mean()
+
+    # Let's keep the price of our asset in a more handy variable
+    price = data.current(context.asset, 'price')
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+    price_change = (price - context.base_price) / context.base_price
+
+    # Save values for later inspection
+    record(price=price,
+           cash=context.portfolio.cash,
+           price_change=price_change,
+           short_mavg=short_mavg,
+           long_mavg=long_mavg)
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.asset):
+        return
+
+    # We check what's our position on our portfolio and trade accordingly
+    pos_amount = context.portfolio.positions[context.asset].amount
+
+    # Trading logic
+    if short_mavg > long_mavg and pos_amount == 0:
+        # we buy 100% of our portfolio for this asset
+        order_target_percent(context.asset, 1)
+    elif short_mavg < long_mavg and pos_amount > 0:
+        # we sell all our positions for this asset
+        order_target_percent(context.asset, 0)
+
+
+def analyze(context, perf):
+
+    # Get the base_currency that was passed as a parameter to the simulation
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # First chart: Plot portfolio value using base_currency
+    ax1 = plt.subplot(411)
+    perf.loc[:, ['portfolio_value']].plot(ax=ax1)
+    ax1.legend_.remove()
+    ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
+    start, end = ax1.get_ylim()
+    ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Second chart: Plot asset price, moving averages and buys/sells
+    ax2 = plt.subplot(412, sharex=ax1)
+    perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
+        ax=ax2,
+        label='Price')
+    ax2.legend_.remove()
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.asset.symbol,
+        base=base_currency
+        ))
+    start, end = ax2.get_ylim()
+    ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    # Third chart: Compare percentage change between our portfolio
+    # and the price of the asset
+    ax3 = plt.subplot(413, sharex=ax1)
+    perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
+    ax3.legend_.remove()
+    ax3.set_ylabel('Percent Change')
+    start, end = ax3.get_ylim()
+    ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Fourth chart: Plot our cash
+    ax4 = plt.subplot(414, sharex=ax1)
+    perf.cash.plot(ax=ax4)
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+    start, end = ax4.get_ylim()
+    ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='usd',
+            start=pd.to_datetime('2017-9-22', utc=True),
+            end=pd.to_datetime('2017-9-23', utc=True),
+        )

catalyst/examples/dual_vwap.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2017 Enigma MPC, Inc.
-# Copyright 2014 Quantopian, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from catalyst.api import (
-    order_target_percent,
-    record,
-    symbol,
-    get_open_orders,
-    set_max_leverage,
-    schedule_function,
-    date_rules,
-    attach_pipeline,
-    pipeline_output,
-)
-
-from catalyst.pipeline import Pipeline
-from catalyst.pipeline.data import CryptoPricing
-from catalyst.pipeline.factors.crypto import VWAP
-
-
-def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
-    context.TARGET_INVESTMENT_RATIO = 0.8
-    context.SHORT_WINDOW = 30
-    context.LONG_WINDOW = 100
-
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
-    context.i = 0
-    context.asset = symbol(context.ASSET_NAME)
-
-    set_max_leverage(1.0)
-
-    attach_pipeline(make_pipeline(context), 'vwap_pipeline')
-
-    schedule_function(
-        rebalance,
-        time_rules=times_rules.every_minute(),
-    )
-
-
-def before_trading_start(context, data):
-    context.pipeline_data = pipeline_output('vwap_pipeline')
-
-def make_pipeline(context):
-    return Pipeline(
-        columns={
-            'price': CryptoPricing.open.latest,
-            'volume': CryptoPricing.volume.latest,
-            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
-            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
-        }
-    )
-
-def rebalance(context, data):
-    context.i += 1
-
-    # skip first LONG_WINDOW bars to fill windows
-    if context.i < context.LONG_WINDOW:
-        return
-
-    # get pipeline data for asset of interest
-    pipeline_data = context.pipeline_data
-    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
-
-    # retrieve long and short moving averages from pipeline
-    short_mavg = pipeline_data.short_mavg
-    long_mavg = pipeline_data.long_mavg
-    price = pipeline_data.price
-    volume = pipeline_data.volume
-
-    # check that order has not already been placed
-    open_orders = get_open_orders()
-    if context.asset not in open_orders:
-        # check that the asset of interest can currently be traded
-        if data.can_trade(context.asset):
-            # adjust portfolio based on comparison of long and short vwap
-            if short_mavg > long_mavg:
-                order_target_percent(
-                    context.asset,
-                    context.TARGET_INVESTMENT_RATIO,
-                )
-            elif short_mavg < long_mavg:
-                order_target_percent(
-                    context.asset,
-                    0.0,
-                )
-
-    record(
-        price=price,
-        cash=context.portfolio.cash,
-        leverage=context.account.leverage,
-        short_mavg=short_mavg,
-        long_mavg=long_mavg,
-        volume=volume,
-    )
-    
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(611)
-    results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio value (USD)')
-
-    ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
-
-    trans = results.ix[[t != [] for t in results.transactions]]
-    amounts = [t[0]['amount'] for t in trans.transactions]
-
-    buys = trans.ix[
-        [t[0]['amount'] > 0 for t in trans.transactions]
-    ]
-    sells = trans.ix[
-        [t[0]['amount'] < 0 for t in trans.transactions]
-    ]
-
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        context.TICK_SIZE * results.price[sells.index],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(613, sharex=ax1)
-    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-    ax3.set_ylabel('Leverage (USD)')
-
-    ax4 = plt.subplot(614, sharex=ax1)
-    results[['cash']].plot(ax=ax4)
-    ax4.set_ylabel('Cash (USD)')
-
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]] = results[[
-        'treasury_period_return',
-        'algorithm_period_return',
-        'benchmark_period_return',
-    ]]
-
-    ax5 = plt.subplot(615, sharex=ax1)
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(616, sharex=ax1)
-    results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mBTC/day)')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()

catalyst/examples/mean_reversion.py

@@ -1,283 +0,0 @@
-# For this example, we're going to write a simple momentum script.  When the 
-# stock goes up quickly, we're going to buy; when it goes down quickly, we're
-# going to sell.  Hopefully we'll ride the waves.
-from datetime import timedelta
-
-import pandas as pd
-import talib
-# To run an algorithm in Catalyst, you need two functions: initialize and
-# handle_data.
-from logbook import Logger
-from talib.common import MA_Type
-
-from catalyst import run_algorithm
-from catalyst.api import symbol, record, order_target_percent, \
-    get_open_orders
-# We give a name to the algorithm which Catalyst will use to persist its state.
-# In this example, Catalyst will create the `.catalyst/data/live_algos`
-# directory. If we stop and start the algorithm, Catalyst will resume its
-# state using the files included in the folder.
-from catalyst.exchange.stats_utils import extract_transactions, trend_direction
-
-algo_namespace = 'momentum'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    # This initialize function sets any data or variables that you'll use in
-    # your algorithm.  For instance, you'll want to define the trading pair (or
-    # trading pairs) you want to backtest.  You'll also want to define any
-    # parameters or values you're going to use.
-
-    # In our example, we're looking at Ether in USD Tether.
-    context.eth_btc = symbol('etc_usdt')
-    context.base_price = None
-    context.current_day = None
-    context.trigger = None
-
-
-def handle_data(context, data):
-    # This handle_data function is where the real work is done.  Our data is
-    # minute-level tick data, and each minute is called a frame.  This function
-    # runs on each frame of the data.
-
-    # We flag the first period of each day.
-    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
-    # would only execute once. This method works with minute and daily
-    # frequencies.
-    today = data.current_dt.floor('1D')
-    if today != context.current_day:
-        context.traded_today = False
-        context.current_day = today
-
-    # We're computing the volume-weighted-average-price of the security
-    # defined above, in the context.eth_btc variable.  For this example, we're 
-    # using three bars on the 15 min bars.
-
-    # The frequency attribute determine the bar size. We use this convention
-    # for the frequency alias:
-    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
-    prices = data.history(
-        context.eth_btc,
-        fields='close',
-        bar_count=50,
-        frequency='15T'
-    )
-
-    # Ta-lib calculates various technical indicator based on price and
-    # volume arrays.
-
-    # In this example, we are comp
-    rsi = talib.RSI(prices.values, timeperiod=14)
-    upper, middle, lower = talib.BBANDS(
-        prices.values,
-        timeperiod=20,
-        nbdevup=2,
-        nbdevdn=2,
-        matype=MA_Type.EMA
-    )
-
-    # We need a variable for the current price of the security to compare to
-    # the average. Since we are requesting two fields, data.current()
-    # returns a DataFrame with
-    current = data.current(context.eth_btc, fields=['close', 'volume'])
-    price = current['close']
-
-    # If base_price is not set, we use the current value. This is the
-    # price at the first bar which we reference to calculate price_change.
-    if context.base_price is None:
-        context.base_price = price
-
-    price_change = (price - context.base_price) / context.base_price
-    cash = context.portfolio.cash
-
-    # Now that we've collected all current data for this frame, we use
-    # the record() method to save it. This data will be available as
-    # a parameter of the analyze() function for further analysis.
-    record(
-        price=price,
-        volume=current['volume'],
-        upper_band=upper[-1],
-        lower_band=lower[-1],
-        price_change=price_change,
-        rsi=rsi[-1],
-        cash=cash
-    )
-
-    # We are trying to avoid over-trading by limiting our trades to
-    # one per day.
-    if context.traded_today:
-        return
-
-    # Since we are using limit orders, some orders may not execute immediately
-    # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.eth_btc)
-    if len(orders) > 0:
-        return
-
-    # Exit if we cannot trade
-    if not data.can_trade(context.eth_btc):
-        return
-
-    # Another powerful built-in feature of the Catalyst backtester is the
-    # portfolio object.  The portfolio object tracks your positions, cash,
-    # cost basis of specific holdings, and more.  In this line, we calculate
-    # how long or short our position is at this minute.   
-    pos_amount = context.portfolio.positions[context.eth_btc].amount
-
-    # In this example, we're using a trigger instead of buying directly after
-    # a signal. Since this is mean reversion, our signals go against the
-    # momentum. Using a trigger allow us to spot the opportunity but trade
-    # only when a trade reversal begins.
-    if context.trigger is not None:
-        # The tread_direction() method determines the trend based on the last
-        # two bars of the series.
-        direction = trend_direction(rsi)
-        if context.trigger[1] == 'buy' and direction == 'up':
-            log.info(
-                '{}: buying - price: {}, rsi: {}, bband: {}'.format(
-                    data.current_dt, price, rsi[-1], lower[-1]
-                )
-            )
-            order_target_percent(context.eth_btc, 1)
-            context.traded_today = True
-            context.trigger = None
-
-        elif context.trigger[1] == 'sell' and direction == 'down':
-            log.info(
-                '{}: selling - price: {}, rsi: {}, bband: {}'.format(
-                    data.current_dt, price, rsi[-1], upper[-1]
-                )
-            )
-            order_target_percent(context.eth_btc, 0)
-            context.traded_today = True
-            context.trigger = None
-
-        # If we found a signal but no trade reversal within two hours, we
-        # reset the trigger.
-        elif context.trigger[0] + timedelta(hours=2) < data.current_dt:
-            context.trigger = None
-
-    else:
-        # Determining the entry and exit signals based on RSI and SMA
-        if rsi[-1] <= 30 and pos_amount == 0:
-            context.trigger = (data.current_dt, 'buy')
-
-        elif rsi[-1] >= 80 and pos_amount > 0:
-            context.trigger = (data.current_dt, 'sell')
-
-
-def analyze(context=None, perf=None):
-    import matplotlib.pyplot as plt
-
-    # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
-
-    # Plot the portfolio value over time.
-    ax1 = plt.subplot(611)
-    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
-
-    # Plot the price increase or decrease over time.
-    ax2 = plt.subplot(612, sharex=ax1)
-    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
-    perf.loc[:, 'upper_band'].plot(ax=ax2, label='Upper')
-    perf.loc[:, 'lower_band'].plot(ax=ax2, label='Lower')
-
-    ax2.set_ylabel('{asset} ({base})'.format(
-        asset=context.eth_btc.symbol, base=base_currency
-    ))
-
-    transaction_df = extract_transactions(perf)
-    if not transaction_df.empty:
-        buy_df = transaction_df[transaction_df['amount'] > 0]
-        sell_df = transaction_df[transaction_df['amount'] < 0]
-        ax2.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'price'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax2.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'price'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-
-    ax4 = plt.subplot(613, sharex=ax1)
-    perf.loc[:, 'cash'].plot(
-        ax=ax4, label='Base Currency ({})'.format(base_currency)
-    )
-    ax4.set_ylabel('Cash ({})'.format(base_currency))
-
-    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
-
-    ax5 = plt.subplot(614, sharex=ax1)
-    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(615, sharex=ax1)
-    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-    ax6.axhline(70, color='darkgoldenrod')
-    ax6.axhline(30, color='darkgoldenrod')
-
-    if not transaction_df.empty:
-        ax6.scatter(
-            buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'rsi'],
-            marker='^',
-            s=100,
-            c='green',
-            label=''
-        )
-        ax6.scatter(
-            sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'rsi'],
-            marker='v',
-            s=100,
-            c='red',
-            label=''
-        )
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-    pass
-
-
-if __name__ == '__main__':
-    # The execution mode: backtest or live
-    MODE = 'backtest'
-
-    if MODE == 'backtest':
-        run_algorithm(
-            capital_base=1,
-            data_frequency='minute',
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            algo_namespace=algo_namespace,
-            base_currency='usdt',
-            start=pd.to_datetime('2017-7-1', utc=True),
-            # end=pd.to_datetime('2017-9-30', utc=True),
-            end=pd.to_datetime('2017-10-31', utc=True),
-        )
-
-    elif MODE == 'live':
-        run_algorithm(
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='poloniex',
-            live=True,
-            algo_namespace=algo_namespace,
-            base_currency='usdt',
-            live_graph=True
-        )

catalyst/examples/mean_reversion_simple.py

@@ -1,39 +1,52 @@
-# For this example, we're going to write a simple momentum script.  When the 
+# For this example, we're going to write a simple momentum script.  When the
 # stock goes up quickly, we're going to buy; when it goes down quickly, we're
 # going to sell.  Hopefully we'll ride the waves.
-from datetime import timedelta
+import os
+import tempfile
+import time
 
+import numpy as np
 import pandas as pd
 import talib
-# To run an algorithm in Catalyst, you need two functions: initialize and
-# handle_data.
 from logbook import Logger
-from talib.common import MA_Type
 
 from catalyst import run_algorithm
-from catalyst.api import symbol, record, order_target_percent, \
-    get_open_orders
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.stats_utils import extract_transactions
+
 # We give a name to the algorithm which Catalyst will use to persist its state.
 # In this example, Catalyst will create the `.catalyst/data/live_algos`
 # directory. If we stop and start the algorithm, Catalyst will resume its
 # state using the files included in the folder.
-from catalyst.exchange.stats_utils import extract_transactions, trend_direction
+from catalyst.utils.paths import ensure_directory
 
-algo_namespace = 'momentum'
-log = Logger(algo_namespace)
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
 
 
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
 def initialize(context):
     # This initialize function sets any data or variables that you'll use in
     # your algorithm.  For instance, you'll want to define the trading pair (or
     # trading pairs) you want to backtest.  You'll also want to define any
     # parameters or values you're going to use.
 
-    # In our example, we're looking at Ether in USD Tether.
-    context.eth_btc = symbol('etc_usdt')
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('neo_eth')
     context.base_price = None
     context.current_day = None
 
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERBOUGHT = 80
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    # context.set_commission(maker=0.1, taker=0.2)
+    context.set_slippage(spread=0.0001)
+
 
 def handle_data(context, data):
     # This handle_data function is where the real work is done.  Our data is
@@ -50,17 +63,17 @@ def handle_data(context, data):
         context.current_day = today
 
     # We're computing the volume-weighted-average-price of the security
-    # defined above, in the context.eth_btc variable.  For this example, we're 
+    # defined above, in the context.market variable.  For this example, we're
     # using three bars on the 15 min bars.
 
     # The frequency attribute determine the bar size. We use this convention
     # for the frequency alias:
     # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
     prices = data.history(
-        context.eth_btc,
+        context.market,
         fields='close',
         bar_count=50,
-        frequency='15T'
+        frequency=context.CANDLE_SIZE
     )
 
     # Ta-lib calculates various technical indicator based on price and
@@ -72,7 +85,7 @@ def handle_data(context, data):
     # We need a variable for the current price of the security to compare to
     # the average. Since we are requesting two fields, data.current()
     # returns a DataFrame with
-    current = data.current(context.eth_btc, fields=['close', 'volume'])
+    current = data.current(context.market, fields=['close', 'volume'])
     price = current['close']
 
     # If base_price is not set, we use the current value. This is the
@@ -86,71 +99,82 @@ def handle_data(context, data):
     # Now that we've collected all current data for this frame, we use
     # the record() method to save it. This data will be available as
     # a parameter of the analyze() function for further analysis.
+
     record(
-        price=price,
         volume=current['volume'],
+        price=price,
         price_change=price_change,
         rsi=rsi[-1],
         cash=cash
     )
-
     # We are trying to avoid over-trading by limiting our trades to
     # one per day.
     if context.traded_today:
         return
 
+    # TODO: retest with open orders
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.eth_btc)
+    orders = get_open_orders(context.market)
     if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
         return
 
     # Exit if we cannot trade
-    if not data.can_trade(context.eth_btc):
+    if not data.can_trade(context.market):
         return
 
     # Another powerful built-in feature of the Catalyst backtester is the
     # portfolio object.  The portfolio object tracks your positions, cash,
     # cost basis of specific holdings, and more.  In this line, we calculate
-    # how long or short our position is at this minute.   
-    pos_amount = context.portfolio.positions[context.eth_btc].amount
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
 
-    if rsi[-1] <= 30 and pos_amount == 0:
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
         log.info(
             '{}: buying - price: {}, rsi: {}'.format(
                 data.current_dt, price, rsi[-1]
             )
         )
-        order_target_percent(context.eth_btc, 1)
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
         context.traded_today = True
 
-    elif rsi[-1] >= 80 and pos_amount > 0:
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
         log.info(
             '{}: selling - price: {}, rsi: {}'.format(
                 data.current_dt, price, rsi[-1]
             )
         )
-        order_target_percent(context.eth_btc, 0)
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
         context.traded_today = True
 
 
 def analyze(context=None, perf=None):
-    import matplotlib.pyplot as plt
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
 
+    import matplotlib.pyplot as plt
     # The base currency of the algo exchange
     base_currency = context.exchanges.values()[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
     perf.loc[:, 'portfolio_value'].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
 
     # Plot the price increase or decrease over time.
     ax2 = plt.subplot(612, sharex=ax1)
     perf.loc[:, 'price'].plot(ax=ax2, label='Price')
 
-    ax2.set_ylabel('{asset} ({base})'.format(
-        asset=context.eth_btc.symbol, base=base_currency
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
     ))
 
     transaction_df = extract_transactions(perf)
@@ -159,7 +183,7 @@ def analyze(context=None, perf=None):
         sell_df = transaction_df[transaction_df['amount'] < 0]
         ax2.scatter(
             buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'price'],
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
             marker='^',
             s=100,
             c='green',
@@ -167,7 +191,7 @@ def analyze(context=None, perf=None):
         )
         ax2.scatter(
             sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'price'],
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
             marker='v',
             s=100,
             c='red',
@@ -178,23 +202,24 @@ def analyze(context=None, perf=None):
     perf.loc[:, 'cash'].plot(
         ax=ax4, label='Base Currency ({})'.format(base_currency)
     )
-    ax4.set_ylabel('Cash ({})'.format(base_currency))
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
 
     perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
 
     ax5 = plt.subplot(614, sharex=ax1)
     perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
+    ax5.set_ylabel('Percent\nChange')
 
     ax6 = plt.subplot(615, sharex=ax1)
     perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
-    ax6.axhline(70, color='darkgoldenrod')
-    ax6.axhline(30, color='darkgoldenrod')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
 
     if not transaction_df.empty:
         ax6.scatter(
             buy_df.index.to_pydatetime(),
-            perf.loc[buy_df.index, 'rsi'],
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
             marker='^',
             s=100,
             c='green',
@@ -202,13 +227,15 @@ def analyze(context=None, perf=None):
         )
         ax6.scatter(
             sell_df.index.to_pydatetime(),
-            perf.loc[sell_df.index, 'rsi'],
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
             marker='v',
             s=100,
             c='red',
             label=''
         )
     plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
@@ -221,28 +248,42 @@ if __name__ == '__main__':
     MODE = 'backtest'
 
     if MODE == 'backtest':
-        # catalyst run -f catalyst/examples/mean_reversion_simple.py -x poloniex -s 2017-7-1 -e 2017-7-31 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
         run_algorithm(
-            capital_base=10000,
+            capital_base=0.1,
             data_frequency='minute',
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
-            exchange_name='poloniex',
-            algo_namespace=algo_namespace,
-            base_currency='usdt',
-            start=pd.to_datetime('2017-7-1', utc=True),
-            end=pd.to_datetime('2017-7-31', utc=True),
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
         )
+        log.info('saved perf stats: {}'.format(out))
 
     elif MODE == 'live':
         run_algorithm(
+            capital_base=0.05,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
-            exchange_name='poloniex',
+            exchange_name='binance',
             live=True,
-            algo_namespace=algo_namespace,
-            base_currency='usdt',
-            live_graph=True
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True,
+            stats_output=None
         )

catalyst/examples/portfolio_optimization.py

@@ -0,0 +1,149 @@
+'''Use this code to execute a portfolio optimization model. This code
+   will select the portfolio with the maximum Sharpe Ratio. The parameters
+   are set to use 180 days of historical data and rebalance every 30 days.
+
+   This is the code used in the following article:
+   https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
+
+   You can run this code using the Python interpreter:
+
+   $ python portfolio_optimization.py
+'''
+
+from __future__ import division
+import os
+import pytz
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+from catalyst.api import record, symbols, order_target_percent
+from catalyst.utils.run_algo import run_algorithm
+
+np.set_printoptions(threshold='nan', suppress=True)
+
+
+def initialize(context):
+    # Portfolio assets list
+    context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
+                             'xmr_usdt')
+    context.nassets = len(context.assets)
+    # Set the time window that will be used to compute expected return
+    # and asset correlations
+    context.window = 180
+    # Set the number of days between each portfolio rebalancing
+    context.rebalance_period = 30
+    context.i = 0
+
+
+def handle_data(context, data):
+    # Only rebalance at the beggining of the algorithm execution and
+    # every multiple of the rebalance period
+    if context.i == 0 or context.i % context.rebalance_period == 0:
+        n = context.window
+        prices = data.history(context.assets, fields='price',
+                              bar_count=n + 1, frequency='1d')
+        pr = np.asmatrix(prices)
+        t_prices = prices.iloc[1:n + 1]
+        t_val = t_prices.values
+        tminus_prices = prices.iloc[0:n]
+        tminus_val = tminus_prices.values
+        # Compute daily returns (r)
+        r = np.asmatrix(t_val / tminus_val - 1)
+        # Compute the expected returns of each asset with the average
+        # daily return for the selected time window
+        m = np.asmatrix(np.mean(r, axis=0))
+        # ###
+        stds = np.std(r, axis=0)
+        # Compute excess returns matrix (xr)
+        xr = r - m
+        # Matrix algebra to get variance-covariance matrix
+        cov_m = np.dot(np.transpose(xr), xr) / n
+        # Compute asset correlation matrix (informative only)
+        corr_m = cov_m / np.dot(np.transpose(stds), stds)
+
+        # Define portfolio optimization parameters
+        n_portfolios = 50000
+        results_array = np.zeros((3 + context.nassets, n_portfolios))
+        for p in xrange(n_portfolios):
+            weights = np.random.random(context.nassets)
+            weights /= np.sum(weights)
+            w = np.asmatrix(weights)
+            p_r = np.sum(np.dot(w, np.transpose(m))) * 365
+            p_std = np.sqrt(np.dot(np.dot(w, cov_m),
+                                   np.transpose(w))) * np.sqrt(365)
+
+            # store results in results array
+            results_array[0, p] = p_r
+            results_array[1, p] = p_std
+            # store Sharpe Ratio (return / volatility) - risk free rate element
+            # excluded for simplicity
+            results_array[2, p] = results_array[0, p] / results_array[1, p]
+            i = 0
+            for iw in weights:
+                results_array[3 + i, p] = weights[i]
+                i += 1
+
+        # convert results array to Pandas DataFrame
+        results_frame = pd.DataFrame(np.transpose(results_array),
+                                     columns=['r', 'stdev', 'sharpe']
+                                             + context.assets)
+        # locate position of portfolio with highest Sharpe Ratio
+        max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
+        # locate positon of portfolio with minimum standard deviation
+        # min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+
+        # order optimal weights for each asset
+        for asset in context.assets:
+            if data.can_trade(asset):
+                order_target_percent(asset, max_sharpe_port[asset])
+
+        # create scatter plot coloured by Sharpe Ratio
+        plt.scatter(results_frame.stdev,
+                    results_frame.r,
+                    c=results_frame.sharpe,
+                    cmap='RdYlGn')
+        plt.xlabel('Volatility')
+        plt.ylabel('Returns')
+        plt.colorbar()
+        # plot red star to highlight position of portfolio
+        # with highest Sharpe Ratio
+        plt.scatter(max_sharpe_port[1],
+                    max_sharpe_port[0],
+                    marker='o',
+                    color='b',
+                    s=200)
+        # plot green star to highlight position of minimum variance portfolio
+        plt.show()
+        print(max_sharpe_port)
+        record(pr=pr,
+               r=r,
+               m=m,
+               stds=stds,
+               max_sharpe_port=max_sharpe_port,
+               corr_m=corr_m)
+    context.i += 1
+
+
+def analyze(context=None, results=None):
+    # Form DataFrame with selected data
+    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
+                    'portfolio_value']]
+
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename(__file__))[0]
+    data.to_csv(filename + '.csv')
+
+
+if __name__ == '__main__':
+    # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
+    start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+    end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
+    results = run_algorithm(initialize=initialize,
+                            handle_data=handle_data,
+                            analyze=analyze,
+                            start=start,
+                            end=end,
+                            exchange_name='poloniex',
+                            capital_base=100000, )

catalyst/examples/rsi_profit_target.py

@@ -11,7 +11,6 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.stats_utils import crossover, crossunder
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'rsi'
@@ -55,7 +54,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
             stop=None
         )
 
-    action = None
+    # action = None
     if context.position is not None:
         cost_basis = context.position['cost_basis']
         amount = context.position['amount']
@@ -80,7 +79,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
                 amount=-amount,
                 limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
             )
-            action = 0
+            # action = 0
             context.position = None
 
     else:
@@ -97,7 +96,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
                 amount=buy_amount,
                 stop=None
             )
-            action = 0
+            # action = 0
 
 
 def _handle_data_rsi_only(context, data):
@@ -115,7 +114,7 @@ def _handle_data_rsi_only(context, data):
         prices = data.history(
             context.asset,
             fields='price',
-            bar_count=17,
+            bar_count=20,
             frequency='30T'
         )
     except Exception as e:
@@ -157,7 +156,7 @@ def handle_data(context, data):
     dt = data.current_dt
 
     if context.last_bar is None or (
-                context.last_bar + timedelta(minutes=15)) <= dt:
+            context.last_bar + timedelta(minutes=15)) <= dt:
         context.last_bar = dt
     else:
         return
@@ -250,27 +249,17 @@ def analyze(context=None, results=None):
     pass
 
 
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bittrex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc',
-    live_graph=False
-)
-
-# Backtest
-# run_algorithm(
-#     capital_base=0.5,
-#     data_frequency='minute',
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='poloniex',
-#     algo_namespace=algo_namespace,
-#     base_currency='btc',
-#     start=pd.to_datetime('2017-9-1', utc=True),
-#     end=pd.to_datetime('2017-10-1', utc=True),
-# )
+if __name__ == '__main__':
+    # Backtest
+    run_algorithm(
+        capital_base=0.5,
+        data_frequency='minute',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        algo_namespace=algo_namespace,
+        base_currency='btc',
+        start=pd.to_datetime('2017-9-1', utc=True),
+        end=pd.to_datetime('2017-10-1', utc=True),
+    )

catalyst/examples/simple_loop.py

@@ -2,12 +2,15 @@ import talib
 import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.api import symbol
+from catalyst.api import symbol, record
+from catalyst.exchange.stats_utils import get_pretty_stats, \
+    extract_transactions
 
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('swift_btc')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
 
 
 def handle_data(context, data):
@@ -16,37 +19,107 @@ def handle_data(context, data):
     price = data.current(context.asset, 'close')
     print('got price {price}'.format(price=price))
 
-    try:
-        prices = data.history(
-            context.asset,
-            fields='price',
-            bar_count=15,
-            frequency='1D'
+    prices = data.history(
+        context.asset,
+        fields='price',
+        bar_count=20,
+        frequency='30T'
+    )
+    last_traded = prices.index[-1]
+    print('last candle date: {}'.format(last_traded))
+
+    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
+    print('got rsi: {}'.format(rsi))
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        price_change=price_change,
+        cash=cash
+    )
+
+
+def analyze(context, perf):
+    import matplotlib.pyplot as plt
+    print('the stats: {}'.format(get_pretty_stats(perf)))
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
         )
-        rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-        print('got rsi: {}'.format(rsi))
-    except Exception as e:
-        print(e)
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
 
 
-run_algorithm(
-    capital_base=250,
-    start=pd.to_datetime('2015-4-1', utc=True),
-    end=pd.to_datetime('2017-11-1', utc=True),
-    data_frequency='daily',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=None,
-    exchange_name='bittrex',
-    algo_namespace='simple_loop',
-    base_currency='btc'
-)
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=None,
-#     exchange_name='poloniex',
-#     live=True,
-#     algo_namespace='simple_loop',
-#     base_currency='eth',
-#     live_graph=False
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=1,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=None,
+        exchange_name='poloniex',
+        live=True,
+        algo_namespace='simple_loop',
+        base_currency='eth',
+        live_graph=False,
+        simulate_orders=True
+    )

catalyst/examples/simple_universe.py

@@ -0,0 +1,171 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.3
+
+This example aims to provide an easy way for users to learn how to
+collect data from any given exchange and select a subset of the available
+currency pairs for trading. You simply need to specify the exchange and
+the market (base_currency) that you want to focus on. You will then see
+how to create a universe of assets, and filter it based the market you
+desire.
+
+The example prints out the closing price of all the pairs for a given
+market in a given exchange every 30 minutes. The example also contains
+the OHLCV data with minute-resolution for the past seven days which
+could be used to create indicators. Use this code as the backbone to
+create your own trading strategy.
+
+The lookback_date variable is used to ensure data for a coin existed on
+the lookback period specified.
+
+To run, execute the following two commands in a terminal (inside catalyst
+environment). The first one retrieves all the pricing data needed for this
+script to run (only needs to be run once), and the second one executes this
+script with the parameters specified in the run_algorithm() call at the end
+of the file:
+
+catalyst ingest-exchange -x bitfinex -f minute
+
+python simple_universe.py
+
+"""
+from datetime import timedelta
+
+import numpy as np
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+from catalyst.api import (symbols, )
+
+
+def initialize(context):
+    context.i = -1  # minute counter
+    context.exchange = context.exchanges.values()[0].name.lower()
+    context.base_currency = context.exchanges.values()[0].base_currency.lower()
+
+
+def handle_data(context, data):
+    context.i += 1
+    lookback_days = 7  # 7 days
+
+    # current date & time in each iteration formatted into a string
+    now = data.current_dt
+    date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = now - timedelta(days=lookback_days)
+    # keep only the date as a string, discard the time
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
+
+    one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
+    # update universe everyday at midnight
+    if not context.i % one_day_in_minutes:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+
+    # get lookback_days of history data: that is 'lookback' number of bins
+    lookback = one_day_in_minutes / minutes * lookback_days
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # Get 30 minute interval OHLCV data. This is the standard data
+            # required for candlestick or indicators/signals. Return Pandas
+            # DataFrames. 30T means 30-minute re-sampling of one minute data.
+            # Adjust it to your desired time interval as needed.
+            opened = fill(data.history(coin,
+                                       'open',
+                                       bar_count=lookback,
+                                       frequency='30T')).values
+            high = fill(data.history(coin,
+                                     'high',
+                                     bar_count=lookback,
+                                     frequency='30T')).values
+            low = fill(data.history(coin,
+                                    'low',
+                                    bar_count=lookback,
+                                    frequency='30T')).values
+            close = fill(data.history(coin,
+                                      'price',
+                                      bar_count=lookback,
+                                      frequency='30T')).values
+            volume = fill(data.history(coin,
+                                       'volume',
+                                       bar_count=lookback,
+                                       frequency='30T')).values
+
+            # close[-1] is the last value in the set, which is the equivalent
+            # to current price (as in the most recent value)
+            # displays the minute price for each pair every 30 minutes
+            print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
+                  '\tV:{v}'.format(
+                    now=now,
+                    pair=pair,
+                    o=opened[-1],
+                    h=high[-1],
+                    l=low[-1],
+                    c=close[-1],
+                    v=volume[-1],
+                  ))
+
+            # -------------------------------------------------------------
+            # --------------- Insert Your Strategy Here -------------------
+            # -------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    # get all the pairs for the given exchange
+    json_symbols = get_exchange_symbols(context.exchange)
+    # convert into a DataFrame for easier processing
+    df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
+    df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
+                                   axis=1)
+    df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
+                                     axis=1)
+
+    # Filter all the pairs to get only the ones for a given base_currency
+    df = df[df['base_currency'] == context.base_currency]
+
+    # Filter all pairs to ensure that pair existed in the current date range
+    df = df[df.start_date < lookback_date]
+    df = df[df.end_daily >= current_date]
+    context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
+
+    return df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace(
+                     [np.inf, -np.inf], np.nan
+                    ).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-11-10', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=100.0,  # amount of base_currency
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')

catalyst/examples/talib_simple.py

@@ -0,0 +1,366 @@
+# Run Command
+# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
+#   -f talib_simple.py -x poloniex
+#
+# Description
+# Simple TALib Example showing how to use various indicators
+# in you strategy. Based loosly on
+# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
+
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import talib as ta
+from logbook import Logger
+from matplotlib.dates import date2num
+from matplotlib.finance import candlestick_ohlc
+
+from catalyst import run_algorithm
+from catalyst.api import (
+    order,
+    order_target_percent,
+    symbol,
+)
+from catalyst.exchange.stats_utils import get_pretty_stats
+
+algo_namespace = 'talib_sample'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('Starting TALib Simple Example')
+
+    context.ASSET_NAME = 'BTC_USDT'
+    context.asset = symbol(context.ASSET_NAME)
+
+    context.ORDER_SIZE = 10
+    context.SLIPPAGE_ALLOWED = 0.05
+
+    context.swallow_errors = True
+    context.errors = []
+
+    # Bars to look at per iteration should be bigger than SMA_SLOW
+    context.BARS = 365
+    context.COUNT = 0
+
+    # Technical Analysis Settings
+    context.SMA_FAST = 50
+    context.SMA_SLOW = 100
+    context.RSI_PERIOD = 14
+    context.RSI_OVER_BOUGHT = 80
+    context.RSI_OVER_SOLD = 20
+    context.RSI_AVG_PERIOD = 15
+    context.MACD_FAST = 12
+    context.MACD_SLOW = 26
+    context.MACD_SIGNAL = 9
+    context.STOCH_K = 14
+    context.STOCH_D = 3
+    context.STOCH_OVER_BOUGHT = 80
+    context.STOCH_OVER_SOLD = 20
+
+    pass
+
+
+def _handle_data(context, data):
+    # Get price, open, high, low, close
+    prices = data.history(
+        context.asset,
+        bar_count=context.BARS,
+        fields=['price', 'open', 'high', 'low', 'close'],
+        frequency='1d')
+
+    # Create a analysis data frame
+    analysis = pd.DataFrame(index=prices.index)
+
+    # SMA FAST
+    analysis['sma_f'] = ta.SMA(prices.close.as_matrix(), context.SMA_FAST)
+    # SMA SLOW
+    analysis['sma_s'] = ta.SMA(prices.close.as_matrix(), context.SMA_SLOW)
+
+    # Relative Strength Index
+    analysis['rsi'] = ta.RSI(prices.close.as_matrix(), context.RSI_PERIOD)
+    # RSI SMA
+    analysis['sma_r'] = ta.SMA(analysis.rsi.as_matrix(),
+                               context.RSI_AVG_PERIOD)
+
+    # MACD, MACD Signal, MACD Histogram
+    analysis['macd'], analysis['macdSignal'], analysis['macdHist'] = ta.MACD(
+        prices.close.as_matrix(), fastperiod=context.MACD_FAST,
+        slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
+
+    # Stochastics %K %D
+    # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
+    # %D = 3-day SMA of %K
+    analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
+        prices.high.as_matrix(), prices.low.as_matrix(),
+        prices.close.as_matrix(), slowk_period=context.STOCH_K,
+        slowd_period=context.STOCH_D)
+
+    # SMA FAST over SLOW Crossover
+    analysis['sma_test'] = np.where(analysis.sma_f > analysis.sma_s, 1, 0)
+
+    # MACD over Signal Crossover
+    analysis['macd_test'] = np.where((analysis.macd > analysis.macdSignal), 1,
+                                     0)
+
+    # Stochastics OVER BOUGHT & Decreasing
+    analysis['stoch_over_bought'] = np.where(
+        (analysis.stoch_k > context.STOCH_OVER_BOUGHT) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # Stochastics OVER SOLD & Increasing
+    analysis['stoch_over_sold'] = np.where(
+        (analysis.stoch_k < context.STOCH_OVER_SOLD) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # RSI OVER BOUGHT & Decreasing
+    analysis['rsi_over_bought'] = np.where(
+        (analysis.rsi > context.RSI_OVER_BOUGHT) & (
+            analysis.rsi < analysis.rsi.shift(1)), 1, 0)
+
+    # RSI OVER SOLD & Increasing
+    analysis['rsi_over_sold'] = np.where(
+        (analysis.rsi < context.RSI_OVER_SOLD) & (
+            analysis.rsi > analysis.rsi.shift(1)), 1, 0)
+
+    # Save the prices and analysis to send to analyze
+    context.prices = prices
+    context.analysis = analysis
+    context.price = data.current(context.asset, 'price')
+
+    makeOrders(context, analysis)
+
+    # Log the values of this bar
+    logAnalysis(analysis)
+
+
+def handle_data(context, data):
+    log.info('handling bar {}'.format(data.current_dt))
+    try:
+        _handle_data(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    log.info('completed bar {}, total execution errors {}'.format(
+        data.current_dt,
+        len(context.errors)
+    ))
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context, results):
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename('talib_simple'))[0]
+    results.to_csv(filename + '.csv')
+
+    log.info('the daily stats:\n{}'.format(get_pretty_stats(results)))
+    chart(context, context.prices, context.analysis, results)
+    pass
+
+
+def makeOrders(context, analysis):
+    if context.asset in context.portfolio.positions:
+
+        # Current position
+        position = context.portfolio.positions[context.asset]
+
+        if (position == 0):
+            log.info('Position Zero')
+            return
+
+        # Cost Basis
+        cost_basis = position.cost_basis
+
+        log.info(
+            'Holdings: {amount} @ {cost_basis}'.format(
+                amount=position.amount,
+                cost_basis=cost_basis
+            )
+        )
+
+        # Sell when holding and got sell singnal
+        if isSell(context, analysis):
+            profit = (context.price * position.amount) - (
+                cost_basis * position.amount)
+            order_target_percent(
+                asset=context.asset,
+                target=0,
+                limit_price=context.price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            log.info(
+                'Sold {amount} @ {price} Profit: {profit}'.format(
+                    amount=position.amount,
+                    price=context.price,
+                    profit=profit
+                )
+            )
+        else:
+            log.info('no buy or sell opportunity found')
+    else:
+        # Buy when not holding and got buy signal
+        if isBuy(context, analysis):
+            order(
+                asset=context.asset,
+                amount=context.ORDER_SIZE,
+                limit_price=context.price * (1 + context.SLIPPAGE_ALLOWED)
+            )
+            log.info(
+                'Bought {amount} @ {price}'.format(
+                    amount=context.ORDER_SIZE,
+                    price=context.price
+                )
+            )
+
+
+def isBuy(context, analysis):
+    # Bullish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 1):
+        # Bullish MACD
+        if (getLast(analysis, 'macd_test') == 1):
+            return True
+
+    # # Bullish Stochastics
+    # if(getLast(analysis, 'stoch_over_sold') == 1):
+    #     return True
+
+    # # Bullish RSI
+    # if(getLast(analysis, 'rsi_over_sold') == 1):
+    #     return True
+
+    return False
+
+
+def isSell(context, analysis):
+    # Bearish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 0):
+        # Bearish MACD
+        if (getLast(analysis, 'macd_test') == 0):
+            return True
+
+    # # Bearish Stochastics
+    # if(getLast(analysis, 'stoch_over_bought') == 0):
+    #     return True
+
+    # # Bearish RSI
+    # if(getLast(analysis, 'rsi_over_bought') == 0):
+    #     return True
+
+    return False
+
+
+def chart(context, prices, analysis, results):
+    results.portfolio_value.plot()
+
+    # Data for matplotlib finance plot
+    dates = date2num(prices.index.to_pydatetime())
+
+    # Create the Open High Low Close Tuple
+    prices_ohlc = [tuple([dates[i],
+                          prices.open[i],
+                          prices.high[i],
+                          prices.low[i],
+                          prices.close[i]]) for i in range(len(dates))]
+
+    fig = plt.figure(figsize=(14, 18))
+
+    # Draw the candle sticks
+    ax1 = fig.add_subplot(411)
+    ax1.set_ylabel(context.ASSET_NAME, size=20)
+    candlestick_ohlc(ax1, prices_ohlc, width=0.4, colorup='g', colordown='r')
+
+    # Draw Moving Averages
+    analysis.sma_f.plot(ax=ax1, c='r')
+    analysis.sma_s.plot(ax=ax1, c='g')
+
+    # RSI
+    ax2 = fig.add_subplot(412)
+    ax2.set_ylabel('RSI', size=12)
+    analysis.rsi.plot(ax=ax2, c='g',
+                      label='Period: ' + str(context.RSI_PERIOD))
+    analysis.sma_r.plot(ax=ax2, c='r',
+                        label='MA: ' + str(context.RSI_AVG_PERIOD))
+    ax2.axhline(y=30, c='b')
+    ax2.axhline(y=50, c='black')
+    ax2.axhline(y=70, c='b')
+    ax2.set_ylim([0, 100])
+    handles, labels = ax2.get_legend_handles_labels()
+    ax2.legend(handles, labels)
+
+    # Draw MACD computed with Talib
+    ax3 = fig.add_subplot(413)
+    ax3.set_ylabel('MACD: ' + str(context.MACD_FAST) + ', ' + str(
+        context.MACD_SLOW) + ', ' + str(context.MACD_SIGNAL), size=12)
+    analysis.macd.plot(ax=ax3, color='b', label='Macd')
+    analysis.macdSignal.plot(ax=ax3, color='g', label='Signal')
+    analysis.macdHist.plot(ax=ax3, color='r', label='Hist')
+    ax3.axhline(0, lw=2, color='0')
+    handles, labels = ax3.get_legend_handles_labels()
+    ax3.legend(handles, labels)
+
+    # Stochastic plot
+    ax4 = fig.add_subplot(414)
+    ax4.set_ylabel('Stoch (k,d)', size=12)
+    analysis.stoch_k.plot(ax=ax4, label='stoch_k:' + str(context.STOCH_K),
+                          color='r')
+    analysis.stoch_d.plot(ax=ax4, label='stoch_d:' + str(context.STOCH_D),
+                          color='g')
+    handles, labels = ax4.get_legend_handles_labels()
+    ax4.legend(handles, labels)
+    ax4.axhline(y=20, c='b')
+    ax4.axhline(y=50, c='black')
+    ax4.axhline(y=80, c='b')
+
+    plt.show()
+
+
+def logAnalysis(analysis):
+    # Log only the last value in the array
+    log.info('- sma_f:          {:.2f}'.format(getLast(analysis, 'sma_f')))
+    log.info('- sma_s:          {:.2f}'.format(getLast(analysis, 'sma_s')))
+
+    log.info('- rsi:            {:.2f}'.format(getLast(analysis, 'rsi')))
+    log.info('- sma_r:          {:.2f}'.format(getLast(analysis, 'sma_r')))
+
+    log.info('- macd:           {:.2f}'.format(getLast(analysis, 'macd')))
+    log.info(
+        '- macdSignal:     {:.2f}'.format(getLast(analysis, 'macdSignal')))
+    log.info('- macdHist:       {:.2f}'.format(getLast(analysis, 'macdHist')))
+
+    log.info('- stoch_k:        {:.2f}'.format(getLast(analysis, 'stoch_k')))
+    log.info('- stoch_d:        {:.2f}'.format(getLast(analysis, 'stoch_d')))
+
+    log.info('- sma_test:       {}'.format(getLast(analysis, 'sma_test')))
+    log.info('- macd_test:      {}'.format(getLast(analysis, 'macd_test')))
+
+    log.info('- stoch_over_bought:   {}'.format(
+        getLast(analysis, 'stoch_over_bought')))
+    log.info(
+        '- stoch_over_sold:   {}'.format(getLast(analysis, 'stoch_over_sold')))
+
+    log.info('- rsi_over_bought:       {}'.format(
+        getLast(analysis, 'rsi_over_bought')))
+    log.info(
+        '- rsi_over_sold:       {}'.format(getLast(analysis, 'rsi_over_sold')))
+
+
+def getLast(arr, name):
+    return arr[name][arr[name].index[-1]]
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        base_currency='usdt',
+        start=pd.to_datetime('2016-11-1', utc=True),
+        end=pd.to_datetime('2017-11-10', utc=True),
+    )

catalyst/exchange/asset_finder_exchange.py

@@ -41,14 +41,15 @@ class AssetFinderExchange(object):
         SidsNotFound
             When a requested sid is not found and default_none=False.
         """
-        for sid in sids:
-            if sid in self._asset_cache:
-                log.debug('got asset from cache: {}'.format(sid))
-            else:
-                log.debug('fetching asset: {}'.format(sid))
+        # for sid in sids:
+        #     if sid in self._asset_cache:
+        #         log.debug('got asset from cache: {}'.format(sid))
+        #     else:
+        #         log.debug('fetching asset: {}'.format(sid))
         return list()
 
-    def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
         """Lookup an asset by symbol.
 
         Parameters
@@ -84,10 +85,15 @@ class AssetFinderExchange(object):
         """
         log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
 
-        key = ','.join([exchange.name, symbol])
+        if data_frequency is not None:
+            key = ','.join([exchange.name, symbol, data_frequency])
+
+        else:
+            key = ','.join([exchange.name, symbol])
+
         if key in self._asset_cache:
             return self._asset_cache[key]
         else:
-            asset = exchange.get_asset(symbol)
+            asset = exchange.get_asset(symbol, data_frequency)
             self._asset_cache[key] = asset
             return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -14,6 +14,7 @@ import six
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
@@ -29,16 +30,17 @@ from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
+from catalyst.utils.deprecate import deprecated
+
 requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 
+@deprecated
 class Bitfinex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
         self.url = BITFINEX_URL
@@ -46,8 +48,13 @@ class Bitfinex(Exchange):
         self.secret = secret.encode('UTF-8')
         self.name = 'bitfinex'
         self.color = 'green'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -61,7 +68,7 @@ class Bitfinex(Exchange):
         self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def _request(self, operation, data, version='v1'):
         payload_object = {
@@ -167,7 +174,8 @@ class Bitfinex(Exchange):
 
         executed_price = float(order_status['avg_execution_price'])
 
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
+        # TODO: bitfinex does not specify comission.
+        # I could calculate it but not sure if it's worth it.
         commission = None
 
         date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
@@ -594,17 +602,17 @@ class Bitfinex(Exchange):
             else:
                 try:
                     start_date = cached_symbols[symbol]['start_date']
-                except KeyError as e:
+                except KeyError:
                     start_date = time.strftime('%Y-%m-%d')
 
             try:
                 end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[symbol] = dict(
@@ -655,15 +663,16 @@ class Bitfinex(Exchange):
 
         """
             Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than now: time.time()
+            the startmonth we got above. Avoid end dates greater than
+            now: time.time()
         """
-        url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
+        url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
+               '&end={end}').format(
             url=self.url,
             symbol=symbol_v2,
             start=startmonth - 3600 * 24 * 31 * 1000,
             end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000))
-        )
+                    int(time.time() * 1000)))
 
         try:
             self.ask_request()

catalyst/exchange/bittrex/bittrex.py

@@ -19,12 +19,14 @@ from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
 
 # TODO: consider using this: https://github.com/mondeja/bittrex_v2
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 
 
+@deprecated
 class Bittrex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
         self.api = Bittrex_api(key=key, secret=secret)
@@ -46,7 +48,10 @@ class Bittrex(Exchange):
         self.assets = dict()
         self.load_assets()
 
-        self.bundle = ExchangeBundle(self)
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
+        self.bundle = ExchangeBundle(self.name)
 
     @property
     def account(self):
@@ -259,11 +264,10 @@ class Bittrex(Exchange):
             end = int(time.mktime(end_dt.timetuple()))
             url = '{url}/pub/market/GetTicks?marketName={symbol}' \
                   '&tickInterval={frequency}&_={end}'.format(
-                url=URL2,
-                symbol=self.get_symbol(asset),
-                frequency=frequency,
-                end=end
-            )
+                    url=URL2,
+                    symbol=self.get_symbol(asset),
+                    frequency=frequency,
+                    end=end, )
 
             try:
                 data = json.loads(urllib.request.urlopen(url).read().decode())
@@ -356,12 +360,12 @@ class Bittrex(Exchange):
 
             try:
                 end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[exchange_symbol] = dict(

catalyst/exchange/bittrex/extensions-example.py

@@ -4,4 +4,4 @@ from catalyst.exchange.exchange_bundle import exchange_bundle
 symbols = (
     'neo_btc',
 )
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
+register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/bundle_utils.py

@@ -78,9 +78,8 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
     if not os.path.isdir(path):
         url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
               'exchange-{exchange}/{name}.tar.gz'.format(
-            exchange=exchange_name,
-            name=name
-        )
+                exchange=exchange_name,
+                name=name)
 
         bytes = download_without_progress(url)
         with tarfile.open('r', fileobj=bytes) as tar:
@@ -191,8 +190,10 @@ def get_period_label(dt, data_frequency):
     str
 
     """
-    return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
-        else '{}'.format(dt.year)
+    if data_frequency == 'minute':
+        return '{}-{:02d}'.format(dt.year, dt.month)
+    else:
+        return '{}'.format(dt.year)
 
 
 def get_month_start_end(dt, first_day=None, last_day=None):
@@ -313,7 +314,45 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
             if np.isnan(close):
                 has_data = False
 
-        except Exception as e:
+        except Exception:
             has_data = False
 
     return has_data
+
+
+def get_assets(exchange, include_symbols, exclude_symbols):
+    """
+    Get assets from an exchange, including or excluding the specified
+    symbols.
+
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
+
+        return exchange.get_assets(include_symbols_list)
+
+    else:
+        all_assets = exchange.get_assets()
+
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -0,0 +1,638 @@
+import re
+from collections import defaultdict
+
+import ccxt
+import pandas as pd
+import six
+from ccxt import ExchangeNotAvailable, InvalidOrder
+from logbook import Logger
+from six import string_types
+
+from catalyst.algorithm import MarketOrder
+from catalyst.assets._assets import TradingPair
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
+    ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
+    ExchangeNotFoundError, CreateOrderError
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.exchange_utils import mixin_market_params, \
+    from_ms_timestamp, get_epoch
+from catalyst.finance.order import Order, ORDER_STATUS
+
+log = Logger('CCXT', level=LOG_LEVEL)
+
+SUPPORTED_EXCHANGES = dict(
+    binance=ccxt.binance,
+    bitfinex=ccxt.bitfinex,
+    bittrex=ccxt.bittrex,
+    poloniex=ccxt.poloniex,
+    bitmex=ccxt.bitmex,
+    gdax=ccxt.gdax,
+)
+
+
+class CCXT(Exchange):
+    def __init__(self, exchange_name, key, secret, base_currency):
+        log.debug(
+            'finding {} in CCXT exchanges:\n{}'.format(
+                exchange_name, ccxt.exchanges
+            )
+        )
+        try:
+            # Making instantiation as explicit as possible for code tracking.
+            if exchange_name in SUPPORTED_EXCHANGES:
+                exchange_attr = SUPPORTED_EXCHANGES[exchange_name]
+
+            else:
+                exchange_attr = getattr(ccxt, exchange_name)
+
+            self.api = exchange_attr({
+                'apiKey': key,
+                'secret': secret,
+            })
+
+        except Exception:
+            raise ExchangeNotFoundError(exchange_name=exchange_name)
+
+        self._symbol_maps = [None, None]
+
+        try:
+            markets_symbols = self.api.load_markets()
+            log.debug('the markets:\n{}'.format(markets_symbols))
+
+        except ExchangeNotAvailable as e:
+            raise ExchangeRequestError(error=e)
+
+        self.name = exchange_name
+
+        self.markets = self.api.fetch_markets()
+        self.load_assets()
+
+        self.base_currency = base_currency
+        self.transactions = defaultdict(list)
+
+        self.num_candles_limit = 2000
+        self.max_requests_per_minute = 60
+        self.request_cpt = dict()
+
+        self.bundle = ExchangeBundle(self.name)
+
+    def account(self):
+        return None
+
+    def time_skew(self):
+        return None
+
+    def get_market(self, symbol):
+        """
+        The CCXT market.
+
+        Parameters
+        ----------
+        symbol:
+            The CCXT symbol.
+
+        Returns
+        -------
+        dict[str, Object]
+
+        """
+        s = self.get_symbol(symbol)
+        market = next(
+            (market for market in self.markets if market['symbol'] == s),
+            None,
+        )
+        return market
+
+    def get_symbol(self, asset_or_symbol):
+        """
+        The CCXT symbol.
+
+        Parameters
+        ----------
+        asset_or_symbol
+
+        Returns
+        -------
+
+        """
+        symbol = asset_or_symbol if isinstance(
+            asset_or_symbol, string_types
+        ) else asset_or_symbol.symbol
+
+        parts = symbol.split('_')
+        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+
+    def get_catalyst_symbol(self, market_or_symbol):
+        """
+        The Catalyst symbol.
+
+        Parameters
+        ----------
+        market_or_symbol
+
+        Returns
+        -------
+
+        """
+        if isinstance(market_or_symbol, string_types):
+            parts = market_or_symbol.split('/')
+            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+        else:
+            return '{}_{}'.format(
+                market_or_symbol['base'].lower(),
+                market_or_symbol['quote'].lower(),
+            )
+
+    def get_timeframe(self, freq):
+        """
+        The CCXT timeframe from the Catalyst frequency.
+
+        Parameters
+        ----------
+        freq: str
+            The Catalyst frequency (Pandas convention)
+
+        Returns
+        -------
+        str
+
+        """
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) \
+                if freq_match.group(1) else 1
+
+            unit = freq_match.group(2)
+
+        else:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+        if unit.lower() == 'd':
+            timeframe = '{}d'.format(candle_size)
+
+        elif unit.lower() == 'm' or unit == 'T':
+            timeframe = '{}m'.format(candle_size)
+
+        elif unit.lower() == 'h' or unit == 'T':
+            timeframe = '{}h'.format(candle_size)
+
+        return timeframe
+
+    def get_candles(self, freq, assets, bar_count=None, start_dt=None,
+                    end_dt=None):
+        is_single = (isinstance(assets, TradingPair))
+        if is_single:
+            assets = [assets]
+
+        symbols = self.get_symbols(assets)
+        timeframe = self.get_timeframe(freq)
+
+        ms = None
+        if start_dt is not None:
+            delta = start_dt - get_epoch()
+            ms = int(delta.total_seconds()) * 1000
+
+        candles = dict()
+        for asset in assets:
+            try:
+                ohlcvs = self.api.fetch_ohlcv(
+                    symbol=symbols[0],
+                    timeframe=timeframe,
+                    since=ms,
+                    limit=bar_count,
+                    params={}
+                )
+
+                candles[asset] = []
+                for ohlcv in ohlcvs:
+                    candles[asset].append(dict(
+                        last_traded=pd.to_datetime(
+                            ohlcv[0], unit='ms', utc=True
+                        ),
+                        open=ohlcv[1],
+                        high=ohlcv[2],
+                        low=ohlcv[3],
+                        close=ohlcv[4],
+                        volume=ohlcv[5]
+                    ))
+
+            except Exception as e:
+                raise ExchangeRequestError(error=e)
+
+        if is_single:
+            return six.next(six.itervalues(candles))
+
+        else:
+            return candles
+
+    def _fetch_symbol_map(self, is_local):
+        try:
+            return self.fetch_symbol_map(is_local)
+        except ExchangeSymbolsNotFound:
+            return None
+
+    def get_asset_defs(self, market):
+        """
+        The local and Catalyst definitions of the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dicts.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        asset_defs = []
+
+        for is_local in (False, True):
+            asset_def = self.get_asset_def(market, is_local)
+            asset_defs.append((asset_def, is_local))
+
+        return asset_defs
+
+    def get_asset_def(self, market, is_local=False):
+        """
+        The asset definition (in symbols.json files) corresponding
+        to the the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dict.
+        is_local
+            Whether to search in local or Catalyst asset definitions.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        exchange_symbol = market['id']
+
+        symbol_map = self._fetch_symbol_map(is_local)
+        if symbol_map is not None:
+            assets_lower = {k.lower(): v for k, v in symbol_map.items()}
+            key = exchange_symbol.lower()
+
+            asset = assets_lower[key] if key in assets_lower else None
+            if asset is not None:
+                return asset
+
+            else:
+                return None
+
+        else:
+            return None
+
+    def create_trading_pair(self, market, asset_def=None, is_local=False):
+        """
+        Creating a TradingPair from market and asset data.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+        asset_def: dict[str, Object]
+        is_local: bool
+
+        Returns
+        -------
+
+        """
+        data_source = 'local' if is_local else 'catalyst'
+        params = dict(
+            exchange=self.name,
+            data_source=data_source,
+            exchange_symbol=market['id'],
+        )
+        mixin_market_params(self.name, params, market)
+
+        if asset_def is not None:
+            params['symbol'] = asset_def['symbol']
+
+            params['start_date'] = asset_def['start_date'] \
+                if 'start_date' in asset_def else None
+
+            params['end_date'] = asset_def['end_date'] \
+                if 'end_date' in asset_def else None
+
+            params['leverage'] = asset_def['leverage'] \
+                if 'leverage' in asset_def else 1.0
+
+            params['asset_name'] = asset_def['asset_name'] \
+                if 'asset_name' in asset_def else None
+
+            params['end_daily'] = asset_def['end_daily'] \
+                if 'end_daily' in asset_def \
+                   and asset_def['end_daily'] != 'N/A' else None
+
+            params['end_minute'] = asset_def['end_minute'] \
+                if 'end_minute' in asset_def \
+                   and asset_def['end_minute'] != 'N/A' else None
+
+        else:
+            params['symbol'] = self.get_catalyst_symbol(market)
+            # TODO: add as an optional column
+            params['leverage'] = 1.0
+
+        return TradingPair(**params)
+
+    def load_assets(self):
+        self.assets = []
+
+        for market in self.markets:
+            asset_defs = self.get_asset_defs(market)
+
+            asset = None
+            for asset_def in asset_defs:
+                if asset_def[0] is not None or not asset_defs[1]:
+                    try:
+                        asset = self.create_trading_pair(
+                            market=market,
+                            asset_def=asset_def[0],
+                            is_local=asset_def[1]
+                        )
+                        self.assets.append(asset)
+
+                    except TypeError as e:
+                        log.warn('unable to add asset: {}'.format(e))
+
+            if asset is None:
+                asset = self.create_trading_pair(market=market)
+                self.assets.append(asset)
+
+    def get_balances(self):
+        try:
+            log.debug('retrieving wallets balances')
+            balances = self.api.fetch_balance()
+
+            balances_lower = dict()
+            for key in balances:
+                balances_lower[key.lower()] = balances[key]
+
+        except Exception as e:
+            log.debug('error retrieving balances: {}', e)
+            raise ExchangeRequestError(error=e)
+
+        return balances_lower
+
+    def _create_order(self, order_status):
+        """
+        Create a Catalyst order object from a CCXT order dictionary
+
+        Parameters
+        ----------
+        order_status: dict[str, Object]
+            The order dict from the CCXT api.
+
+        Returns
+        -------
+        Order
+            The Catalyst order object
+
+        """
+        if order_status['status'] == 'canceled':
+            status = ORDER_STATUS.CANCELLED
+
+        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
+            log.debug('found executed order {}'.format(order_status))
+            status = ORDER_STATUS.FILLED
+
+        elif order_status['status'] == 'open':
+            status = ORDER_STATUS.OPEN
+
+        else:
+            raise ValueError('invalid state for order')
+
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if order_status['side'] == 'sell':
+            amount = -amount
+            filled = -filled
+
+        price = order_status['price']
+        order_type = order_status['type']
+
+        limit_price = price if order_type == 'limit' else None
+        stop_price = None  # TODO: add support
+
+        executed_price = order_status['cost'] / order_status['amount']
+        commission = order_status['fee']
+        date = from_ms_timestamp(order_status['timestamp'])
+
+        # order_id = str(order_status['info']['clientOrderId'])
+        order_id = order_status['id']
+
+        # TODO: this won't work, redo the packages with a different key.
+        symbol = order_status['info']['symbol'] \
+            if 'symbol' in order_status['info'] \
+            else order_status['info']['Exchange']
+
+        order = Order(
+            dt=date,
+            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            amount=amount,
+            stop=stop_price,
+            limit=limit_price,
+            filled=filled,
+            id=order_id,
+            commission=commission
+        )
+        order.status = status
+
+        return order, executed_price
+
+    def create_order(self, asset, amount, is_buy, style):
+        symbol = self.get_symbol(asset)
+
+        if isinstance(style, ExchangeLimitOrder):
+            price = style.get_limit_price(is_buy)
+            order_type = 'limit'
+
+        elif isinstance(style, MarketOrder):
+            price = None
+            order_type = 'market'
+
+        else:
+            raise InvalidOrderStyle(
+                exchange=self.name,
+                style=style.__class__.__name__
+            )
+
+        side = 'buy' if amount > 0 else 'sell'
+
+        if hasattr(self.api, 'amount_to_lots'):
+            adj_amount = self.api.amount_to_lots(
+                symbol=symbol,
+                amount=abs(amount),
+            )
+            if adj_amount != abs(amount):
+                log.info(
+                    'adjusted order amount {} to {} based on lot size'.format(
+                        abs(amount), adj_amount,
+                    )
+                )
+        else:
+            adj_amount = abs(amount)
+
+        try:
+            result = self.api.create_order(
+                symbol=symbol,
+                type=order_type,
+                side=side,
+                amount=adj_amount,
+                price=price
+            )
+        except ExchangeNotAvailable as e:
+            log.debug('unable to create order: {}'.format(e))
+            raise ExchangeRequestError(error=e)
+
+        except InvalidOrder as e:
+            log.warn('the exchange rejected the order: {}'.format(e))
+            raise CreateOrderError(exchange=self.name, error=e)
+
+        if 'info' not in result:
+            raise ValueError('cannot use order without info attribute')
+
+        final_amount = adj_amount if side == 'buy' else -adj_amount
+        order_id = result['id']
+        order = Order(
+            dt=pd.Timestamp.utcnow(),
+            asset=asset,
+            amount=final_amount,
+            stop=style.get_stop_price(is_buy),
+            limit=style.get_limit_price(is_buy),
+            id=order_id
+        )
+        return order
+
+    def get_open_orders(self, asset):
+        try:
+            symbol = self.get_symbol(asset)
+            result = self.api.fetch_open_orders(
+                symbol=symbol,
+                since=None,
+                limit=None,
+                params=dict()
+            )
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        orders = []
+        for order_status in result:
+            order, executed_price = self._create_order(order_status)
+            if asset is None or asset == order.sid:
+                orders.append(order)
+
+        return orders
+
+    def get_order(self, order_id, asset_or_symbol=None):
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, the request might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            order_status = self.api.fetch_order(id=order_id, symbol=symbol)
+            order, executed_price = self._create_order(order_status)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        return order, executed_price
+
+    def cancel_order(self, order_param, asset_or_symbol=None):
+        order_id = order_param.id \
+            if isinstance(order_param, Order) else order_param
+
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, cancelling order might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            self.api.cancel_order(id=order_id, symbol=symbol)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+    def tickers(self, assets):
+        """
+        Retrieve current tick data for the given assets
+
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[dict[str, float]
+
+        """
+        tickers = dict()
+        for asset in assets:
+            try:
+                ccxt_symbol = self.get_symbol(asset)
+                ticker = self.api.fetch_ticker(ccxt_symbol)
+
+                ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
+
+                if 'last_price' not in ticker:
+                    # TODO: any more exceptions?
+                    ticker['last_price'] = ticker['last']
+
+                # Using the volume represented in the base currency
+                ticker['volume'] = ticker['baseVolume'] \
+                    if 'baseVolume' in ticker else 0
+
+                tickers[asset] = ticker
+
+            except ExchangeNotAvailable as e:
+                log.warn(
+                    'unable to fetch ticker: {} {}'.format(
+                        self.name, asset.symbol
+                    )
+                )
+                raise ExchangeRequestError(error=e)
+
+        return tickers
+
+    def get_account(self):
+        return None
+
+    def get_orderbook(self, asset, order_type='all', limit=None):
+        ccxt_symbol = self.get_symbol(asset)
+
+        params = dict()
+        if limit is not None:
+            params['depth'] = limit
+
+        order_book = self.api.fetch_order_book(ccxt_symbol, params)
+
+        order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
+        result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
+        for index, order_type in enumerate(order_types):
+            if limit is not None and index > limit - 1:
+                break
+
+            result[order_type] = []
+            for entry in order_book[order_type]:
+                result[order_type].append(dict(
+                    rate=float(entry[0]),
+                    quantity=float(entry[1])
+                ))
+
+        return result

catalyst/exchange/exchange.py

@@ -5,7 +5,6 @@ from time import sleep
 
 import numpy as np
 import pandas as pd
-from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
@@ -14,17 +13,11 @@ from catalyst.exchange.bundle_utils import get_start_dt, \
     get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange
-from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
-    ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
+    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError
 from catalyst.exchange.exchange_utils import get_exchange_symbols, \
     get_frequency, resample_history_df
-from catalyst.finance.order import ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.utils.deprecate import deprecated
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -34,8 +27,8 @@ class Exchange:
 
     def __init__(self):
         self.name = None
-        self.assets = {}
-        self._portfolio = None
+        self.assets = []
+        self._symbol_maps = [None, None]
         self.minute_writer = None
         self.minute_reader = None
         self.base_currency = None
@@ -43,28 +36,7 @@ class Exchange:
         self.num_candles_limit = None
         self.max_requests_per_minute = None
         self.request_cpt = None
-        self.bundle = ExchangeBundle(self)
-
-    @property
-    def positions(self):
-        return self.portfolio.positions
-
-    @property
-    def portfolio(self):
-        """
-        The exchange portfolio
-
-        Returns
-        -------
-        ExchangePortfolio
-        """
-        if self._portfolio is None:
-            self._portfolio = ExchangePortfolio(
-                start_date=pd.Timestamp.utcnow()
-            )
-            self.synchronize_portfolio()
-
-        return self._portfolio
+        self.bundle = ExchangeBundle(self.name)
 
     @abstractproperty
     def account(self):
@@ -132,7 +104,7 @@ class Exchange:
 
     def get_symbol(self, asset):
         """
-        The the exchange specific symbol of the specified market.
+        The exchange specific symbol of the specified market.
 
         Parameters
         ----------
@@ -145,9 +117,9 @@ class Exchange:
         """
         symbol = None
 
-        for key in self.assets:
-            if not symbol and self.assets[key].symbol == asset.symbol:
-                symbol = key
+        for a in self.assets:
+            if not symbol and a.symbol == asset.symbol:
+                symbol = a.symbol
 
         if not symbol:
             raise ValueError('Currency %s not supported by exchange %s' %
@@ -174,54 +146,112 @@ class Exchange:
 
         return symbols
 
-    def get_assets(self, symbols=None):
+    def get_assets(self, symbols=None, data_frequency=None,
+                   is_exchange_symbol=False,
+                   is_local=None):
         """
         The list of markets for the specified symbols.
 
         Parameters
         ----------
         symbols: list[str]
+        data_frequency: str
+        is_exchange_symbol: bool
+        is_local: bool
 
         Returns
         -------
         list[TradingPair]
+            A list of asset objects.
+
+        Notes
+        -----
+        See get_asset for details of each parameter.
 
         """
+        if symbols is None:
+            # Make a distinct list of all symbols
+            symbols = list(set([asset.symbol for asset in self.assets]))
+            is_exchange_symbol = False
+
         assets = []
-
-        if symbols is not None:
-            for symbol in symbols:
-                asset = self.get_asset(symbol)
+        for symbol in symbols:
+            try:
+                asset = self.get_asset(
+                    symbol, data_frequency, is_exchange_symbol, is_local
+                )
                 assets.append(asset)
-        else:
-            for key in self.assets:
-                assets.append(self.assets[key])
 
+            except SymbolNotFoundOnExchange:
+                log.debug(
+                    'skipping non-existent market {} {}'.format(
+                        self.name, symbol
+                    )
+                )
         return assets
 
-    def get_asset(self, symbol):
+    def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
+                  is_local=None):
         """
         The market for the specified symbol.
 
         Parameters
         ----------
         symbol: str
+            The Catalyst or exchange symbol.
+
+        data_frequency: str
+            Check for asset corresponding to the specified data_frequency.
+            The same asset might exist in the Catalyst repository or
+            locally (following a CSV ingestion). Filtering by
+            data_frequency picks the right asset.
+
+        is_exchange_symbol: bool
+            Whether the symbol uses the Catalyst or exchange convention.
+
+        is_local: bool
+            For the local or Catalyst asset.
 
         Returns
         -------
         TradingPair
+            The asset object.
 
         """
         asset = None
 
-        for key in self.assets:
-            if not asset and self.assets[key].symbol.lower() == symbol.lower():
-                asset = self.assets[key]
+        log.debug(
+            'searching assets for: {} {}'.format(
+                self.name, symbol
+            )
+        )
+        for a in self.assets:
+            if asset is not None:
+                break
 
-        if not asset:
-            supported_symbols = [
-                pair.symbol for pair in list(self.assets.values())
-            ]
+            if is_local is not None:
+                data_source = 'local' if is_local else 'catalyst'
+                applies = (a.data_source == data_source)
+
+            elif data_frequency is not None:
+                applies = (
+                        (
+                                data_frequency == 'minute' and a.end_minute is not None)
+                        or (
+                                data_frequency == 'daily' and a.end_daily is not None)
+                )
+
+            else:
+                applies = True
+
+            # The symbol provided may use the Catalyst or the exchange
+            # convention
+            key = a.exchange_symbol if is_exchange_symbol else a.symbol
+            if not asset and key.lower() == symbol.lower() and applies:
+                asset = a
+
+        if asset is None:
+            supported_symbols = sorted([a.symbol for a in self.assets])
 
             raise SymbolNotFoundOnExchange(
                 symbol=symbol,
@@ -229,12 +259,21 @@ class Exchange:
                 supported_symbols=supported_symbols
             )
 
+        log.debug('found asset: {}'.format(asset))
         return asset
 
-    def fetch_symbol_map(self):
-        return get_exchange_symbols(self.name)
+    def fetch_symbol_map(self, is_local=False):
+        index = 1 if is_local else 0
+        if self._symbol_maps[index] is not None:
+            return self._symbol_maps[index]
 
-    def load_assets(self):
+        else:
+            symbol_map = get_exchange_symbols(self.name, is_local)
+            self._symbol_maps[index] = symbol_map
+            return symbol_map
+
+    @abstractmethod
+    def load_assets(self, is_local=False):
         """
         Populate the 'assets' attribute with a dictionary of Assets.
         The key of the resulting dictionary is the exchange specific
@@ -247,109 +286,11 @@ class Exchange:
         universal symbol. This simple approach avoids maintaining a mapping
         of sids.
 
-        This method can be overridden if an exchange offers equivalent data
+        This method can be omerridden if an exchange offers equivalent data
         via its api.
 
         """
-        symbol_map = self.fetch_symbol_map()
-        for exchange_symbol in symbol_map:
-            asset = symbol_map[exchange_symbol]
-
-            if 'start_date' in asset:
-                start_date = pd.to_datetime(asset['start_date'], utc=True)
-            else:
-                start_date = None
-
-            if 'end_date' in asset:
-                end_date = pd.to_datetime(asset['end_date'], utc=True)
-            else:
-                end_date = None
-
-            if 'leverage' in asset:
-                leverage = asset['leverage']
-            else:
-                leverage = 1.0
-
-            if 'asset_name' in asset:
-                asset_name = asset['asset_name']
-            else:
-                asset_name = None
-
-            if 'min_trade_size' in asset:
-                min_trade_size = asset['min_trade_size']
-            else:
-                min_trade_size = 0.0000001
-
-            if 'end_daily' in asset and asset['end_daily'] != 'N/A':
-                end_daily = pd.to_datetime(asset['end_daily'], utc=True)
-            else:
-                end_daily = None
-
-            if 'end_minute' in asset and asset['end_minute'] != 'N/A':
-                end_minute = pd.to_datetime(asset['end_minute'], utc=True)
-            else:
-                end_minute = None
-
-            trading_pair = TradingPair(
-                symbol=asset['symbol'],
-                exchange=self.name,
-                start_date=start_date,
-                end_date=end_date,
-                leverage=leverage,
-                asset_name=asset_name,
-                min_trade_size=min_trade_size,
-                end_daily=end_daily,
-                end_minute=end_minute,
-                exchange_symbol=exchange_symbol
-            )
-
-            self.assets[exchange_symbol] = trading_pair
-
-    def check_open_orders(self):
-        """
-        Loop through the list of open orders in the Portfolio object.
-        For each executed order found, create a transaction and apply to the
-        Portfolio.
-
-        Returns
-        -------
-        list[Transaction]
-
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-                log.debug('found open order: {}'.format(order_id))
-
-                order, executed_price = self.get_order(order_id)
-                log.debug('got updated order {} {}'.format(
-                    order, executed_price))
-
-                if order.status == ORDER_STATUS.FILLED:
-                    transaction = Transaction(
-                        asset=order.asset,
-                        amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
-                        price=executed_price,
-                        order_id=order.id,
-                        commission=order.commission
-                    )
-                    transactions.append(transaction)
-
-                    self.portfolio.execute_order(order, transaction)
-
-                elif order.status == ORDER_STATUS.CANCELLED:
-                    self.portfolio.remove_order(order)
-
-                else:
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta
-                        )
-                    )
-        return transactions
+        pass
 
     def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
         """
@@ -386,12 +327,15 @@ class Exchange:
         if field not in BASE_FIELDS:
             raise KeyError('Invalid column: {}'.format(field))
 
-        values = []
-        for asset in assets:
-            value = self.get_single_spot_value(asset, field, data_frequency)
-            values.append(value)
+        tickers = self.tickers(assets)
+        if field == 'close' or field == 'price':
+            return [tickers[asset]['last'] for asset in tickers]
 
-        return values
+        elif field == 'volume':
+            return [tickers[asset]['volume'] for asset in tickers]
+
+        else:
+            raise NoValueForField(field=field)
 
     def get_single_spot_value(self, asset, field, data_frequency):
         """
@@ -465,18 +409,17 @@ class Exchange:
             method='ffill',
             fill_value=previous_value,
         )
-
+        series.sort_index(inplace=True)
         return series
 
-    @deprecated
-    def get_history_window_direct(self,
-                                  assets,
-                                  end_dt,
-                                  bar_count,
-                                  frequency,
-                                  field,
-                                  data_frequency=None,
-                                  ffill=True):
+    def get_history_window(self,
+                           assets,
+                           end_dt,
+                           bar_count,
+                           frequency,
+                           field,
+                           data_frequency=None,
+                           is_current=False):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -503,10 +446,15 @@ class Exchange:
             The frequency of the data to query; i.e. whether the data is
             'daily' or 'minute' bars.
 
-        # TODO: fill how?
-        ffill: boolean
-            Forward-fill missing values. Only has effect if field
-            is 'price'.
+        is_current: bool
+            Skip date filters when current data is requested (last few bars
+            until now).
+
+        Notes
+        -----
+        Catalysts requires an end data with bar count both CCXT wants a
+        start data with bar count. Since we have to make calculations here,
+        we ensure that the last candle match the end_dt parameter.
 
         Returns
         -------
@@ -514,35 +462,58 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
-        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        freq, candle_size, unit, data_frequency = get_frequency(
+            frequency, data_frequency
+        )
+        adj_bar_count = candle_size * bar_count
+
+        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
 
         # The get_history method supports multiple asset
         candles = self.get_candles(
-            data_frequency=frequency,
+            freq=freq,
             assets=assets,
             bar_count=bar_count,
-            start_dt=start_dt,
-            end_dt=end_dt
-        )
-        candle_series = self.get_series_from_candles(
-            candles=candles,
-            start_dt=start_dt,
-            end_dt=end_dt,
-            data_frequency=frequency,
-            field=field,
+            start_dt=start_dt if not is_current else None,
+            end_dt=end_dt if not is_current else None,
         )
 
-        df = pd.DataFrame(candle_series)
+        series = dict()
+        for asset in candles:
+            asset_series = self.get_series_from_candles(
+                candles=candles[asset],
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=frequency,
+                field=field,
+            )
+            if end_dt is not None:
+                delta = get_delta(candle_size, data_frequency)
+                adj_end_dt = end_dt - delta
+                last_traded = asset_series.index[-1]
+
+                if last_traded < adj_end_dt:
+                    raise LastCandleTooEarlyError(
+                        last_traded=last_traded,
+                        end_dt=adj_end_dt,
+                        exchange=self.name,
+                    )
+            series[asset] = asset_series
+
+        df = pd.DataFrame(series)
+        df.dropna(inplace=True)
+
         return df
 
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency=None,
-                           ffill=True):
+    def get_history_window_with_bundle(self,
+                                       assets,
+                                       end_dt,
+                                       bar_count,
+                                       frequency,
+                                       field,
+                                       data_frequency=None,
+                                       ffill=True,
+                                       force_auto_ingest=False):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -584,14 +555,17 @@ class Exchange:
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
+
         try:
             series = self.bundle.get_history_window_series_and_load(
                 assets=assets,
                 end_dt=end_dt,
                 bar_count=adj_bar_count,
                 field=field,
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                force_auto_ingest=force_auto_ingest
             )
+
         except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
             series = dict()
 
@@ -645,50 +619,48 @@ class Exchange:
 
         return df
 
-    def synchronize_portfolio(self):
+    def calculate_totals(self, check_cash=False, positions=None):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
         """
         log.debug('synchronizing portfolio with exchange {}'.format(self.name))
-        balances = self.get_balances()
 
-        base_position_available = balances[self.base_currency] \
-            if self.base_currency in balances else None
+        cash = None
+        if check_cash:
+            balances = self.get_balances()
 
-        if base_position_available is None:
-            raise BaseCurrencyNotFoundError(
-                base_currency=self.base_currency,
-                exchange=self.name.title()
-            )
+            cash = balances[self.base_currency]['free'] \
+                if self.base_currency in balances else None
 
-        portfolio = self._portfolio
-        portfolio.cash = base_position_available
-        log.debug('found base currency balance: {}'.format(portfolio.cash))
+            if cash is None:
+                raise BaseCurrencyNotFoundError(
+                    base_currency=self.base_currency,
+                    exchange=self.name
+                )
+            log.debug('found base currency balance: {}'.format(cash))
 
-        if portfolio.starting_cash is None:
-            portfolio.starting_cash = portfolio.cash
-
-        if portfolio.positions:
-            assets = list(portfolio.positions.keys())
+        positions_value = 0.0
+        if positions:
+            assets = set([position.asset for position in positions])
             tickers = self.tickers(assets)
+            log.debug('got tickers for positions: {}'.format(tickers))
 
-            portfolio.positions_value = 0.0
             for asset in tickers:
-                # TODO: convert if the position is not in the base currency
                 ticker = tickers[asset]
-                position = portfolio.positions[asset]
-                position.last_sale_price = ticker['last_price']
-                position.last_sale_date = ticker['timestamp']
+                positions = [p for p in positions if p.asset == asset]
 
-                portfolio.positions_value += \
-                    position.amount * position.last_sale_price
-                portfolio.portfolio_value = \
-                    portfolio.positions_value + portfolio.cash
+                for position in positions:
+                    position.last_sale_price = ticker['last_price']
+                    position.last_sale_date = ticker['last_traded']
 
-    def order(self, asset, amount, limit_price=None, stop_price=None,
-              style=None):
+                    positions_value += \
+                        position.amount * position.last_sale_price
+
+        return cash, positions_value
+
+    def order(self, asset, amount, style):
         """Place an order.
 
         Parameters
@@ -737,45 +709,30 @@ class Exchange:
             log.warn('skipping order amount of 0')
             return None
 
-        if asset.base_currency != self.base_currency.lower():
+        if self.base_currency is None:
+            raise ValueError('no base_currency defined for this exchange')
+
+        if asset.quote_currency != self.base_currency.lower():
             raise MismatchingBaseCurrencies(
-                base_currency=asset.base_currency,
+                base_currency=asset.quote_currency,
                 algo_currency=self.base_currency
             )
 
         is_buy = (amount > 0)
+        display_price = style.get_limit_price(is_buy)
 
-        if limit_price is not None and stop_price is not None:
-            style = ExchangeStopLimitOrder(limit_price, stop_price,
-                                           exchange=self.name)
-        elif limit_price is not None:
-            style = ExchangeLimitOrder(limit_price, exchange=self.name)
-
-        elif stop_price is not None:
-            style = ExchangeStopOrder(stop_price, exchange=self.name)
-
-        elif style is not None:
-            raise InvalidOrderStyle(exchange=self.name.title(),
-                                    style=style.__class__.__name__)
-        else:
-            raise ValueError('Incomplete order data.')
-
-        display_price = limit_price if limit_price is not None else stop_price
         log.debug(
-            'issuing {side} order of {amount} {symbol} for {type}: {price}'.format(
+            'issuing {side} order of {amount} {symbol} for {type}:'
+            ' {price}'.format(
                 side='buy' if is_buy else 'sell',
                 amount=amount,
                 symbol=asset.symbol,
                 type=style.__class__.__name__,
-                price='{}{}'.format(display_price, asset.base_currency)
+                price='{}{}'.format(display_price, asset.quote_currency)
             )
         )
-        order = self.create_order(asset, amount, is_buy, style)
-        if order:
-            self._portfolio.create_order(order)
-            return order.id
-        else:
-            return None
+
+        return self.create_order(asset, amount, is_buy, style)
 
     # The methods below must be implemented for each exchange.
     @abstractmethod
@@ -838,7 +795,7 @@ class Exchange:
         pass
 
     @abstractmethod
-    def get_order(self, order_id):
+    def get_order(self, order_id, symbol_or_asset=None):
         """Lookup an order based on the order id returned from one of the
         order functions.
 
@@ -846,6 +803,8 @@ class Exchange:
         ----------
         order_id : str
             The unique identifier for the order.
+        symbol_or_asset: str|TradingPair
+            The catalyst symbol, some exchanges need this
 
         Returns
         -------
@@ -857,13 +816,15 @@ class Exchange:
         pass
 
     @abstractmethod
-    def cancel_order(self, order_param):
+    def cancel_order(self, order_param, symbol_or_asset=None):
         """Cancel an open order.
 
         Parameters
         ----------
         order_param : str or Order
             The order_id or order object to cancel.
+        symbol_or_asset: str|TradingPair
+            The catalyst symbol, some exchanges need this
         """
         pass
 

catalyst/exchange/exchange_algorithm.py

@@ -13,7 +13,6 @@
 import pickle
 import signal
 import sys
-from collections import deque
 from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
@@ -21,34 +20,32 @@ from time import sleep
 
 import logbook
 import pandas as pd
-from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
-from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangePortfolioDataError,
-    ExchangeTransactionError,
-    OrphanOrderError)
-from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
-    ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
-    get_algo_folder, get_algo_df, \
-    save_algo_df
+    OrderTypeNotSupported, )
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.exchange_utils import (
+    save_algo_object,
+    get_algo_object,
+    get_algo_folder,
+    get_algo_df,
+    save_algo_df,
+    group_assets_by_exchange, )
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
+    stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
-from catalyst.utils.api_support import (
-    api_method,
-    disallowed_in_before_trading_start)
-from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
-    expect_types
+from catalyst.utils.api_support import api_method
+from catalyst.utils.input_validation import error_keywords, ensure_upper_case
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
 
@@ -63,9 +60,90 @@ class ExchangeAlgorithmExecutor(AlgorithmSimulator):
 class ExchangeTradingAlgorithmBase(TradingAlgorithm):
     def __init__(self, *args, **kwargs):
         self.exchanges = kwargs.pop('exchanges', None)
+        self.simulate_orders = kwargs.pop('simulate_orders', None)
 
         super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
 
+        self.current_day = None
+
+        if self.simulate_orders is None \
+                and self.sim_params.arena == 'backtest':
+            self.simulate_orders = True
+
+        self.blotter = ExchangeBlotter(
+            data_frequency=self.data_frequency,
+            # Default to NeverCancel in catalyst
+            cancel_policy=self.cancel_policy,
+            simulate_orders=self.simulate_orders,
+            exchanges=self.exchanges
+        )
+
+    @staticmethod
+    def __convert_order_params_for_blotter(limit_price, stop_price, style):
+        """
+        Helper method for converting deprecated limit_price and stop_price
+        arguments into ExecutionStyle instances.
+
+        This function assumes that either style == None or (limit_price,
+        stop_price) == (None, None).
+        """
+        if stop_price:
+            raise OrderTypeNotSupported(order_type='stop')
+
+        if style:
+            if limit_price is not None:
+                raise ValueError(
+                    'An order style and a limit price was included in the '
+                    'order. Please pick one to avoid any possible conflict.'
+                )
+
+            # Currently limiting order types or limit and market to
+            # be in-line with CXXT and many exchanges. We'll consider
+            # adding more order types in the future.
+            if not isinstance(style, ExchangeLimitOrder) or \
+                    not isinstance(style, MarketOrder):
+                raise OrderTypeNotSupported(
+                    order_type=style.__class__.__name__
+                )
+
+            return style
+
+        if limit_price:
+            return ExchangeLimitOrder(limit_price)
+        else:
+            return MarketOrder()
+
+    @api_method
+    def set_commission(self, maker=None, taker=None):
+        key = self.blotter.commission_models.keys()[0]
+        if maker is not None:
+            self.blotter.commission_models[key].maker = maker
+
+        if taker is not None:
+            self.blotter.commission_models[key].taker = taker
+
+    @api_method
+    def set_slippage(self, spread=None):
+        key = self.blotter.slippage_models.keys()[0]
+        if spread is not None:
+            self.blotter.slippage_models[key].spread = spread
+
+    def _calculate_order(self, asset, amount,
+                         limit_price=None, stop_price=None, style=None):
+        # Raises a ZiplineError if invalid parameters are detected.
+        self.validate_order_params(asset,
+                                   amount,
+                                   limit_price,
+                                   stop_price,
+                                   style)
+
+        # Convert deprecated limit_price and stop_price parameters to use
+        # ExecutionStyle objects.
+        style = self.__convert_order_params_for_blotter(limit_price,
+                                                        stop_price,
+                                                        style)
+        return amount, style
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies
@@ -114,9 +192,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         else:
             exchange = self.exchanges[exchange_name]
 
+        data_frequency = self.data_frequency \
+            if self.sim_params.arena == 'backtest' else None
         return self.asset_finder.lookup_symbol(
             symbol=symbol_str,
             exchange=exchange,
+            data_frequency=data_frequency,
             as_of_date=_lookup_date
         )
 
@@ -201,71 +282,55 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
         self.frame_stats = list()
-        self.blotter = ExchangeBlotter(
-            data_frequency=self.data_frequency,
-            # Default to NeverCancel in catalyst
-            cancel_policy=self.cancel_policy,
-        )
         log.info('initialized trading algorithm in backtest mode')
 
-    def _calculate_order(self, asset, amount,
-                         limit_price=None, stop_price=None, style=None):
-        # Raises a ZiplineError if invalid parameters are detected.
-        self.validate_order_params(asset,
-                                   amount,
-                                   limit_price,
-                                   stop_price,
-                                   style)
-
-        # Convert deprecated limit_price and stop_price parameters to use
-        # ExecutionStyle objects.
-        style = self.__convert_order_params_for_blotter(limit_price,
-                                                        stop_price,
-                                                        style)
-        return amount, style
-
-    @staticmethod
-    def __convert_order_params_for_blotter(limit_price, stop_price, style):
-        """
-        Helper method for converting deprecated limit_price and stop_price
-        arguments into ExecutionStyle instances.
-
-        This function assumes that either style == None or (limit_price,
-        stop_price) == (None, None).
-        """
-        if style:
-            assert (limit_price, stop_price) == (None, None)
-            return style
-        if limit_price and stop_price:
-            return ExchangeStopLimitOrder(limit_price, stop_price)
-        if limit_price:
-            return ExchangeLimitOrder(limit_price)
-        if stop_price:
-            return ExchangeStopOrder(stop_price)
+    def is_last_frame_of_day(self, data):
+        # TODO: adjust here to support more intervals
+        next_frame_dt = data.current_dt + timedelta(minutes=1)
+        if next_frame_dt.date() > data.current_dt.date():
+            return True
         else:
-            return MarketOrder()
+            return False
 
     def handle_data(self, data):
         super(ExchangeTradingAlgorithmBacktest, self).handle_data(data)
 
-        minute_stats = self.prepare_period_stats(
-            data.current_dt, data.current_dt + timedelta(minutes=1))
-        self.frame_stats.append(minute_stats)
+        if self.data_frequency == 'minute':
+            frame_stats = self.prepare_period_stats(
+                data.current_dt, data.current_dt + timedelta(minutes=1)
+            )
+            self.frame_stats.append(frame_stats)
 
-    def analyze(self, perf):
+        self.current_day = data.current_dt.floor('1D')
+
+    def _create_stats_df(self):
         stats = pd.DataFrame(self.frame_stats)
         stats.set_index('period_close', inplace=True, drop=False)
+        return stats
 
+    def analyze(self, perf):
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
         super(ExchangeTradingAlgorithmBacktest, self).analyze(stats)
 
+    def run(self, data=None, overwrite_sim_params=True):
+        perf = super(ExchangeTradingAlgorithmBacktest, self).run(
+            data, overwrite_sim_params
+        )
+        # Rebuilding the stats to support minute data
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
+        return stats
+
 
 class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
+        self.stats_output = kwargs.pop('stats_output', None)
 
         self._clock = None
-        self.minute_stats = deque(maxlen=60)
+        self.frame_stats = list()
 
         self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
 
@@ -283,7 +348,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.retry_order = 2
         self.retry_delay = 5
 
-        self.stats_minutes = 5
+        self.stats_minutes = 10
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
@@ -351,7 +416,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         # This method is taken from TradingAlgorithm.
         # The clock has been replaced to use RealtimeClock
-        # TODO: should we apply a time skew? not sure to understand the utility.
+        # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
         if self.live_graph:
@@ -389,47 +454,83 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         return self.trading_client.transform()
 
     def updated_portfolio(self):
-        """
-        We skip the entire performance tracker business and update the
-        portfolio directly.
-
-        Returns
-        -------
-        ExchangePortfolio
-
-        """
-        # TODO: build cumulative portfolio
         return self.perf_tracker.get_portfolio(False)
 
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def _synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self, attempt_index=0):
+        """
+        Synchronizes the portfolio tracked by the algorithm to refresh
+        its current value.
+
+        This includes updating the last_sale_price of all tracked
+        positions, returning the available cash, and raising error
+        if the data goes out of sync.
+
+        Parameters
+        ----------
+        attempt_index: int
+
+        Returns
+        -------
+        float
+            The amount of base currency available for trading.
+
+        float
+            The total value of all tracked positions.
+
+        """
+        tracker = self.perf_tracker.position_tracker
+        total_cash = 0.0
+        total_positions_value = 0.0
+
         try:
+            # Position keys correspond to assets
+            positions = self.portfolio.positions
+            assets = list(positions)
+            exchange_assets = group_assets_by_exchange(assets)
             for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
+                assets = exchange_assets[exchange_name] \
+                    if exchange_name in exchange_assets else []
 
-                exchange.synchronize_portfolio()
+                exchange_positions = \
+                    [positions[asset] for asset in assets]
 
-                # Applying the updated last_sales_price to the positions
-                # in the performance tracker. This seems a bit redundant
-                # but it will make sense when we have multiple exchange portfolios
-                # feeding into the same performance tracker.
-                tracker = self.perf_tracker.todays_performance.position_tracker
-                for asset in exchange.portfolio.positions:
-                    position = exchange.portfolio.positions[asset]
+                check_cash = (not self.simulate_orders)
+
+                exchange = self.exchanges[exchange_name]  # Type: Exchange
+                cash, positions_value = exchange.calculate_totals(
+                    positions=exchange_positions,
+                    check_cash=check_cash,
+                )
+                total_positions_value += positions_value
+
+                if cash is not None:
+                    total_cash += cash
+
+                for position in exchange_positions:
                     tracker.update_position(
-                        asset=asset,
+                        asset=position.asset,
                         last_sale_date=position.last_sale_date,
                         last_sale_price=position.last_sale_price
                     )
+
+            if cash is None:
+                total_cash = self.portfolio.cash
+
+            elif total_cash < self.portfolio.cash:
+                raise ValueError('Cash on exchanges is lower than the algo.')
+
+            return total_cash, total_positions_value
+
         except ExchangeRequestError as e:
             log.warn(
                 'update portfolio attempt {}: {}'.format(attempt_index, e)
             )
             if attempt_index < self.retry_synchronize_portfolio:
                 sleep(self.retry_delay)
-                self._synchronize_portfolio(attempt_index + 1)
+                return self.synchronize_portfolio(attempt_index + 1)
             else:
                 raise ExchangePortfolioDataError(
                     data_type='update-portfolio',
@@ -437,30 +538,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                     error=e
                 )
 
-    def _check_open_orders(self, attempt_index=0):
-        try:
-            orders = list()
-            for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
-                exchange_orders = exchange.check_open_orders()
-
-                orders += exchange_orders
-
-            return orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self._check_open_orders(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
-
     def add_pnl_stats(self, period_stats):
         """
         Save p&l stats.
@@ -534,8 +611,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.exposure_stats = pd.concat([self.exposure_stats, df])
 
-        save_algo_df(self.algo_namespace, 'exposure_stats',
-                     self.exposure_stats)
+        save_algo_df(
+            self.algo_namespace, 'exposure_stats', self.exposure_stats
+        )
 
     def handle_data(self, data):
         """
@@ -549,12 +627,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if not self.is_running:
             return
 
-        self._synchronize_portfolio()
+        # Resetting the frame stats every day to minimize memory footprint
+        today = data.current_dt.floor('1D')
+        if self.current_day is not None and today > self.current_day:
+            self.frame_stats = list()
 
-        transactions = self._check_open_orders()
-        for transaction in transactions:
-            self.perf_tracker.process_transaction(transaction)
+        new_transactions, new_commissions, closed_orders = \
+            self.blotter.get_transactions(data)
 
+        if len(new_transactions) > 0:
+            self.perf_tracker.update_performance()
+
+        cash, positions_value = self.synchronize_portfolio()
+        log.info(
+            'got totals from exchanges, cash: {} positions: {}'.format(
+                cash, positions_value
+            )
+        )
         if self._handle_data:
             self._handle_data(self, data)
 
@@ -564,48 +653,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.validate_account_controls()
 
         try:
-            # Since the clock runs 24/7, I trying to disable the daily
-            # Performance tracker and keep only minute and cumulative
-            self.perf_tracker.update_performance()
-
-            minute_stats = self.prepare_period_stats(
-                data.current_dt, data.current_dt + timedelta(minutes=1))
-
-            # Saving the last hour in memory
-            self.minute_stats.append(minute_stats)
-
-            self.add_pnl_stats(minute_stats)
-            if self.recorded_vars:
-                self.add_custom_signals_stats(minute_stats)
-                recorded_cols = list(self.recorded_vars.keys())
-            else:
-                recorded_cols = None
-
-            self.add_exposure_stats(minute_stats)
-
-            print_df = pd.DataFrame(list(self.minute_stats))
-            log.info(
-                'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
-                    stats_minutes=self.stats_minutes,
-                    stats=get_pretty_stats(
-                        stats_df=print_df,
-                        recorded_cols=recorded_cols,
-                        num_rows=self.stats_minutes
-                    )
-                ))
-
-            today = pd.to_datetime('today', utc=True)
-            daily_stats = self.prepare_period_stats(
-                start_dt=today,
-                end_dt=pd.Timestamp.utcnow()
-            )
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key=today.strftime('%Y-%m-%d'),
-                obj=daily_stats,
-                rel_path='daily_perf'
-            )
-
+            self._save_stats_csv(self._process_stats(data))
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
@@ -619,93 +667,85 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         except Exception as e:
             log.warn('unable to save minute perfs to disk: {}'.format(e))
 
-        try:
-            for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
-                save_algo_object(
-                    algo_name=self.algo_namespace,
-                    key='portfolio_{}'.format(exchange_name),
-                    obj=exchange.portfolio
-                )
-        except Exception as e:
-            log.warn('unable to save portfolio to disk: {}'.format(e))
+        self.current_day = data.current_dt.floor('1D')
 
-    def _order(self,
-               asset,
-               amount,
-               limit_price=None,
-               stop_price=None,
-               style=None,
-               attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(asset, amount, limit_price,
-                                  stop_price,
-                                  style)
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-                return self._order(
-                    asset, amount, limit_price, stop_price, style,
-                    attempt_index + 1)
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
+    def _process_stats(self, data):
+        today = data.current_dt.floor('1D')
 
-    @api_method
-    @disallowed_in_before_trading_start(OrderInBeforeTradingStart())
-    @expect_types(asset=TradingPair)
-    def order(self,
-              asset,
-              amount,
-              limit_price=None,
-              stop_price=None,
-              style=None):
-        """
-        We use the exchange specific portfolio to place orders.
-        The cumulative portfolio does not contain open orders but exchange
-        portfolios do.
+        # Since the clock runs 24/7, I trying to disable the daily
+        # Performance tracker and keep only minute and cumulative
+        self.perf_tracker.update_performance()
 
-        Parameters
-        ----------
-        asset: TradingPair
-        amount: float
-        limit_price: float
-        stop_price: float
-        style: Style
-        order: Order
-            The catalyst order object or None
-        """
-        amount, style = self._calculate_order(asset, amount,
-                                              limit_price, stop_price,
-                                              style)
+        frame_stats = self.prepare_period_stats(
+            data.current_dt, data.current_dt + timedelta(minutes=1))
 
-        order_id = self._order(asset, amount, limit_price, stop_price, style)
+        # Saving the last hour in memory
+        self.frame_stats.append(frame_stats)
 
-        exchange = self.exchanges[asset.exchange]
-        exchange_portfolio = exchange.portfolio
-        if order_id is not None:
+        self.add_pnl_stats(frame_stats)
+        if self.recorded_vars:
+            self.add_custom_signals_stats(frame_stats)
+            recorded_cols = list(self.recorded_vars.keys())
 
-            if order_id in exchange_portfolio.open_orders:
-                order = exchange_portfolio.open_orders[order_id]
-                self.perf_tracker.process_order(order)
-                return order
-
-            else:
-                raise OrphanOrderError(
-                    order_id=order_id,
-                    exchange=exchange.name
-                )
         else:
-            log.warn('unable to order {} {} on exchange {}'.format(
-                amount, asset.symbol, asset.exchange))
-            return None
+            recorded_cols = None
+
+        self.add_exposure_stats(frame_stats)
+
+        log.info(
+            'statistics for the last {stats_minutes} minutes:\n'
+            '{stats}'.format(
+                stats_minutes=self.stats_minutes,
+                stats=get_pretty_stats(
+                    stats=self.frame_stats,
+                    recorded_cols=recorded_cols,
+                    num_rows=self.stats_minutes
+                )
+            ))
+
+        # Saving the daily stats in a format usable for performance
+        # analysis.
+        daily_stats = self.prepare_period_stats(
+            start_dt=today,
+            end_dt=data.current_dt
+        )
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=daily_stats,
+            rel_path='daily_perf'
+        )
+
+        return recorded_cols
+
+    def _save_stats_csv(self, recorded_cols):
+        # Writing the stats output
+        csv_bytes = None
+        try:
+            csv_bytes = stats_to_algo_folder(
+                stats=self.frame_stats,
+                algo_namespace=self.algo_namespace,
+                recorded_cols=recorded_cols,
+            )
+        except Exception as e:
+            log.warn('unable save stats locally: {}'.format(e))
+
+        try:
+            if self.stats_output is not None:
+                if 's3://' in self.stats_output:
+                    stats_to_s3(
+                        uri=self.stats_output,
+                        stats=self.frame_stats,
+                        algo_namespace=self.algo_namespace,
+                        recorded_cols=recorded_cols,
+                        bytes_to_write=csv_bytes
+                    )
+                else:
+                    raise ValueError(
+                        'Only S3 stats output is supported for now.'
+                    )
+        except Exception as e:
+            log.warn('unable save stats externally: {}'.format(e))
 
     @api_method
     def batch_market_order(self, share_counts):

catalyst/exchange/exchange_blotter.py

@@ -1,21 +1,21 @@
+from time import sleep
+
+import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange_errors import ExchangeRequestError, \
+    ExchangePortfolioDataError, ExchangeTransactionError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
+from catalyst.finance.order import ORDER_STATUS, Order
 from catalyst.finance.slippage import SlippageModel
-from catalyst.finance.transaction import create_transaction
+from catalyst.finance.transaction import create_transaction, Transaction
+from catalyst.utils.input_validation import expect_types
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
-# It seems like we need to accept greater slippage risk in cryptos
-# Orders won't often close at Equity levels.
-# TODO: should work with set_commission and set_slippage
-DEFAULT_SLIPPAGE_SPREAD = 0.0001
-DEFAULT_MAKER_FEE = 0.0015
-DEFAULT_TAKER_FEE = 0.0025
-
 
 class TradingPairFeeSchedule(CommissionModel):
     """
@@ -23,23 +23,24 @@ class TradingPairFeeSchedule(CommissionModel):
 
     Parameters
     ----------
-    fee : float, optional
-        The percentage fee.
+    maker : float, optional
+        The percentage maker fee.
+
+    taker: float, optional
+        The percentage taker fee.
     """
 
-    def __init__(self,
-                 maker_fee=DEFAULT_MAKER_FEE,
-                 taker_fee=DEFAULT_TAKER_FEE):
-        self.maker_fee = maker_fee
-        self.taker_fee = taker_fee
+    def __init__(self, maker=None, taker=None):
+        self.maker = maker
+        self.taker = taker
 
     def __repr__(self):
         return (
-            '{class_name}(maker_fee={maker_fee}, '
-            'taker_fee={taker_fee})'.format(
+            '{class_name}(maker={maker}, '
+            'taker={taker})'.format(
                 class_name=self.__class__.__name__,
-                maker_fee=self.maker_fee,
-                taker_fee=self.taker_fee,
+                maker=self.maker,
+                taker=self.taker,
             )
         )
 
@@ -47,16 +48,25 @@ class TradingPairFeeSchedule(CommissionModel):
         """
         Calculate the final fee based on the order parameters.
 
-        :param order:
-        :param transaction:
+        :param order: Order
+        :param transaction: Transaction
 
         :return float:
             The total commission.
         """
         cost = abs(transaction.amount) * transaction.price
 
+        asset = order.asset
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+
+        multiplier = maker \
+            if ((order.amount > 0 and order.limit < transaction.price)
+                or (order.amount < 0 and order.limit > transaction.price)) \
+               and order.limit_reached else taker
+
         # Assuming just the taker fee for now
-        fee = cost * self.taker_fee
+        fee = cost * multiplier
         return fee
 
 
@@ -70,7 +80,7 @@ class TradingPairFixedSlippage(SlippageModel):
         spread / 2 will be added to buys and subtracted from sells.
     """
 
-    def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
+    def __init__(self, spread=0.0001):
         super(TradingPairFixedSlippage, self).__init__()
         self.spread = spread
 
@@ -121,6 +131,14 @@ class TradingPairFixedSlippage(SlippageModel):
 
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
+        self.simulate_orders = kwargs.pop('simulate_orders', False)
+
+        self.exchanges = kwargs.pop('exchanges', None)
+        if not self.exchanges:
+            raise ValueError(
+                'ExchangeBlotter must have an `exchanges` attribute.'
+            )
+
         super(ExchangeBlotter, self).__init__(*args, **kwargs)
 
         # Using the equity models for now
@@ -132,3 +150,148 @@ class ExchangeBlotter(Blotter):
         self.commission_models = {
             TradingPair: TradingPairFeeSchedule()
         }
+
+        self.retry_delay = 5
+        self.retry_check_open_orders = 5
+
+    def exchange_order(self, asset, amount, style=None, attempt_index=0):
+        try:
+            exchange = self.exchanges[asset.exchange]
+            return exchange.order(
+                asset, amount, style
+            )
+        except ExchangeRequestError as e:
+            log.warn(
+                'order attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_order:
+                sleep(self.retry_delay)
+
+                return self.exchange_order(
+                    asset, amount, style, attempt_index + 1
+                )
+            else:
+                raise ExchangeTransactionError(
+                    transaction_type='order',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    @expect_types(asset=TradingPair)
+    def order(self, asset, amount, style, order_id=None):
+        log.debug('ordering {} {}'.format(amount, asset.symbol))
+        if amount == 0:
+            log.warn('skipping 0 amount orders')
+            return None
+
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).order(
+                asset, amount, style, order_id
+            )
+
+        else:
+            order = self.exchange_order(
+                asset, amount, style
+            )
+
+            self.open_orders[order.asset].append(order)
+            self.orders[order.id] = order
+            self.new_orders.append(order)
+
+            return order.id
+
+    def check_open_orders(self):
+        """
+        Loop through the list of open orders in the Portfolio object.
+        For each executed order found, create a transaction and apply to the
+        Portfolio.
+
+        Returns
+        -------
+        list[Transaction]
+
+        """
+        for asset in self.open_orders:
+            exchange = self.exchanges[asset.exchange]
+
+            for order in self.open_orders[asset]:
+                log.debug('found open order: {}'.format(order.id))
+
+                new_order, executed_price = exchange.get_order(order.id, asset)
+                log.debug(
+                    'got updated order {} {}'.format(
+                        new_order, executed_price
+                    )
+                )
+                order.status = new_order.status
+
+                if order.status == ORDER_STATUS.FILLED:
+                    order.commission = new_order.commission
+                    if order.amount != new_order.amount:
+                        log.warn(
+                            'executed order amount {} differs '
+                            'from original'.format(
+                                new_order.amount, order.amount
+                            )
+                        )
+                        order.amount = new_order.amount
+
+                    transaction = Transaction(
+                        asset=order.asset,
+                        amount=order.amount,
+                        dt=pd.Timestamp.utcnow(),
+                        price=executed_price,
+                        order_id=order.id,
+                        commission=order.commission
+                    )
+                    yield order, transaction
+
+                elif order.status == ORDER_STATUS.CANCELLED:
+                    yield order, None
+
+                else:
+                    delta = pd.Timestamp.utcnow() - order.dt
+                    log.info(
+                        'order {order_id} still open after {delta}'.format(
+                            order_id=order.id,
+                            delta=delta
+                        )
+                    )
+
+    def get_exchange_transactions(self, attempt_index=0):
+        closed_orders = []
+        transactions = []
+        commissions = []
+
+        try:
+            for order, txn in self.check_open_orders():
+                order.dt = txn.dt
+
+                transactions.append(txn)
+
+                if not order.open:
+                    closed_orders.append(order)
+
+            return transactions, commissions, closed_orders
+
+        except ExchangeRequestError as e:
+            log.warn(
+                'check open orders attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_check_open_orders:
+                sleep(self.retry_delay)
+                return self.get_exchange_transactions(attempt_index + 1)
+
+            else:
+                raise ExchangePortfolioDataError(
+                    data_type='order-status',
+                    attempts=attempt_index,
+                    error=e
+                )
+
+    def get_transactions(self, bar_data):
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).get_transactions(bar_data)
+
+        else:
+            return self.get_exchange_transactions()

catalyst/exchange/exchange_bundle.py

@@ -1,13 +1,14 @@
 import os
 import shutil
+from datetime import timedelta
 from functools import partial
 from itertools import chain
 from operator import is_not
 
 import numpy as np
 import pandas as pd
+import pytz
 from catalyst.assets._assets import TradingPair
-from datetime import datetime, timedelta
 from logbook import Logger
 from pytz import UTC
 from six import itervalues
@@ -19,14 +20,16 @@ from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label
+    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
+    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
-    PricingDataNotLoadedError
-from catalyst.exchange.exchange_utils import get_exchange_folder
+    PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
+from catalyst.exchange.exchange_utils import get_exchange_folder, \
+    save_exchange_symbols, mixin_market_params
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -40,23 +43,14 @@ def _cachpath(symbol, type_):
 
 
 class ExchangeBundle:
-    def __init__(self, exchange):
-        self.exchange = exchange
+    def __init__(self, exchange_name):
+        self.exchange_name = exchange_name
         self.minutes_per_day = 1440
         self.default_ohlc_ratio = 1000000
         self._writers = dict()
         self._readers = dict()
         self.calendar = get_calendar('OPEN')
-
-    def get_assets(self, include_symbols, exclude_symbols):
-        # TODO: filter exclude symbols assets
-        if include_symbols is not None:
-            include_symbols_list = include_symbols.split(',')
-
-            return self.exchange.get_assets(include_symbols_list)
-
-        else:
-            return self.exchange.get_assets()
+        self.exchange = None
 
     def get_reader(self, data_frequency, path=None):
         """
@@ -68,7 +62,7 @@ class ExchangeBundle:
 
         """
         if path is None:
-            root = get_exchange_folder(self.exchange.name)
+            root = get_exchange_folder(self.exchange_name)
             path = BUNDLE_NAME_TEMPLATE.format(
                 root=root,
                 frequency=data_frequency
@@ -99,7 +93,7 @@ class ExchangeBundle:
         BcolzMinuteBarWriter | BcolzDailyBarWriter
 
         """
-        root = get_exchange_folder(self.exchange.name)
+        root = get_exchange_folder(self.exchange_name)
         path = BUNDLE_NAME_TEMPLATE.format(
             root=root,
             frequency=data_frequency
@@ -157,9 +151,9 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
             The assets is scope.
-        start_dt: datetime
+        start_dt: pd.Timestamp
             The chunk start date.
-        end_dt: datetime
+        end_dt: pd.Timestamp
             The chunk end date.
         data_frequency: str
 
@@ -208,8 +202,8 @@ class ExchangeBundle:
 
         Parameters
         ----------
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
         data_frequency: str
 
         Returns
@@ -239,11 +233,13 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                    name=asset.symbol,
-                    start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                    dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
-                )
+                            name=asset.symbol,
+                            start_dt=asset.start_date.strftime(
+                                DATE_TIME_FORMAT),
+                            end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                            dates=[date.strftime(
+                                DATE_TIME_FORMAT) for date in dates])
+
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
 
@@ -251,8 +247,7 @@ class ExchangeBundle:
                     raise EmptyValuesInBundleError(
                         name=asset.symbol,
                         end_minute=end_dt,
-                        dates=dates
-                    )
+                        dates=dates, )
 
                 else:
                     ohlcv_df.dropna(inplace=True)
@@ -292,13 +287,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                name=asset.symbol,
-                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                threshold=threshold,
-                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                       for date in dates]
-            )
+                        name=asset.symbol,
+                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                        threshold=threshold,
+                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                               for date in dates])
 
             problems.append(problem)
 
@@ -366,7 +360,7 @@ class ExchangeBundle:
 
         # Download and extract the bundle
         path = get_bcolz_chunk(
-            exchange_name=self.exchange.name,
+            exchange_name=self.exchange_name,
             symbol=asset.symbol,
             data_frequency=data_frequency,
             period=period
@@ -435,14 +429,14 @@ class ExchangeBundle:
 
         Parameters
         ----------
-        start: datetime
-        end: datetime
+        start: pd.Timestamp
+        end: pd.Timestamp
         assets: list[TradingPair]
         data_frequency: str
 
         Returns
         -------
-        datetime, datetime
+        pd.Timestamp, pd.Timestamp
         """
         earliest_trade = None
         last_entry = None
@@ -469,7 +463,8 @@ class ExchangeBundle:
             start = earliest_trade
 
         if end is None or (last_entry is not None and end > last_entry):
-            end = last_entry
+            end = last_entry.replace(minute=59, hour=23) \
+                if data_frequency == 'minute' else last_entry
 
         if end is None or start is None or start > end:
             raise NoDataAvailableOnExchange(
@@ -489,8 +484,8 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
         data_frequency: str
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
 
         Returns
         -------
@@ -573,8 +568,8 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
         data_frequency: str
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
         show_progress: bool
         show_breakdown: bool
 
@@ -610,7 +605,7 @@ class ExchangeBundle:
                         show_progress,
                         label='Ingesting {frequency} price data for '
                               '{symbol} on {exchange}'.format(
-                            exchange=self.exchange.name,
+                            exchange=self.exchange_name,
                             frequency=data_frequency,
                             symbol=asset.symbol
                         )) as it:
@@ -635,8 +630,8 @@ class ExchangeBundle:
                     show_progress,
                     label='Ingesting {frequency} price data on '
                           '{exchange}'.format(
-                        exchange=self.exchange.name,
-                        frequency=data_frequency,
+                            exchange=self.exchange_name,
+                            frequency=data_frequency,
                     )) as it:
                 for chunk in it:
                     problems += self.ingest_ctable(
@@ -653,8 +648,142 @@ class ExchangeBundle:
                 '\n'.join(problems)
             ))
 
+    def ingest_csv(self, path, data_frequency, empty_rows_behavior='strip',
+                   duplicates_threshold=100):
+        """
+        Ingest price data from a CSV file.
+
+        Parameters
+        ----------
+        path: str
+        data_frequency: str
+
+        Returns
+        -------
+        list[str]
+            A list of potential problems detected during ingestion.
+
+        """
+        log.info('ingesting csv file: {}'.format(path))
+
+        if self.exchange is None:
+            # Avoid circular dependencies
+            from catalyst.exchange.factory import get_exchange
+            self.exchange = get_exchange(self.exchange_name)
+
+        problems = []
+        df = pd.read_csv(
+            path,
+            header=0,
+            sep=',',
+            dtype=dict(
+                symbol=np.object_,
+                last_traded=np.object_,
+                open=np.float64,
+                high=np.float64,
+                close=np.float64,
+                volume=np.float64
+            ),
+            parse_dates=['last_traded'],
+            index_col=None
+        )
+        min_start_dt = None
+        max_end_dt = None
+
+        symbols = df['symbol'].unique()
+
+        # Apply the timezone before creating an index for simplicity
+        df['last_traded'] = df['last_traded'].dt.tz_localize(pytz.UTC)
+        df.set_index(['symbol', 'last_traded'], drop=True, inplace=True)
+
+        assets = dict()
+        for symbol in symbols:
+            start_dt = df.index.get_level_values(1).min()
+            end_dt = df.index.get_level_values(1).max()
+            end_dt_key = 'end_{}'.format(data_frequency)
+
+            market = self.exchange.get_market(symbol)
+            if market is None:
+                raise ValueError('symbol not available in the exchange.')
+
+            params = dict(
+                exchange=self.exchange.name,
+                data_source='local',
+                exchange_symbol=market['id'],
+            )
+            mixin_market_params(self.exchange_name, params, market)
+
+            asset_def = self.exchange.get_asset_def(market, True)
+            if asset_def is not None:
+                params['symbol'] = asset_def['symbol']
+
+                params['start_date'] = asset_def['start_date'] \
+                    if asset_def['start_date'] < start_dt else start_dt
+
+                params['end_date'] = asset_def[end_dt_key] \
+                    if asset_def[end_dt_key] > end_dt else end_dt
+
+                params['end_daily'] = end_dt \
+                    if data_frequency == 'daily' else asset_def['end_daily']
+
+                params['end_minute'] = end_dt \
+                    if data_frequency == 'minute' else asset_def['end_minute']
+
+            else:
+                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+
+                params['end_daily'] = end_dt \
+                    if data_frequency == 'daily' else 'N/A'
+                params['end_minute'] = end_dt \
+                    if data_frequency == 'minute' else 'N/A'
+
+            if min_start_dt is None or start_dt < min_start_dt:
+                min_start_dt = start_dt
+
+            if max_end_dt is None or end_dt > max_end_dt:
+                max_end_dt = end_dt
+
+            asset = TradingPair(**params)
+            assets[market['id']] = asset
+
+        save_exchange_symbols(self.exchange_name, assets, True)
+
+        writer = self.get_writer(
+            start_dt=min_start_dt.replace(hour=00, minute=00),
+            end_dt=max_end_dt.replace(hour=23, minute=59),
+            data_frequency=data_frequency
+        )
+
+        for symbol in assets:
+            asset = assets[symbol]
+            ohlcv_df = df.loc[
+                (df.index.get_level_values(0) == symbol)
+            ]  # type: pd.DataFrame
+            ohlcv_df.index = ohlcv_df.index.droplevel(0)
+
+            period_start = start_dt.replace(hour=00, minute=00)
+            period_end = end_dt.replace(hour=23, minute=59)
+            periods = self.get_calendar_periods_range(
+                period_start, period_end, data_frequency
+            )
+
+            # We're not really resampling but ensuring that each frame
+            # contains data
+            ohlcv_df = ohlcv_df.reindex(periods, method='ffill')
+            ohlcv_df['volume'] = ohlcv_df['volume'].fillna(0)
+
+            problems += self.ingest_df(
+                ohlcv_df=ohlcv_df,
+                data_frequency=data_frequency,
+                asset=asset,
+                writer=writer,
+                empty_rows_behavior=empty_rows_behavior,
+                duplicates_threshold=duplicates_threshold
+            )
+        return filter(partial(is_not, None), problems)
+
     def ingest(self, data_frequency, include_symbols=None,
-               exclude_symbols=None, start=None, end=None,
+               exclude_symbols=None, start=None, end=None, csv=None,
                show_progress=True, show_breakdown=True, show_report=True):
         """
         Inject data based on specified parameters.
@@ -664,17 +793,34 @@ class ExchangeBundle:
         data_frequency: str
         include_symbols: str
         exclude_symbols: str
-        start: datetime
-        end: datetime
+        start: pd.Timestamp
+        end: pd.Timestamp
         show_progress: bool
         environ:
 
         """
-        assets = self.get_assets(include_symbols, exclude_symbols)
+        if csv is not None:
+            self.ingest_csv(csv, data_frequency)
 
-        for frequency in data_frequency.split(','):
-            self.ingest_assets(assets, frequency, start, end,
-                               show_progress, show_breakdown, show_report)
+        else:
+            if self.exchange is None:
+                # Avoid circular dependencies
+                from catalyst.exchange.factory import get_exchange
+                self.exchange = get_exchange(self.exchange_name)
+
+            assets = get_assets(
+                self.exchange, include_symbols, exclude_symbols
+            )
+            for frequency in data_frequency.split(','):
+                self.ingest_assets(
+                    assets=assets,
+                    data_frequency=frequency,
+                    start_dt=start,
+                    end_dt=end,
+                    show_progress=show_progress,
+                    show_breakdown=show_breakdown,
+                    show_report=show_report
+                )
 
     def get_history_window_series_and_load(self,
                                            assets,
@@ -682,7 +828,9 @@ class ExchangeBundle:
                                            bar_count,
                                            field,
                                            data_frequency,
-                                           algo_end_dt=None
+                                           algo_end_dt=None,
+                                           trailing_bar_count=None,
+                                           force_auto_ingest=False
                                            ):
         """
         Retrieve price data history, ingest missing data.
@@ -690,25 +838,26 @@ class ExchangeBundle:
         Parameters
         ----------
         assets: list[TradingPair]
-        end_dt: datetime
+        end_dt: pd.Timestamp
         bar_count: int
         field: str
         data_frequency: str
-        algo_end_dt: datetime
+        algo_end_dt: pd.Timestamp
 
         Returns
         -------
         Series
 
         """
-        if AUTO_INGEST:
+        if AUTO_INGEST or force_auto_ingest:
             try:
                 series = self.get_history_window_series(
                     assets=assets,
                     end_dt=end_dt,
                     bar_count=bar_count,
                     field=field,
-                    data_frequency=data_frequency
+                    data_frequency=data_frequency,
+                    trailing_bar_count=trailing_bar_count,
                 )
                 return pd.DataFrame(series)
 
@@ -725,7 +874,7 @@ class ExchangeBundle:
                 self.ingest_assets(
                     assets=assets,
                     start_dt=start_dt,
-                    end_dt=algo_end_dt,
+                    end_dt=algo_end_dt,  # TODO: apply trailing bars
                     data_frequency=data_frequency,
                     show_progress=True,
                     show_breakdown=True
@@ -736,7 +885,8 @@ class ExchangeBundle:
                     bar_count=bar_count,
                     field=field,
                     data_frequency=data_frequency,
-                    reset_reader=True
+                    reset_reader=True,
+                    trailing_bar_count=trailing_bar_count,
                 )
                 return series
 
@@ -746,7 +896,8 @@ class ExchangeBundle:
                 end_dt=end_dt,
                 bar_count=bar_count,
                 field=field,
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                trailing_bar_count=trailing_bar_count,
             )
             return pd.DataFrame(series)
 
@@ -796,7 +947,7 @@ class ExchangeBundle:
             raise PricingDataNotLoadedError(
                 field=field,
                 first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
+                exchange=self.exchange_name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
                 data_frequency=data_frequency,
@@ -810,12 +961,20 @@ class ExchangeBundle:
                                   bar_count,
                                   field,
                                   data_frequency,
+                                  trailing_bar_count=None,
                                   reset_reader=False):
         start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
-        start_dt, end_dt = self.get_adj_dates(
+        start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
 
+        if trailing_bar_count:
+            delta = get_delta(trailing_bar_count, data_frequency)
+            end_dt += delta
+
+        # This is an attempt to resolve some caching with the reader
+        # when auto-ingesting data.
+        # TODO: needs more work
         reader = self.get_reader(data_frequency)
         if reset_reader:
             del self._readers[reader._rootdir]
@@ -826,7 +985,7 @@ class ExchangeBundle:
             raise PricingDataNotLoadedError(
                 field=field,
                 first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
+                exchange=self.exchange_name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
                 data_frequency=data_frequency,
@@ -834,57 +993,61 @@ class ExchangeBundle:
                 end_dt=end_dt
             )
 
+        series = dict()
         for asset in assets:
-            asset_start_dt, asset_end_dt = self.get_adj_dates(
+            asset_start_dt, _ = self.get_adj_dates(
                 start_dt, end_dt, assets, data_frequency
             )
-
             in_bundle = range_in_bundle(
-                asset, asset_start_dt, asset_end_dt, reader
+                asset, asset_start_dt, end_dt, reader
             )
             if not in_bundle:
                 raise PricingDataNotLoadedError(
                     field=field,
                     first_trading_day=asset.start_date,
-                    exchange=self.exchange.name,
+                    exchange=self.exchange_name,
                     symbols=asset.symbol,
                     symbol_list=asset.symbol,
                     data_frequency=data_frequency,
                     start_dt=asset_start_dt,
-                    end_dt=asset_end_dt
+                    end_dt=end_dt
                 )
 
-        series = dict()
-        try:
+            periods = self.get_calendar_periods_range(
+                asset_start_dt, end_dt, data_frequency
+            )
+            # This does not behave well when requesting multiple assets
+            # when the start or end date of one asset is outside of the range
+            # looking at the logic in load_raw_arrays(), we are not achieving
+            # any performance gain by requesting multiple sids at once. It's
+            # looping through the sids and making separate requests anyway.
             arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
+                sids=[asset.sid],
                 fields=[field],
                 start_dt=start_dt,
                 end_dt=end_dt
             )
+            if len(arrays) == 0:
+                raise DataCorruptionError(
+                    exchange=self.exchange_name,
+                    symbols=asset.symbol,
+                    start_dt=asset_start_dt,
+                    end_dt=end_dt
+                )
 
-        except Exception:
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
-                symbols=symbols,
-                symbol_list=','.join(symbols),
-                data_frequency=data_frequency,
-                start_dt=start_dt,
-                end_dt=end_dt
-            )
+            field_values = arrays[0][:, 0]
 
-        periods = self.get_calendar_periods_range(
-            start_dt, end_dt, data_frequency
-        )
-
-        for asset_index, asset in enumerate(assets):
-            asset_values = arrays[asset_index]
-
-            value_series = pd.Series(asset_values.flatten(), index=periods)
-            series[asset] = value_series
+            try:
+                value_series = pd.Series(field_values, index=periods)
+                series[asset] = value_series
+            except ValueError as e:
+                raise PricingDataValueError(
+                    exchange=asset.exchange,
+                    symbol=asset.symbol,
+                    start_dt=asset_start_dt,
+                    end_dt=end_dt,
+                    error=e
+                )
 
         return series
 
@@ -898,14 +1061,18 @@ class ExchangeBundle:
 
         """
         log.debug('cleaning exchange {}, frequency {}'.format(
-            self.exchange.name, data_frequency
+            self.exchange_name, data_frequency
         ))
-        root = get_exchange_folder(self.exchange.name)
+        root = get_exchange_folder(self.exchange_name)
 
         symbols = os.path.join(root, 'symbols.json')
         if os.path.isfile(symbols):
             os.remove(symbols)
 
+        local_symbols = os.path.join(root, 'symbols_local.json')
+        if os.path.isfile(local_symbols):
+            os.remove(local_symbols)
+
         temp_bundles = os.path.join(root, 'temp_bundles')
 
         if os.path.isdir(temp_bundles):

catalyst/exchange/exchange_data_portal.py

@@ -13,7 +13,8 @@ from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
     PricingDataNotLoadedError)
-from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
+from catalyst.exchange.exchange_utils import get_frequency, \
+    resample_history_df, group_assets_by_exchange
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
@@ -21,7 +22,6 @@ log = Logger('DataPortalExchange', level=LOG_LEVEL)
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
 
-        self.exchanges = kwargs.pop('exchanges', None)
         # TODO: put somewhere accessible by each algo
         self.retry_get_history_window = 5
         self.retry_get_spot_value = 5
@@ -39,21 +39,14 @@ class DataPortalExchangeBase(DataPortal):
                             ffill=True,
                             attempt_index=0):
         try:
-            exchange_assets = dict()
-            for asset in assets:
-                if asset.exchange not in exchange_assets:
-                    exchange_assets[asset.exchange] = list()
-
-                exchange_assets[asset.exchange].append(asset)
-
+            exchange_assets = group_assets_by_exchange(assets)
             if len(exchange_assets) > 1:
                 df_list = []
                 for exchange_name in exchange_assets:
-                    exchange = self.exchanges[exchange_name]
                     assets = exchange_assets[exchange_name]
 
                     df_exchange = self.get_exchange_history_window(
-                        exchange,
+                        exchange_name,
                         assets,
                         end_dt,
                         bar_count,
@@ -68,9 +61,9 @@ class DataPortalExchangeBase(DataPortal):
                 return pd.concat(df_list)
 
             else:
-                exchange = self.exchanges[list(exchange_assets.keys())[0]]
+                exchange_name = list(exchange_assets.keys())[0]
                 return self.get_exchange_history_window(
-                    exchange,
+                    exchange_name,
                     assets,
                     end_dt,
                     bar_count,
@@ -122,7 +115,7 @@ class DataPortalExchangeBase(DataPortal):
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -136,9 +129,8 @@ class DataPortalExchangeBase(DataPortal):
                         attempt_index=0):
         try:
             if isinstance(assets, TradingPair):
-                exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
-                    exchange, [assets], field, dt, data_frequency)
+                    assets.exchange, [assets], field, dt, data_frequency)
 
                 if not spot_values:
                     return np.nan
@@ -154,17 +146,16 @@ class DataPortalExchangeBase(DataPortal):
                     exchange_assets[asset.exchange].append(asset)
 
                 if len(list(exchange_assets.keys())) == 1:
-                    exchange = self.exchanges[list(exchange_assets.keys())[0]]
+                    exchange_name = list(exchange_assets.keys())[0]
                     return self.get_exchange_spot_value(
-                        exchange, assets, field, dt, data_frequency)
+                        exchange_name, assets, field, dt, data_frequency)
 
                 else:
                     spot_values = []
                     for exchange_name in exchange_assets:
-                        exchange = self.exchanges[exchange_name]
                         assets = exchange_assets[exchange_name]
                         exchange_spot_values = self.get_exchange_spot_value(
-                            exchange,
+                            exchange_name,
                             assets,
                             field,
                             dt,
@@ -199,7 +190,7 @@ class DataPortalExchangeBase(DataPortal):
         return self._get_spot_value(assets, field, dt, data_frequency)
 
     @abc.abstractmethod
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
         return
 
@@ -214,10 +205,11 @@ class DataPortalExchangeBase(DataPortal):
 
 class DataPortalExchangeLive(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchanges = kwargs.pop('exchanges', None)
         super(DataPortalExchangeLive, self).__init__(*args, **kwargs)
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -230,7 +222,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: Exchange
         assets: list[TradingPair]
         end_dt: datetime
         bar_count: int
@@ -244,6 +236,8 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
         DataFrame
 
         """
+        exchange = self.exchanges[exchange_name]
+
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -251,17 +245,17 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
             frequency,
             field,
             data_frequency,
-            ffill)
+            False)
         return df
 
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
         """
         A spot value for the exchange.
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: str
         assets: list[TradingPair]
         field: str
         dt: datetime
@@ -272,6 +266,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
         float
 
         """
+        exchange = self.exchanges[exchange_name]
         exchange_spot_values = exchange.get_spot_value(
             assets, field, dt, data_frequency)
 
@@ -280,16 +275,16 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
 class DataPortalExchangeBacktest(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchange_names = kwargs.pop('exchange_names', None)
+
         super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs)
 
         self.exchange_bundles = dict()
-
         self.history_loaders = dict()
         self.minute_history_loaders = dict()
 
-        for exchange_name in self.exchanges:
-            exchange = self.exchanges[exchange_name]
-            self.exchange_bundles[exchange_name] = ExchangeBundle(exchange)
+        for name in self.exchange_names:
+            self.exchange_bundles[name] = ExchangeBundle(name)
 
     def _get_first_trading_day(self, assets):
         first_date = None
@@ -299,7 +294,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         return first_date
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -326,12 +321,13 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         DataFrame
 
         """
-        bundle = self.exchange_bundles[exchange.name]  # type: ExchangeBundle
+        bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
+        trailing_bar_count = candle_size - 1
 
         if data_frequency == 'minute' and adj_data_frequency == 'daily':
             end_dt = end_dt.floor('1D')
@@ -343,13 +339,14 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
+            trailing_bar_count=trailing_bar_count
         )
 
         df = resample_history_df(pd.DataFrame(series), freq, field)
         return df
 
     def get_exchange_spot_value(self,
-                                exchange,
+                                exchange_name,
                                 assets,
                                 field,
                                 dt,
@@ -361,7 +358,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: str
         assets: list[TradingPair]
         field: str
         dt: datetime
@@ -372,7 +369,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         float
 
         """
-        bundle = self.exchange_bundles[exchange.name]
+        bundle = self.exchange_bundles[exchange_name]
         if data_frequency == 'daily':
             dt = dt.floor('1D')
         else:

catalyst/exchange/exchange_errors.py

@@ -143,7 +143,8 @@ class OrphanOrderError(ZiplineError):
 
 class OrphanOrderReverseError(ZiplineError):
     msg = (
-        'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
+        'Order {order_id} tracked by algorithm, but not found in exchange '
+        '{exchange}.'
     ).strip()
 
 
@@ -206,8 +207,9 @@ class EmptyValuesInBundleError(ZiplineError):
 
 class PricingDataBeforeTradingError(ZiplineError):
     msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
-           'starts on {first_trading_day}, but you are either trying to trade or '
-           'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
+           'starts on {first_trading_day}, but you are either trying to trade '
+           'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
+           ).strip()
 
 
 class PricingDataNotLoadedError(ZiplineError):
@@ -218,12 +220,52 @@ class PricingDataNotLoadedError(ZiplineError):
            'for details.').strip()
 
 
+class PricingDataValueError(ZiplineError):
+    msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
+           '[{start_dt} - {end_dt}]: {error}').strip()
+
+
+class DataCorruptionError(ZiplineError):
+    msg = ('Unable to validate data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]. The data is either corrupted or '
+           'unavailable. Please try deleting this bundle:'
+           '\n`catalyst clean-exchange -x {exchange}\n'
+           'Then, ingest the data again. Please contact the Catalyst team if '
+           'the issue persists.').strip()
+
+
 class ApiCandlesError(ZiplineError):
     msg = ('Unable to fetch candles from the remote API: {error}.').strip()
 
 
 class NoDataAvailableOnExchange(ZiplineError):
     msg = (
-        'Requested data for trading pair {symbol} is not available on exchange {exchange} '
+        'Requested data for trading pair {symbol} is not available on '
+        'exchange {exchange} '
         'in `{data_frequency}` frequency at this time. '
-        'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+        'Check `http://enigma.co/catalyst/status` for market coverage.'
+    ).strip()
+
+
+class NoValueForField(ZiplineError):
+    msg = ('Value not found for field: {field}.').strip()
+
+
+class OrderTypeNotSupported(ZiplineError):
+    msg = (
+        'Order type `{order_type}` not currencly supported by Catalyst. '
+        'Please use `limit` or `market` orders only.').strip()
+
+
+class NotEnoughCapitalError(ZiplineError):
+    msg = (
+        'Not enough capital on exchange {exchange} for trading. Each '
+        'exchange should contain at least as much {base_currency} '
+        'as the specified `capital_base`. The current balance {balance} is '
+        'lower than the `capital_base`: {capital_base}').strip()
+
+class LastCandleTooEarlyError(ZiplineError):
+    msg = (
+        'The trade date of the last candle {last_traded} is before the '
+        'specified end date minus one candle {end_dt}. Please verify how '
+        '{exchange} calculates the start date of OHLCV candles.').strip()

catalyst/exchange/exchange_portfolio.py

@@ -3,7 +3,6 @@ from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
-from catalyst.utils.deprecate import deprecated
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
@@ -11,7 +10,8 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 class ExchangePortfolio(Portfolio):
     """
     Since the goal is to support multiple exchanges, it makes sense to
-    include additional stats in the portfolio object.
+    include additional stats in the portfolio object. This fills the role
+    of Blotter and Portfolio in live mode.
 
     Instead of relying on the performance tracker, each exchange portfolio
     tracks its own holding. This offers a separation between tracking an
@@ -40,7 +40,13 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.debug('creating order {}'.format(order.id))
-        self.open_orders[order.id] = order
+
+        open_orders = self.open_orders[order.asset] \
+            if order.asset is self.open_orders else []
+
+        open_orders.append(order)
+
+        self.open_orders[order.asset] = open_orders
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
@@ -52,6 +58,17 @@ class ExchangePortfolio(Portfolio):
         order_position.amount += order.amount
         log.debug('open order added to portfolio')
 
+    def _remove_open_order(self, order):
+        try:
+            open_orders = self.open_orders[order.asset]
+            if order in open_orders:
+                open_orders.remove(order)
+
+        except Exception:
+            raise ValueError(
+                'unable to clear order not found in open order list.'
+            )
+
     def execute_order(self, order, transaction):
         """
         Update the open orders and positions to apply an executed order.
@@ -66,14 +83,15 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.debug('executing order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
 
         if order_position is None:
             raise ValueError(
-                'Trying to execute order for a position not held: %s' % order.id
+                'Trying to execute order for a position not held:'
+                ' {}'.format(order.id)
             )
 
         self.capital_used += order.amount * transaction.price
@@ -89,32 +107,6 @@ class ExchangePortfolio(Portfolio):
 
         log.debug('updated portfolio with executed order')
 
-    @deprecated
-    def execute_transaction(self, transaction):
-        # TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
-        log.debug('executing transaction {}'.format(transaction.order_id))
-
-        order_position = self.positions[transaction.asset] \
-            if transaction.asset in self.positions else None
-
-        if order_position is None:
-            raise ValueError(
-                'Trying to execute transaction for a position not held: %s' % transaction.order_id
-            )
-
-        self.capital_used += transaction.amount * transaction.price
-
-        if transaction.amount > 0:
-            if order_position.cost_basis > 0:
-                order_position.cost_basis = np.average(
-                    [order_position.cost_basis, transaction.price],
-                    weights=[order_position.amount, transaction.amount]
-                )
-            else:
-                order_position.cost_basis = transaction.price
-
-        log.debug('updated portfolio with executed order')
-
     def remove_order(self, order):
         """
         Removing an open order.
@@ -125,7 +117,7 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.info('removing cancelled order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None

catalyst/exchange/exchange_utils.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import pickle
@@ -7,15 +8,34 @@ from datetime import date, datetime
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
+from six import string_types
 from six.moves.urllib import request
 
+from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
 from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
     InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
-SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
-              '{exchange}/symbols.json'
+
+def get_sid(symbol):
+    """
+    Create a sid by hashing the symbol of a currency pair.
+
+    Parameters
+    ----------
+    symbol: str
+
+    Returns
+    -------
+    int
+        The resulting sid.
+
+    """
+    sid = int(
+        hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+    ) % 10 ** 6
+    return sid
 
 
 def get_exchange_folder(exchange_name, environ=None):
@@ -42,7 +62,7 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
-def get_exchange_symbols_filename(exchange_name, environ=None):
+def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
 
@@ -56,8 +76,9 @@ def get_exchange_symbols_filename(exchange_name, environ=None):
     str
 
     """
+    name = 'symbols.json' if not is_local else 'symbols_local.json'
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    return os.path.join(exchange_folder, 'symbols.json')
+    return os.path.join(exchange_folder, name)
 
 
 def download_exchange_symbols(exchange_name, environ=None):
@@ -80,13 +101,28 @@ def download_exchange_symbols(exchange_name, environ=None):
     return response
 
 
-def get_exchange_symbols(exchange_name, environ=None):
+def symbols_parser(asset_def):
+    for key, value in asset_def.items():
+        match = isinstance(value, string_types) \
+                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
+
+        if match:
+            try:
+                asset_def[key] = pd.to_datetime(value, utc=True)
+            except ValueError:
+                pass
+
+    return asset_def
+
+
+def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     """
     The de-serialized content of the exchange's symbols.json.
 
     Parameters
     ----------
     exchange_name: str
+    is_local: bool
     environ:
 
     Returns
@@ -94,18 +130,21 @@ def get_exchange_symbols(exchange_name, environ=None):
     Object
 
     """
-    filename = get_exchange_symbols_filename(exchange_name)
+    filename = get_exchange_symbols_filename(exchange_name, is_local)
 
-    if not os.path.isfile(filename) or \
-                    pd.Timedelta(pd.Timestamp('now',
-                                              tz='UTC') - last_modified_time(
-                        filename)).days > 1:
+    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
+                pd.Timestamp('now', tz='UTC') - last_modified_time(
+                    filename)).days > 1):
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
+            try:
+                data = json.load(data_file, object_hook=symbols_parser)
+                return data
+
+            except ValueError:
+                return dict()
     else:
         raise ExchangeSymbolsNotFound(
             exchange=exchange_name,
@@ -113,6 +152,32 @@ def get_exchange_symbols(exchange_name, environ=None):
         )
 
 
+def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
+    """
+    Save assets into an exchange_symbols file.
+
+    Parameters
+    ----------
+    exchange_name: str
+    assets: list[dict[str, object]]
+    is_local: bool
+    environ
+
+    Returns
+    -------
+
+    """
+    asset_dicts = dict()
+    for symbol in assets:
+        asset_dicts[symbol] = assets[symbol].to_dict()
+
+    filename = get_exchange_symbols_filename(
+        exchange_name, is_local, environ
+    )
+    with open(filename, 'wt') as handle:
+        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
+
+
 def get_symbols_string(assets):
     """
     A concatenated string of symbols from a list of assets.
@@ -231,7 +296,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
         try:
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception as e:
+        except Exception:
             return None
     else:
         return None
@@ -363,6 +428,25 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
+def symbols_serial(obj):
+    """
+    JSON serializer for objects not serializable by default json code
+
+    Parameters
+    ----------
+    obj: Object
+
+    Returns
+    -------
+    str
+
+    """
+    if isinstance(obj, (datetime, date)):
+        return obj.floor('1D').strftime(DATE_FORMAT)
+
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
 def perf_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -500,4 +584,65 @@ def resample_history_df(df, freq, field):
     else:
         raise ValueError('Invalid field.')
 
-    return df.resample(freq).agg(agg)
+    resampled_df = df.resample(freq).agg(agg)
+    return resampled_df
+
+
+def mixin_market_params(exchange_name, params, market):
+    """
+    Applies a CCXT market dict to parameters of TradingPair init.
+
+    Parameters
+    ----------
+    params: dict[Object]
+    market: dict[Object]
+
+    Returns
+    -------
+
+    """
+    # TODO: make this more externalized / configurable
+    if 'lot' in market:
+        params['min_trade_size'] = market['lot']
+        params['lot'] = market['lot']
+
+    if exchange_name == 'bitfinex':
+        params['maker'] = 0.001
+        params['taker'] = 0.002
+
+    elif 'maker' in market and 'taker' in market \
+            and market['maker'] is not None and market['taker'] is not None:
+        params['maker'] = market['maker']
+        params['taker'] = market['taker']
+
+    else:
+        # TODO: default commission, make configurable
+        params['maker'] = 0.0015
+        params['taker'] = 0.0025
+
+    info = market['info'] if 'info' in market else None
+    if info:
+        if 'minimum_order_size' in info:
+            params['min_trade_size'] = float(info['minimum_order_size'])
+
+            if 'lot' not in params:
+                params['lot'] = params['min_trade_size']
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
+
+
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)
+
+
+def group_assets_by_exchange(assets):
+    exchange_assets = dict()
+    for asset in assets:
+        if asset.exchange not in exchange_assets:
+            exchange_assets[asset.exchange] = list()
+
+        exchange_assets[asset.exchange].append(asset)
+
+    return exchange_assets

catalyst/exchange/factory.py

@@ -1,38 +1,29 @@
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.exchange_errors import ExchangeNotFoundError
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.exchange.poloniex.poloniex import Poloniex
+import os
+
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_exchange_folder
 
 
-def get_exchange(exchange_name, base_currency=None):
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
     exchange_auth = get_exchange_auth(exchange_name)
-    if exchange_name == 'bitfinex':
-        return Bitfinex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
         )
 
-    elif exchange_name == 'bittrex':
-        return Bittrex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
-        )
-
-    elif exchange_name == 'poloniex':
-        return Poloniex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
-        )
-
-    else:
-        raise ExchangeNotFoundError(exchange_name=exchange_name)
+    return CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+    )
 
 
 def get_exchanges(exchange_names):

catalyst/exchange/poloniex/poloniex.py

@@ -1,5 +1,4 @@
 import json
-import json
 import time
 from collections import defaultdict
 
@@ -18,7 +17,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrphanOrderReverseError)
+    InvalidOrderStyle,
+    OrphanOrderError,
+    OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
@@ -27,16 +28,23 @@ from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 from catalyst.protocol import Account
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('Poloniex', level=LOG_LEVEL)
 
 
+@deprecated
 class Poloniex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
         self.api = Poloniex_api(key=key, secret=secret)
         self.name = 'poloniex'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -47,7 +55,7 @@ class Poloniex(Exchange):
         self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def sanitize_curency_symbol(self, exchange_symbol):
         """
@@ -82,7 +90,6 @@ class Poloniex(Exchange):
             # filled = -filled
 
         price = float(order_status['rate'])
-        order_type = order_status['type']
 
         stop_price = None
         limit_price = None
@@ -96,11 +103,11 @@ class Poloniex(Exchange):
         # executed_price = float(order_status['avg_execution_price'])
         executed_price = price
 
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
+        # TODO: Set Poloniex comission
         commission = None
 
-        # date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date = pytz.utc.localize(date)
+        # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
+        # date=pytz.utc.localize(date)
         date = None
 
         order = Order(
@@ -226,10 +233,9 @@ class Poloniex(Exchange):
         ohlc_map = dict()
 
         for asset in asset_list:
+            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
+            end = int(delta.total_seconds())
 
-            # TODO: what's wrong with this?
-            # end = int(time.mktime(end_dt.timetuple()))
-            end = int(time.time())
             if bar_count is None:
                 start = end - 2 * frequency
             else:
@@ -288,8 +294,8 @@ class Poloniex(Exchange):
         """
         exchange_symbol = self.get_symbol(asset)
 
-        if isinstance(style, ExchangeLimitOrder) or isinstance(style,
-                                                               ExchangeStopLimitOrder):
+        if (isinstance(style, ExchangeLimitOrder)
+                or isinstance(style, ExchangeStopLimitOrder)):
             if isinstance(style, ExchangeStopLimitOrder):
                 log.warn('{} will ignore the stop price'.format(self.name))
 
@@ -346,8 +352,8 @@ class Poloniex(Exchange):
         return self.portfolio.open_orders
 
         """
-            TODO: Why going to the exchange if we already have this info locally?
-                  And why creating all these Orders if we later discard them?
+        TODO: Why going to the exchange if we already have this info locally?
+              And why creating all these Orders if we later discard them?
         """
 
         try:
@@ -361,7 +367,7 @@ class Poloniex(Exchange):
         if 'error' in response:
             raise ExchangeRequestError(
                 error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
+                    response['message'])
             )
 
         print(self.portfolio.open_orders)
@@ -369,8 +375,8 @@ class Poloniex(Exchange):
         # TODO: Need to handle openOrders for 'all'
         orders = list()
         for order_status in response:
-            order, executed_price = self._create_order(
-                order_status)  # will Throw error b/c Polo doesn't track order['symbol']
+            # will Throw error b/c Polo doesn't track order['symbol']
+            order, executed_price = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 
@@ -433,7 +439,8 @@ class Poloniex(Exchange):
 
         if 'error' in response:
             log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
+                'Unable to cancel order {order_id} on exchange {exchange} '
+                '{error}.'.format(
                     order_id=order.id,
                     exchange=self.name,
                     error=response['error']
@@ -508,17 +515,17 @@ class Poloniex(Exchange):
             else:
                 try:
                     start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError as e:
+                except KeyError:
                     start_date = time.strftime('%Y-%m-%d')
 
             try:
                 end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
+            except KeyError:
                 end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
+            except KeyError:
                 end_minute = 'N/A'
 
             symbol_map[exchange_symbol] = dict(
@@ -589,19 +596,21 @@ class Poloniex(Exchange):
                 else:
                     for tx in response:
                         """
-                            We maintain a list of dictionaries of transactions that correspond to
-                            partially filled orders, indexed by order_id. Every time we query 
-                            executed transactions from the exchange, we check if we had that 
-                            transaction for that order already. If not, we process it.
+                        We maintain a list of dictionaries of transactions that
+                        correspond to partially filled orders, indexed by
+                        order_id. Every time we query executed transactions
+                        from the exchange, we check if we had that transaction
+                        for that order already. If not, we process it.
 
-                            When an order if fully filled, we flush the dict of transactions  
-                            associated with that order.
+                        When an order if fully filled, we flush the dict of
+                        transactions associated with that order.
                         """
                         if (not filter(
                                 lambda item: item['order_id'] == tx['tradeID'],
                                 self.transactions[order_id])):
                             log.debug(
-                                'Got new transaction for order {}: amount {}, price {}'.format(
+                                'Got new transaction for order {}: amount {}, '
+                                'price {}'.format(
                                     order_id, tx['amount'], tx['rate']))
                             tx['amount'] = float(tx['amount'])
                             if (tx['type'] == 'sell'):
@@ -612,7 +621,7 @@ class Poloniex(Exchange):
                                 dt=pd.to_datetime(tx['date'], utc=True),
                                 price=float(tx['rate']),
                                 order_id=tx['tradeID'],
-                                # it's a misnomer, but keeping it for compatibility
+                                # it's a misnomer, but keep for compatibility
                                 commission=float(tx['fee'])
                             )
                             self.transactions[order_id].append(transaction)
@@ -622,7 +631,8 @@ class Poloniex(Exchange):
                     if (not order_open):
                         """
                             Since transactions have been executed individually
-                            the only thing left to do is remove them from list of open_orders
+                            the only thing left to do is remove them from list
+                            of open_orders
                         """
                         del self.portfolio.open_orders[order_id]
                         del self.transactions[order_id]

catalyst/exchange/poloniex/poloniex_api.py

@@ -107,8 +107,9 @@ class Poloniex_api(object):
             data=post_data,
             headers=headers,
         )
-        return json.loads(
-            urlopen(req, context=ssl._create_unverified_context()).read())
+        resource = urlopen(req, context=ssl._create_unverified_context())
+        content = resource.read().decode('utf-8')
+        return json.loads(content)
 
     def returnticker(self):
         return self.query('returnTicker', {})
@@ -160,10 +161,6 @@ class Poloniex_api(object):
     def returnopenorders(self, market):
         return self.query('returnOpenOrders', {'currencyPair': market})
 
-    def returntradehistory(self, market):
-        # TODO: optional start and/or end and limit
-        return self.query('returnTradeHistory', {'currencyPair': market})
-
     def returnordertrades(self, ordernumber):
         return self.query('returnOrderTrades', {'orderNumber': ordernumber})
 
@@ -176,7 +173,7 @@ class Poloniex_api(object):
         elif (immediateorcancel):
             return self.query('buy', {'currencyPair': market, 'rate': rate,
                                       'amount': amount,
-                                      'immediateOrCancel': immediateorcancel, })
+                                      'immediateOrCancel': immediateorcancel})
         elif (postonly):
             return self.query('buy', {'currencyPair': market, 'rate': rate,
                                       'amount': amount,
@@ -194,7 +191,7 @@ class Poloniex_api(object):
         elif (immediateorcancel):
             return self.query('sell', {'currencyPair': market, 'rate': rate,
                                        'amount': amount,
-                                       'immediateOrCancel': immediateorcancel, })
+                                       'immediateOrCancel': immediateorcancel})
         elif (postonly):
             return self.query('sell', {'currencyPair': market, 'rate': rate,
                                        'amount': amount,

catalyst/exchange/simple_clock.py

@@ -31,7 +31,8 @@ class SimpleClock(object):
     This class is a drop-in replacement for
     :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
 
-    This is a stripped down version because crypto exchanges run around the clock.
+    This is a stripped down version because crypto exchanges run
+    around the clock.
 
     The :param:`time_skew` parameter represents the time difference between
     the Broker and the live trading machine's clock.

catalyst/exchange/stats_utils.py

@@ -1,7 +1,18 @@
+import csv
 import numbers
 
+import copy
 import numpy as np
+import os
 import pandas as pd
+import boto3
+import time
+
+from catalyst.assets._assets import TradingPair
+
+from catalyst.exchange.exchange_utils import get_algo_folder
+
+s3 = boto3.resource('s3')
 
 
 def trend_direction(series):
@@ -30,14 +41,25 @@ def crossover(source, target):
     bool
 
     """
-    if source[-1] is np.nan or source[-2] is np.nan \
-            or target[-1] is np.nan or target[-2] is np.nan:
-        return False
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] >= target > source[-2]:
+            return True
+        else:
+            return False
 
-    if source[-1] > target[-1] and source[-2] < target[-2]:
-        return True
     else:
-        return False
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] > target[-1] and source[-2] < target[-2]:
+            return True
+        else:
+            return False
 
 
 def crossunder(source, target):
@@ -108,62 +130,256 @@ def vwap(df):
     return ret
 
 
-def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
+def set_position_row(row, asset, asset_values=list()):
+    """
+    Apply the position data as individual columns.
+
+    Parameters
+    ----------
+    row: dict[str, Object]
+    asset: TradingPair
+    asset_values: list[str]
+        If a recorded_col contains a tuple which first value is an asset
+        matching a position, its value will be displayed with the
+        position and not in the index.
+
+    Returns
+    -------
+
+    """
+    asset_cols = ['symbol']
+    row['symbol'] = asset.symbol
+
+    position = next((p for p in row['positions'] if p['sid'] == asset), None)
+
+    columns = ['amount', 'cost_basis', 'last_sale_price']
+    for column in columns:
+        if position is not None:
+            row[column] = position[column]
+
+        else:
+            row[column] = 0
+
+        asset_cols.append(column)
+
+    values = asset_values[asset] if asset in asset_values else list()
+    for column in values:
+        row[column] = values[column]
+
+        asset_cols.append(column)
+
+    return asset_cols
+
+
+def prepare_stats(stats, recorded_cols=list()):
+    """
+    Prepare the stats DataFrame for user-friendly output.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    asset_cols = list()
+
+    stats = copy.deepcopy(stats)
+    # Using a copy since we are adding rows inside the loop.
+    for row_index, row_data in enumerate(list(stats)):
+        assets = [p['sid'] for p in row_data['positions']]
+
+        asset_values = dict()
+        if recorded_cols is not None:
+            for column in recorded_cols[:]:
+                value = row_data[column]
+                if type(value) is dict:
+                    for asset in value:
+                        if not isinstance(asset, TradingPair):
+                            break
+
+                        if asset not in assets:
+                            assets.append(asset)
+
+                        if asset not in asset_values:
+                            asset_values[asset] = dict()
+
+                        asset_values[asset][column] = value[asset]
+
+        if len(assets) == 1:
+            row = stats[row_index]
+            asset_cols = set_position_row(row, assets[0], asset_values)
+
+        elif len(assets) > 1:
+            for asset_index, asset in enumerate(assets):
+                if asset_index > 0:
+                    row = copy.deepcopy(row_data)
+                    stats.append(row)
+
+                else:
+                    row = stats[row_index]
+
+                asset_cols = set_position_row(row, assets[asset_index],
+                                              asset_values)
+
+    df = pd.DataFrame(stats)
+
+    index_cols = [
+        'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
+        'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
+    ]
+
+    # Removing the asset specific entries
+    if recorded_cols is not None:
+        recorded_cols = [x for x in recorded_cols if x not in asset_cols]
+        for column in recorded_cols:
+            index_cols.append(column)
+
+    df['orders'] = df['orders'].apply(lambda orders: len(orders))
+    df['transactions'] = df['transactions'].apply(
+        lambda transactions: len(transactions)
+    )
+
+    if asset_cols:
+        columns = asset_cols
+        df.set_index(index_cols, drop=True, inplace=True)
+
+    else:
+        columns = index_cols
+        columns.remove('period_close')
+        df.set_index('period_close', drop=False, inplace=True)
+
+    df.dropna(axis=1, how='all', inplace=True)
+    df.sort_index(axis=0, level=0, inplace=True)
+
+    return df, columns
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
 
     Parameters
     ----------
-    stats_df: DataFrame
+    stats: list[Object]
+        An array of statistics for the period.
+
     num_rows: int
+        The number of rows to display on the screen.
 
     Returns
     -------
     str
 
     """
-    stats_df.set_index('period_close', drop=True, inplace=True)
-    stats_df.dropna(axis=1, how='all', inplace=True)
+    if isinstance(stats, pd.DataFrame):
+        stats = stats.T.to_dict().values()
+
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
 
     pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 3)
+    pd.set_option('precision', 8)
     pd.set_option('display.width', 1000)
     pd.set_option('display.max_colwidth', 1000)
 
-    columns = ['starting_cash', 'ending_cash', 'portfolio_value',
-               'pnl', 'long_exposure', 'short_exposure', 'orders',
-               'transactions', 'positions']
-
-    if recorded_cols is not None:
-        for column in recorded_cols:
-            columns.append(column)
-
-    def format_positions(positions):
-        parts = []
-        for position in positions:
-            msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
-                amount=position['amount'],
-                market=position['sid'].market_currency,
-                cost_basis=position['cost_basis'],
-                base=position['sid'].base_currency
-            )
-            parts.append(msg)
-        return ', '.join(parts)
-
     formatters = {
-        'orders': lambda orders: len(orders),
-        'transactions': lambda transactions: len(transactions),
         'returns': lambda returns: "{0:.4f}".format(returns),
-        'positions': format_positions
     }
 
-    return stats_df.tail(num_rows).to_string(
+    return df.tail(num_rows).to_string(
         columns=columns,
         formatters=formatters
     )
 
 
+def get_csv_stats(stats, recorded_cols=None):
+    """
+    Create a CSV buffer from the stats DataFrame.
+
+    Parameters
+    ----------
+    path: str
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+
+    return df.to_csv(
+        None,
+        columns=columns,
+        # encoding='utf-8',
+        quoting=csv.QUOTE_NONNUMERIC
+    ).encode()
+
+
+def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
+                folder='catalyst/stats', bytes_to_write=None):
+    """
+    Uploads the performance stats to a S3 bucket.
+
+    Parameters
+    ----------
+    uri: str
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+    folder: str
+    bytes_to_write: str
+        Option to reuse bytes instead of re-computing the csv
+
+    Returns
+    -------
+
+    """
+    if bytes_to_write is None:
+        bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    now = pd.Timestamp.utcnow()
+    timestr = now.strftime('%Y%m%d')
+    pid = os.getpid()
+
+    parts = uri.split('//')
+    obj = s3.Object(parts[1], '{}/{}-{}-{}.csv'.format(
+        folder, timestr, algo_namespace, pid
+    ))
+    obj.put(Body=bytes_to_write)
+
+
+def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
+    """
+    Saves the performance stats to the algo local folder.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+
+    Returns
+    -------
+    str
+
+    """
+    bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    timestr = time.strftime('%Y%m%d')
+    folder = get_algo_folder(algo_namespace)
+
+    filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
+
+    with open(filename, 'wb') as handle:
+        handle.write(bytes_to_write)
+
+    return bytes_to_write
+
+
 def df_to_string(df):
     """
     Create a formatted str representation of the DataFrame.

catalyst/exchange/validator.py

@@ -0,0 +1,142 @@
+import os
+import tempfile
+
+import pandas as pd
+import six
+from catalyst.assets._assets import TradingPair, get_calendar
+from logbook import Logger
+from pandas.util.testing import assert_frame_equal
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
+from catalyst.exchange.factory import get_exchanges
+from catalyst.utils.paths import ensure_directory
+
+log = Logger('Validator', level=LOG_LEVEL)
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path
+
+
+class Validator(object):
+    def __init__(self, data_portal):
+        self.data_portal = data_portal
+
+    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
+                                     sample_minutes):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        sample_minutes
+
+        Returns
+        -------
+
+        """
+        freq = '{}T'.format(sample_minutes)
+
+        log.info('creating data sample from bundle')
+        df1 = self.data_portal.get_history_window(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            frequency=freq,
+            field='close',
+            data_frequency='minute'
+        )
+        path = output_df(df1, assets, '{}_resampled'.format(freq))
+        log.info('saved resampled bundle candles: {}\n{}'.format(
+            path, df1.tail(10))
+        )
+
+        log.info('creating data sample from exchange api')
+        candles = exchange.get_candles(
+            end_dt=end_dt,
+            freq='{}T'.format(sample_minutes),
+            assets=assets,
+            bar_count=bar_count
+        )
+
+        series = dict()
+        for asset in assets:
+            series[asset] = pd.Series(
+                data=[candle['close'] for candle in candles[asset]],
+                index=[candle['last_traded'] for candle in candles[asset]]
+            )
+
+        df2 = pd.DataFrame(series)
+        path = output_df(df2, assets, '{}_api'.format(freq))
+        log.info('saved exchange api candles: {}\n{}'.format(
+            path, df2.tail(10))
+        )
+
+        try:
+            assert_frame_equal(df1, df2)
+            return True
+        except:
+            log.warn('differences found in dataframes')
+            return False
+
+
+if __name__ == '__main__':
+    exchanges = get_exchanges(['poloniex'])
+    exchange = six.next(six.itervalues(exchanges))
+    assets = exchange.get_assets(symbols=['eth_btc'])
+
+    open_calendar = get_calendar('OPEN')
+    asset_finder = AssetFinderExchange()
+    data_portal = DataPortalExchangeBacktest(
+        exchanges=exchanges,
+        asset_finder=asset_finder,
+        trading_calendar=open_calendar,
+        first_trading_day=None  # will set dynamically based on assets
+    )
+    validator = Validator(data_portal=data_portal)
+
+    validator.compare_bundle_with_exchange(
+        exchange=exchange,
+        assets=assets,
+        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
+        bar_count=200,
+        sample_minutes=30
+    )

catalyst/finance/execution.py

@@ -15,13 +15,8 @@
 
 import abc
 
-from sys import float_info
-
-from six import with_metaclass
-
-import catalyst.utils.math_utils as zp_math
-
 from numpy import isfinite
+from six import with_metaclass
 
 from catalyst.errors import BadOrderParameters
 

catalyst/finance/risk/cumulative.py

@@ -45,8 +45,6 @@ log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
 
-warnings.filterwarnings('error')
-
 
 class RiskMetricsCumulative(object):
     """
@@ -146,6 +144,8 @@ class RiskMetricsCumulative(object):
         self.num_trading_days = 0
 
     def update(self, dt, algorithm_returns, benchmark_returns, leverage):
+        warnings.filterwarnings('error')
+
         # Keep track of latest dt for use in to_dict and other methods
         # that report current state.
         self.latest_dt = dt
@@ -196,8 +196,8 @@ class RiskMetricsCumulative(object):
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
-        except Exception as e:
-            log.debug('cumulative returns error: {}'.format(e))
+        except Exception:
+            self.benchmark_cumulative_returns[dt_loc] = 0
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -274,12 +274,14 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         )
 
         try:
+            risk = self.downside_risk[dt_loc]
             self.sortino[dt_loc] = sortino_ratio(
                 self.algorithm_returns,
-                _downside_risk=self.downside_risk[dt_loc]
+                _downside_risk=risk
             )
-        except Exception as e:
-            log.debug('sortino ratio error: {}'.format(e))
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino[dt_loc] = 0
 
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
@@ -292,6 +294,8 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/finance/risk/period.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import functools
+import warnings
 
 import logbook
 
@@ -23,7 +24,7 @@ import numpy as np
 import pandas as pd
 
 from . import risk
-from . risk import check_entry
+from .risk import check_entry
 
 from empyrical import (
     alpha_beta_aligned,
@@ -78,14 +79,20 @@ class RiskMetricsPeriod(object):
         self.calculate_metrics()
 
     def calculate_metrics(self):
-        self.benchmark_period_returns = \
-            cum_returns(self.benchmark_returns).iloc[-1]
+        warnings.filterwarnings('error')
+
+        try:
+            self.benchmark_period_returns = \
+                cum_returns(self.benchmark_returns).iloc[-1]
+        except Exception:
+            # TODO: why is there an error
+            self.benchmark_period_returns = 0
 
         self.algorithm_period_returns = \
             cum_returns(self.algorithm_returns).iloc[-1]
 
         if not self.algorithm_returns.index.equals(
-            self.benchmark_returns.index
+                self.benchmark_returns.index
         ):
             message = "Mismatch between benchmark_returns ({bm_count}) and \
             algorithm_returns ({algo_count}) in range {start} : {end}"
@@ -128,10 +135,17 @@ class RiskMetricsPeriod(object):
         self.downside_risk = downside_risk(
             self.algorithm_returns.values
         )
-        self.sortino = sortino_ratio(
-            self.algorithm_returns.values,
-            _downside_risk=self.downside_risk,
-        )
+
+        try:
+            risk = self.downside_risk
+            self.sortino = sortino_ratio(
+                self.algorithm_returns.values,
+                _downside_risk=risk,
+            )
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino = 0
+
         self.information = information_ratio(
             self.algorithm_returns.values,
             self.benchmark_returns.values,
@@ -140,11 +154,13 @@ class RiskMetricsPeriod(object):
             self.algorithm_returns.values,
             self.benchmark_returns.values,
         )
-        self.excess_return = self.algorithm_period_returns - \
-            self.treasury_period_return
+        self.excess_return = self.algorithm_period_returns \
+            - self.treasury_period_return
         self.max_drawdown = max_drawdown(self.algorithm_returns.values)
         self.max_leverage = self.calculate_max_leverage()
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/finance/risk/risk.py

@@ -160,7 +160,8 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
                 )
                 break
 
-        if search_day and trading_calendar.name != 'OPEN':          # Supress warning for 'OPEN' calendar
+        # Supress warning for 'OPEN' calendar
+        if search_day and trading_calendar.name != 'OPEN':
             if (search_dist is None or search_dist > 1) and \
                     search_days[0] <= end_session <= search_days[-1]:
                 message = "No rate within 1 trading day of end date = \

catalyst/finance/slippage.py

@@ -41,7 +41,6 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
 
 
-
 class LiquidityExceeded(Exception):
     pass
 

catalyst/pipeline/factors/equity/__init__.py

@@ -1,9 +1,6 @@
 from .statistical import (
-    RollingPearson,
-    RollingLinearRegression,
     RollingLinearRegressionOfReturns,
     RollingPearsonOfReturns,
-    RollingSpearman,
     RollingSpearmanOfReturns,
 )
 from .technical import (

catalyst/pipeline/loaders/equity_pricing_loader.py

@@ -38,9 +38,11 @@ class USEquityPricingLoader(PipelineLoader):
 
     def __init__(self, bundle, data_frequency, dataset):
 
-        if data_frequency == 'daily':
-            reader = bundle.daily_bar_reader
-        elif daily_bar_reader == 'minute':
+        # TODO: This is currently broken, No Pipeline support for Catalyst
+        # if data_frequency == 'daily':
+        #    reader = bundle.daily_bar_reader
+        # elif daily_bar_reader == 'minute':
+        if data_frequency == 'minute':
             reader = bundle.minute_bar_reader
         else:
             raise ValueError(
@@ -51,7 +53,9 @@ class USEquityPricingLoader(PipelineLoader):
 
         if data_frequency == 'daily':
             all_sessions = cal.all_sessions
-        elif daily_bar_reader == 'minute':
+        # TODO: this cannot be right, but no pipeline support at the moment
+        # elif daily_bar_reader == 'minute':
+        elif data_frequency == 'minute':
             reader = bundle.minute_bar_reader
             all_sessions = cal.all_minutes
 

catalyst/pipeline/loaders/events.py

@@ -231,7 +231,7 @@ class EventsLoader(PipelineLoader):
             self.load_next_events(n, dates, sids, mask),
             self.load_previous_events(p, dates, sids, mask),
         )
-    
+
     @property
     def columns(self):
         return self._columns

catalyst/pipeline/loaders/frame.py

@@ -180,4 +180,3 @@ class DataFrameLoader(PipelineLoader):
     @property
     def columns(self):
         return self._columns
-

catalyst/pipeline/loaders/synthetic.py

@@ -163,7 +163,7 @@ class SeededRandomLoader(PrecomputedLoader):
             bool_dtype: self._bool_values,
             object_dtype: self._object_values,
         }[dtype](shape)
-    
+
     @property
     def columns(self):
         return self._columns

catalyst/support/issue_44.py

@@ -1,109 +0,0 @@
-import pandas as pd
-from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-
-from catalyst.api import (
-    symbols,
-)
-
-
-def initialize(context):
-    context.i = -1
-    context.base_currency = 'btc'
-
-
-def handle_data(context, data):
-    lookback = 60 * 24 * 7  # (minutes, hours, days)
-    context.i += 1
-    if context.i < lookback:
-        return
-
-    today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
-
-    try:
-        # update universe everyday
-        new_day = 60 * 24
-        if not context.i % new_day:
-            context.universe = universe(context, today)
-
-        # get data every 30 minutes
-        minutes = 30
-        if not context.i % minutes and context.universe:
-            for coin in context.coins:
-                pair = str(coin.symbol)
-
-                # ohlcv data
-                open = data.history(coin, 'open', lookback,
-                                    '1m').ffill().bfill().resample(
-                    '30T').first()
-                high = data.history(coin, 'high', lookback,
-                                    '1m').ffill().bfill().resample('30T').max()
-                low = data.history(coin, 'low', lookback,
-                                   '1m').ffill().bfill().resample('30T').min()
-                close = data.history(coin, 'price', lookback,
-                                     '1m').ffill().bfill().resample(
-                    '30T').last()
-                volume = data.history(coin, 'volume', lookback,
-                                      '1m').ffill().bfill().resample(
-                    '30T').sum()
-
-                print(today, pair, close[-1])
-
-    except Exception as e:
-        print(e)
-
-
-def analyze(context=None, results=None):
-    pass
-
-
-def universe(context, today):
-    json_symbols = get_exchange_symbols('poloniex')
-    poloniex_universe_df = pd.DataFrame.from_dict(
-        json_symbols).transpose().astype(str)
-    poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
-        lambda row: row.symbol.split('_')[1],
-        axis=1)
-    poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
-        lambda row: row.symbol.split('_')[0],
-        axis=1)
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df['base_currency'] == context.base_currency]
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df.symbol != 'gas_btc']
-
-    # Markets currently not working on Catalyst 0.3.1
-    # 2017-01-01
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
-    print(poloniex_universe_df.head())
-
-    date = str(today).split(' ')[0]
-
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df.start_date < date]
-    context.coins = symbols(*poloniex_universe_df.symbol)
-    print(len(poloniex_universe_df))
-    return poloniex_universe_df.symbol.tolist()
-
-
-if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-01-01', utc=True)
-    end_date = pd.to_datetime('2017-10-15', utc=True)
-
-    performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=10000.0,
-                                initialize=initialize,
-                                handle_data=handle_data,
-                                analyze=analyze,
-                                exchange_name='poloniex',
-                                data_frequency='minute',
-                                base_currency='btc',
-                                live=False,
-                                live_graph=False,
-                                algo_namespace='test')

catalyst/support/issue_47.py

@@ -1,140 +0,0 @@
-"""
-Requires Catalyst version 0.3.0 or above
-Tested on Catalyst version 0.3.2
-
-These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
-You simply need to specify the exchange and the market that you want to focus on.
-You will all see how to create a universe and filter it base on the exchange and the market you desire.
-
-The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
-The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
-Use this as the backbone to create your own trading strategies.
-
-Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
-"""
-
-import numpy as np
-import pandas as pd
-from datetime import timedelta
-from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-
-from catalyst.api import (
-    symbols,
-)
-
-
-def initialize(context):
-    context.i = -1  # counts the minutes
-    context.exchange = 'poloniex'  # must match the exchange specified in run_algorithm
-    context.base_currency = 'eth'  # must match the base currency specified in run_algorithm
-
-
-def handle_data(context, data):
-    lookback = 60 * 24 * 7  # (minutes, hours, days) of how far to lookback in the data history
-    context.i += 1
-
-    # current date formatted into a string
-    today = context.blotter.current_dt
-    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-    lookback_date = today - timedelta(days=(
-    lookback / (60 * 24)))  # subtract the amount of days specified in lookback
-    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[
-        0]  # get only the date as a string
-
-    # update universe everyday
-    new_day = 60 * 24
-    if not context.i % new_day:
-        context.universe = universe(context, lookback_date, date)
-
-    # get data every 30 minutes
-    minutes = 30
-    if not context.i % minutes and context.universe:
-        # we iterate for every pair in the current universe
-        for coin in context.coins:
-            pair = str(coin.symbol)
-
-            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
-            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
-            open = fill(data.history(coin, 'open', bar_count=lookback,
-                                     frequency='1m')).resample('30T').first()
-            high = fill(data.history(coin, 'high', bar_count=lookback,
-                                     frequency='1m')).resample('30T').max()
-            low = fill(data.history(coin, 'low', bar_count=lookback,
-                                    frequency='1m')).resample('30T').min()
-            close = fill(data.history(coin, 'price', bar_count=lookback,
-                                      frequency='1m')).resample('30T').last()
-            volume = fill(data.history(coin, 'volume', bar_count=lookback,
-                                       frequency='1m')).resample('30T').sum()
-
-            # close[-1] is the equivalent to current price
-            # displays the minute price for each pair every 30 minutes
-            print(
-            today, pair, open[-1], high[-1], low[-1], close[-1], volume[-1])
-
-            # ----------------------------------------------------------------------------------------------------------
-            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
-            # ----------------------------------------------------------------------------------------------------------
-
-
-def analyze(context=None, results=None):
-    pass
-
-
-# Get the universe for a given exchange and a given base_currency market
-# Example: Poloniex BTC Market
-def universe(context, lookback_date, current_date):
-    json_symbols = get_exchange_symbols(
-        context.exchange)  # get all the pairs for the exchange
-    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(
-        str)  # convert into a dataframe
-    universe_df['base_currency'] = universe_df.apply(
-        lambda row: row.symbol.split('_')[1],
-        axis=1)
-    universe_df['market_currency'] = universe_df.apply(
-        lambda row: row.symbol.split('_')[0],
-        axis=1)
-    # Filter all the exchange pairs to only the ones for a give base currency
-    universe_df = universe_df[
-        universe_df['base_currency'] == context.base_currency]
-
-    # Filter all the pairs to ensure that pair existed in the current date range
-    universe_df = universe_df[universe_df.start_date < lookback_date]
-    universe_df = universe_df[universe_df.end_daily >= current_date]
-    context.coins = symbols(
-        *universe_df.symbol)  # convert all the pairs to symbols
-    print(universe_df.head(), len(universe_df))
-    return universe_df.symbol.tolist()
-
-
-# Replace all NA, NAN or infinite values with its nearest value
-def fill(series):
-    if isinstance(series, pd.Series):
-        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-    elif isinstance(series, np.ndarray):
-        return pd.Series(series).replace([np.inf, -np.inf],
-                                         np.nan).ffill().bfill().values
-    else:
-        return series
-
-
-if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-01-01', utc=True)
-    end_date = pd.to_datetime('2017-10-15', utc=True)
-
-    performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=10000.0,
-                                initialize=initialize,
-                                handle_data=handle_data,
-                                analyze=analyze,
-                                exchange_name='poloniex',
-                                data_frequency='minute',
-                                base_currency='eth',
-                                live=False,
-                                live_graph=False,
-                                algo_namespace='simple_universe')
-
-"""
-Run in Terminal (inside catalyst environment):
-python simple_universe.py
-"""

catalyst/support/issue_55.py

@@ -1,4 +1,3 @@
-import talib
 import pandas as pd
 
 from catalyst import run_algorithm

catalyst/support/issue_57.py

@@ -1,46 +0,0 @@
-import talib
-import pandas as pd
-
-from catalyst import run_algorithm
-from catalyst.api import symbol
-
-
-def initialize(context):
-    print('initializing')
-    context.asset = symbol('btc_usdt')
-
-
-def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
-
-    price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
-
-    try:
-        prices = data.history(
-            context.asset,
-            fields='close',
-            bar_count=60,
-            frequency='1D'
-        )
-        print('got {} price entries\n'.format(len(prices), prices))
-    except Exception as e:
-        print(e)
-
-
-run_algorithm(
-    capital_base=1,
-    start=pd.to_datetime('2016-2-11', utc=True),
-    end=pd.to_datetime('2017-8-31', utc=True),
-    data_frequency='daily',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=None,
-    exchange_name='bittrex',
-    algo_namespace='issue_57',
-    base_currency='btc'
-<<<<<<< HEAD
-)
-=======
-)
->>>>>>> develop

catalyst/support/rodrigo_1.py

@@ -1,153 +0,0 @@
-import pandas as pd
-from logbook import Logger, DEBUG
-
-from catalyst import run_algorithm
-from catalyst.api import (schedule_function, order_target_percent, symbol,
-                          date_rules, get_open_orders, cancel_order, record,
-                          set_commission, set_slippage)
-
-log = Logger('rodrigo_1', level=DEBUG)
-"""
-The initialize function sets any data or variables that 
-you'll use in your algorithm. 
-It's only called once at the beginning of your algorithm.
-"""
-
-
-def initialize(context):
-    # Select asset of interest
-    context.asset = symbol('BTC_USD')
-
-    # set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5))
-    # set_slippage(TradingPairFixedSlippage(spread=0.5))
-    # Set up a rebalance method to run every day
-    schedule_function(rebalance, date_rule=date_rules.every_day())
-
-
-"""
-Rebalance function scheduled to run once per day.
-"""
-
-
-def rebalance(context, data):
-    # To make market decisions, we're calculating the token's
-    # moving average for the last 5 days.
-
-    # We get the price history for the last 5 days.
-    price_history = data.history(context.asset, fields='price', bar_count=5,
-                                 frequency='1d')
-
-    # Then we take an average of those 5 days.
-    average_price = price_history.mean()
-
-    # We also get the coin's current price.
-    price = data.current(context.asset, 'price')
-
-    # Cancel any outstanding orders
-    orders = get_open_orders(context.asset) or []
-    for order in orders:
-        cancel_order(order)
-
-    # If our coin is currently listed on a major exchange
-    if data.can_trade(context.asset):
-        # If the current price is 1% above the 5-day average price,
-        # we open a long position. If the current price is below the
-        # average price, then we want to close our position to 0 shares.
-        if price > (1.01 * average_price):
-            # Place the buy order (positive means buy, negative means sell)
-            order_target_percent(context.asset, .99)
-            log.info("Buying %s" % (context.asset.symbol))
-        elif price < average_price:
-            # Sell all of our shares by setting the target position to zero
-            order_target_percent(context.asset, 0)
-            log.info("Selling %s" % (context.asset.symbol))
-
-    # Use the record() method to track up to five custom signals.
-    # Record Apple's current price and the average price over the last
-    # five days.
-    cash = context.portfolio.cash
-    leverage = context.account.leverage
-
-    record(price=price, average_price=average_price, cash=cash,
-           leverage=leverage)
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(511)
-    results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value (USD)')
-
-    ax2 = plt.subplot(512, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset))
-    (results[[
-        'price',
-    ]]).plot(ax=ax2)
-
-    trans = results.ix[[t != [] for t in results.transactions]]
-    buys = trans.ix[
-        [t[0]['amount'] > 0 for t in trans.transactions]
-    ]
-    sells = trans.ix[
-        [t[0]['amount'] < 0 for t in trans.transactions]
-    ]
-
-    ax2.plot(
-        buys.index,
-        results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        results.price[sells.index],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(513, sharex=ax1)
-    results[['leverage']].plot(ax=ax3)
-    ax3.set_ylabel('Leverage ')
-
-    ax4 = plt.subplot(514, sharex=ax1)
-    results[['cash']].plot(ax=ax4)
-    ax4.set_ylabel('Cash (USD)')
-
-    results[[
-        'algorithm',
-        'benchmark',
-    ]] = results[[
-        'algorithm_period_return',
-        'benchmark_period_return',
-    ]]
-
-    ax5 = plt.subplot(515, sharex=ax1)
-    results[[
-        'algorithm',
-        'benchmark',
-    ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-
-
-run_algorithm(
-    capital_base=100000,
-    start=pd.to_datetime('2017-1-1', utc=True),
-    end=pd.to_datetime('2017-10-22', utc=True),
-    data_frequency='minute',
-    initialize=initialize,
-    handle_data=None,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    algo_namespace='rodrigo_1',
-    base_currency='usd'
-)

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,5 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(
+            start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/cli.py

@@ -9,6 +9,7 @@ DEFAULT_BAR_TEMPLATE = '    [%(bar)s]  %(label)s:  %(info)s'
 DEFAULT_EMPTY_CHAR = ' '
 DEFAULT_FILL_CHAR = '='
 
+
 def item_show_count(total=None):
     def maybe_show_total(index):
         if total is not None:
@@ -17,12 +18,13 @@ def item_show_count(total=None):
 
     def item_show_func(item, _it=iter(count())):
         if item is not None:
-            starting = False
+            # starting = False
             return maybe_show_total(next(_it))
         return 'DONE'
 
     return item_show_func
 
+
 def maybe_show_progress(it,
                         show_progress,
                         empty_char=DEFAULT_EMPTY_CHAR,

catalyst/utils/math_utils.py

@@ -17,9 +17,11 @@ import math
 
 from numpy import isnan
 
+
 def round_nearest(x, a):
     return round(round(x / a) * a, -int(math.floor(math.log10(a))))
 
+
 def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False):
     """Check if a and b are equal with some tolerance.
 

catalyst/utils/paths.py

@@ -126,7 +126,7 @@ def catalyst_root(environ=None):
 
     root = environ.get('ZIPLINE_ROOT', None)
     if root is None:
-        root = os.path.join(expanduser('~'),'.catalyst')
+        root = os.path.join(expanduser('~'), '.catalyst')
 
     return root
 

catalyst/utils/run_algo.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 import warnings
 from datetime import timedelta
@@ -7,10 +8,11 @@ from time import sleep
 
 import click
 import pandas as pd
+from logbook import Logger
 
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.poloniex.poloniex import Poloniex
+from catalyst.data.bundles import load
+from catalyst.data.data_portal import DataPortal
+from catalyst.exchange.factory import get_exchange
 
 try:
     from pygments import highlight
@@ -29,19 +31,16 @@ from catalyst.utils.factory import create_simulation_parameters
 from catalyst.data.loader import load_crypto_market_data
 import catalyst.utils.paths as pth
 
-from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
-    ExchangeTradingAlgorithmBacktest
+from catalyst.exchange.exchange_algorithm import (
+    ExchangeTradingAlgorithmLive,
+    ExchangeTradingAlgorithmBacktest,
+)
 from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeAuthEmpty,
-    ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, ExchangeNotFoundError)
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object, get_exchange_folder
-from logbook import Logger
+    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
+    BaseCurrencyNotFoundError, NotEnoughCapitalError)
 
 from catalyst.constants import LOG_LEVEL
 
@@ -91,7 +90,9 @@ def _run(handle_data,
          exchange,
          algo_namespace,
          base_currency,
-         live_graph):
+         live_graph,
+         simulate_orders,
+         stats_output):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
@@ -140,7 +141,8 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
-    mode = 'live' if live else 'backtest'
+    mode = 'paper-trading' if simulate_orders else 'live-trading' \
+        if live else 'backtest'
     log.info('running algo in {mode} mode'.format(mode=mode))
 
     exchange_name = exchange
@@ -151,51 +153,12 @@ def _run(handle_data,
 
     exchanges = dict()
     for exchange_name in exchange_list:
-
-        # Looking for the portfolio from the cache first
-        portfolio = get_algo_object(
-            algo_name=algo_namespace,
-            key='portfolio_{}'.format(exchange_name),
-            environ=environ
+        exchanges[exchange_name] = get_exchange(
+            exchange_name=exchange_name,
+            base_currency=base_currency,
+            must_authenticate=(live and not simulate_orders),
         )
 
-        if portfolio is None:
-            portfolio = ExchangePortfolio(
-                start_date=pd.Timestamp.utcnow()
-            )
-
-        # This corresponds to the json file containing api token info
-        exchange_auth = get_exchange_auth(exchange_name)
-
-        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
-            raise ExchangeAuthEmpty(
-                    exchange=exchange_name.title(), 
-                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
-
-        if exchange_name == 'bitfinex':
-            exchanges[exchange_name] = Bitfinex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'bittrex':
-            exchanges[exchange_name] = Bittrex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'poloniex':
-            exchanges[exchange_name] = Poloniex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        else:
-            raise ExchangeNotFoundError(exchange_name=exchange_name)
-
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
@@ -210,7 +173,7 @@ def _run(handle_data,
         asset_db_path=None  # We don't need an asset db, we have exchanges
     )
     env.asset_finder = AssetFinderExchange()
-    choose_loader = None  # TODO: use the DataPortal for in the algorithm class for this
+    choose_loader = None  # TODO: use the DataPortal in the algo class for this
 
     if live:
         start = pd.Timestamp.utcnow()
@@ -258,17 +221,32 @@ def _run(handle_data,
                     )
 
             if base_currency in balances:
-                return balances[base_currency]
+                base_currency_available = balances[base_currency]['free']
+                log.info(
+                    'base currency available in the account: {} {}'.format(
+                        base_currency_available, base_currency
+                    )
+                )
+
+                return base_currency_available
             else:
                 raise BaseCurrencyNotFoundError(
                     base_currency=base_currency,
                     exchange=exchange_name
                 )
 
-        capital_base = 0
-        for exchange_name in exchanges:
-            exchange = exchanges[exchange_name]
-            capital_base += fetch_capital_base(exchange)
+        if not simulate_orders:
+            for exchange_name in exchanges:
+                exchange = exchanges[exchange_name]
+                balance = fetch_capital_base(exchange)
+
+                if balance < capital_base:
+                    raise NotEnoughCapitalError(
+                        exchange=exchange_name,
+                        base_currency=base_currency,
+                        balance=balance,
+                        capital_base=capital_base,
+                    )
 
         sim_params = create_simulation_parameters(
             start=start,
@@ -285,9 +263,11 @@ def _run(handle_data,
             ExchangeTradingAlgorithmLive,
             exchanges=exchanges,
             algo_namespace=algo_namespace,
-            live_graph=live_graph
+            live_graph=live_graph,
+            simulate_orders=simulate_orders,
+            stats_output=stats_output,
         )
-    else:
+    elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
         # We can add back the complexity if required.
 
@@ -297,7 +277,7 @@ def _run(handle_data,
         # can handle this later.
 
         data = DataPortalExchangeBacktest(
-            exchanges=exchanges,
+            exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,
             trading_calendar=open_calendar,
             first_trading_day=start,
@@ -317,6 +297,36 @@ def _run(handle_data,
             exchanges=exchanges
         )
 
+    elif bundle is not None:
+        bundle_data = load(
+            bundle,
+            environ,
+            bundle_timestamp,
+        )
+
+        prefix, connstr = re.split(
+            r'sqlite:///',
+            str(bundle_data.asset_finder.engine.url),
+            maxsplit=1,
+        )
+        if prefix:
+            raise ValueError(
+                "invalid url %r, must begin with 'sqlite:///'" %
+                str(bundle_data.asset_finder.engine.url),
+            )
+
+        env = TradingEnvironment(asset_db_path=connstr, environ=environ)
+        first_trading_day = \
+            bundle_data.equity_minute_bar_reader.first_trading_day
+
+        data = DataPortal(
+            env.asset_finder, open_calendar,
+            first_trading_day=first_trading_day,
+            equity_minute_reader=bundle_data.equity_minute_bar_reader,
+            equity_daily_reader=bundle_data.equity_daily_bar_reader,
+            adjustment_reader=bundle_data.adjustment_reader,
+        )
+
     perf = algorithm_class(
         namespace=namespace,
         env=env,
@@ -416,7 +426,10 @@ def run_algorithm(initialize,
                   exchange_name=None,
                   base_currency=None,
                   algo_namespace=None,
-                  live_graph=False):
+                  live_graph=False,
+                  simulate_orders=True,
+                  stats_output=None,
+                  output=os.devnull):
     """Run a trading algorithm.
 
     Parameters
@@ -486,8 +499,18 @@ def run_algorithm(initialize,
     --------
     catalyst.data.bundles.bundles : The available data bundles.
     """
-    load_extensions(default_extension, extensions, strict_extensions, environ)
+    load_extensions(
+        default_extension, extensions, strict_extensions, environ
+    )
 
+    if capital_base is None:
+        raise ValueError(
+            'Please specify a `capital_base` parameter which is the maximum '
+            'amount of base currency available for trading. For example, '
+            'if the `capital_base` is 5ETH, the '
+            '`order_target_percent(asset, 1)` command will order 5ETH worth '
+            'of the specified asset.'
+        )
     # I'm not sure that we need this since the modified DataPortal
     # does not require extensions to be explicitly loaded.
 
@@ -527,7 +550,7 @@ def run_algorithm(initialize,
         bundle_timestamp=bundle_timestamp,
         start=start,
         end=end,
-        output=os.devnull,
+        output=output,
         print_algo=False,
         local_namespace=False,
         environ=environ,
@@ -535,5 +558,7 @@ def run_algorithm(initialize,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        simulate_orders=simulate_orders,
+        stats_output=stats_output
     )

docs/source/features.rst

@@ -1,5 +1,61 @@
+Features
+========
+
+This page describes the features that Catalyst provides in the current version,
+and what is planned for future releases.
+
+Current Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+* Backtesting and live-trading modes to run your trading algorithms, with a 
+  seamless transition between the two.
+* Paper trading simulates order in live-trading mode.
+* Support for 3 exchanges: Bitfinex, Bittrex and Poloniex in both modes 
+  (backtesting and live-trading). Historical data for backtesting is provided 
+  with daily resolution for all three exchanges, and minute resolution for 
+  Bitfinex and Poloniex. No minute-resolution data is currently available for 
+  Bittrex. Refer to 
+  `Catalyst Market Coverage <https://www.enigma.co/catalyst/status>`_ for 
+  details.
+* Interface with over 90 exchanges available in live and paper trading modes.
+* Granular commission models which closely simulates each exchange fee
+  structure in backtesting and paper trading.
+* Standardized naming convention for all asset pairs trading on any exchange in 
+  the form ``{market_currency}_{base_currency}``. See 
+  :ref:`naming`.
+* Output of performance statistics based on Pandas DataFrames to integrate 
+  nicely into the existing PyData ecosystem.
+* Support for accessing multiple exchanges per algorithm, which opens the door
+  to cross-exchange arbitrage opportunities.
+* Support for running multiple algorithms on the same exchange independently of
+  one another. Catalyst performance tracker stores just enough data to allow 
+  algorithms to run independently while still sharing critical data through 
+  exchanges.
+* Benchmark defaults to Bitcoin price (btc_usdt in Poloniex exchange) for the 
+  purpose of comparing performance across trading algorithms. A custom benchmark
+  can be specified through ``set_benchmark()`` (but see 
+  `issue #86 <https://github.com/enigmampc/catalyst/issues/86>`_). 
+* Support for MacOS, Linux and Windows installations.
+* Support for Python2 and Python3.
+
+For additional details on the functionality added on recent releases, see the
+:doc:`Release Notes<releases>`.
+
+Upcoming features
+~~~~~~~~~~~~~~~~~
+
+* Additional datasets beyond pricing data (Dec. 2017)
+* API documentation (Jan. 2017)
+* Support for decentralized exchanges (Jan. 2017)
+* Support for data ingestion of community-contributed data sets (Jan. 2017)
+* Pipeline support (Jan. 2018)
+* Web UI (Q2 2018)
+
+
+ .. _naming:
+
 Naming Convention
-=================
+~~~~~~~~~~~~~~~~~
 
 Catalyst introduces a standardized naming convention for all asset pairs 
 trading on any exchange in the following form:

docs/source/index.rst

@@ -1,4 +1,4 @@
-.. include:: welcome.rst
+.. include:: ../../README.rst
 |
 |
 Table of Contents
@@ -9,15 +9,15 @@ Table of Contents
 
    install
    beginner-tutorial
-   jupyter
    live-trading
-   naming-convention
+   features
+   example-algos
+   utilities
    videos
    resources
    development-guidelines
    releases
 ..   bundles
-..   development-guidelines
 ..   appendix
 ..   release-process
 

docs/source/install.rst

@@ -6,7 +6,154 @@ Like any other piece of software, Catalyst has a number of dependencies
 (other software on which it depends to run) that you will need to install, as 
 well. We recommend using a software named ``Conda`` that will manage all 
 these dependencies for you, and set up the environment needed to get you up 
-and running as easily as possible. See :ref:`Installing with Conda <conda>`.
+and running as easily as possible. This is the recommended installation method
+for Windows, MacOS and Linux. See :ref:`Installing with Conda <conda>`.
+
+What conda does is create a pre-configured environment, and inside that 
+environment install Catalyst using ``pip``, Python's package manager. Thus, 
+as an alternative installation method for MacOS and Linux, you can install 
+Catalyst directly with ``pip`` (we recommend in combination with a virtual 
+environemnt). See :ref:`Installing with pip <pip>`.
+
+Regardless of the method, each operating system (OS), has its own 
+prerequisites, make sure to review the corresponding sections for your system:
+:ref:`Linux <linux>`, :ref:`MacOS <macos>` and :ref:`Windows <windows>`.
+
+.. _conda:
+
+Installing with ``conda``
+-------------------------
+
+The preferred method to install Catalyst is via the ``conda`` package manager, 
+which comes as part of Continuum Analytics' `Anaconda
+<http://continuum.io/downloads>`_ distribution.
+
+The primary advantage of using Conda over ``pip`` is that conda natively
+understands the complex binary dependencies of packages like ``numpy`` and
+``scipy``.  This means that ``conda`` can install Catalyst and its 
+dependencies without requiring the use of a second tool to acquire Catalyst's 
+non-Python dependencies.
+
+  For Windows, you will first need to install the *Microsoft Visual C++ 
+  Compiler for Python 2.7*. Follow the instructions on the :ref:`Windows 
+  <windows>` section and come back here.
+
+For instructions on how to install ``conda``, see the `Conda Installation
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
+you can install MiniConda, which is a smaller footprint (fewer packages and 
+smaller size) than its big brother Anaconda, but it still contains all the 
+main packages needed. To install MiniConda, you can follow these steps:
+
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
+   for your Operating System.
+2. Install MiniConda. See the `Installation Instructions 
+   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal 
+   window, which should print the list of packages installed with Conda.
+
+  For Windows, if you accepted the default installation options, you didn't 
+  check an option to add Conda to the PATH, so trying to run ``conda`` from
+  a regular ``Command Prompt`` will result in the following error: ``'conda' 
+  is no recognized as an internal or external command, operatble program or 
+  batch file``. That's to be expected. You will nee to launch an ``Anaconda 
+  Prompt`` that was added at installation time to your list of programs 
+  available from the Start menu. 
+
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+
+     To download, simply click on the 'Raw' button and save the file locally 
+     to a folder you can remember. Make sure that the file gets saved with the
+     ``.yml`` extension, and nothing like a ``.txt`` file or anything else.
+
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
+   saved the above ``python2.7-environment.yml`` file.
+
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new 
+   session to run Catalyst):
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+5. Verify that Catalyst is install correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
+above failed for any reason, you can try setting up the environment manually 
+with the following steps:
+
+1. If the above installation failed, and you have a partially set up catalyst
+   environment, remove it first. If you are starting from scratch, proceed to 
+   step #2:
+
+   .. code-block:: bash
+
+      conda env remove --name catalyst
+
+2. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy zlib
+
+3. Activate the environment:
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+4. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+5. Verify that Catalyst is installed correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst properly installed.
+
+.. _pip:
 
 Installing with ``pip``
 -----------------------
@@ -28,15 +175,21 @@ Because LAPACK and the CPython headers are non-Python dependencies, the
 correctway to install them varies from platform to platform.  If you'd rather 
 use a single tool to install Python and non-Python dependencies, or if you're 
 already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
-distribution, you can skip to the :ref:`Installing with Conda <conda>` 
-section.
+distribution, refer to the :ref:`Installing with Conda <conda>` section.
 
-Once you've installed the necessary additional dependencies (see below for
-your particular platform), you should be able to simply run
+Once you've installed the necessary additional dependencies for your system 
+(see below for your particular platform: :ref:`Linux`, :ref:`MacOS` or 
+:ref:`Windows`), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install enigma-catalyst
+   $ pip install enigma-catalyst matplotlib
+
+Note that in the command above we install two different packages. The second 
+one, ``matplotlib`` is a visualization library. While it's not strictly 
+required to run catalyst simulations or live trading, it comes in very handy
+to visualize the performance of your algorithms, and for this reason we 
+recommend you install it, as well.
 
 If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
@@ -50,153 +203,7 @@ summarized version:
    $ pip install virtualenv
    $ virtualenv catalyst-venv
    $ source ./catalyst-venv/bin/activate
-   $ pip install enigma-catalyst
-
-Though not required by Catalyst directly, our example algorithms use 
-matplotlib to visually display the results of the trading algorithms. If you 
-wish to run any examples or use matplotlib during development, it can be 
-installed using:
-
-.. code-block:: bash
-
-    $ pip install matplotlib
-
-GNU/Linux
-~~~~~~~~~
-
-On `Debian-derived`_ Linux distributions, you can acquire all the necessary
-binary dependencies from ``apt`` by running:
-
-.. code-block:: bash
-
-   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
-
-On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
-following should be sufficient to acquire the necessary additional
-dependencies:
-
-.. code-block:: bash
-
-   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
-
-On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
-
-.. code-block:: bash
-
-   $ pacman -S lapack gcc gcc-fortran pkg-config
-
-.. Commenting it out until Catalyst fully supports Python 3.X
-..
-.. There are also AUR packages available for installing `Python 3.4
-.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
-.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
-.. Python 2 is also installable via:
-
-.. 
-
-..   $ pacman -S python2
-
-OSX
-~~~
-
-The version of Python shipped with OSX by default is generally out of date, 
-and has a number of quirks because it's used directly by the operating system.
-For these reasons, many developers choose to install and use a separate Python
-installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
-to `Installing Python on OSX <http://docs.python-guide.org/en/latest/>`_, 
-which explains how to install Python with the `Homebrew`_ manager.
-
-Assuming you've installed Python with Homebrew, you'll also likely need the
-following brew packages:
-
-.. code-block:: bash
-
-   $ brew install freetype pkg-config gcc openssl
-
-OSX + virtualenv + matplotlib
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-A note about using matplotlib in virtual enviroments on OSX: it may be 
-necessary to run
-
-.. code-block:: bash
-
-   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
-
-in order to override the default ``macosx`` backend for your system, which 
-may not be accessible from inside the virtual environment. This will allow 
-Catalyst to open matplotlib charts from within a virtual environment, which 
-is useful for displaying the performance of your backtests.  To learn more 
-about matplotlib backends, please refer to the
-`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
-
-.. _windows:
-
-Windows
-~~~~~~~
-
-In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
-<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
-package contains the compiler and the set of system headers necessary for 
-producing binary wheels for Python 2.7 packages. If it's not already in your 
-system, download it and install it before proceeding to the next step.
-
-For windows, the easiest and best supported way to install Catalyst is to use
-:ref:`Conda <conda>`.
-
-Some problems we have encountered installing the **Visual C++ Compiler** 
-mentioned above are as follows:
-
-- **The system administrator has set policies to prevent this installation**.
-  
-  In some systems, there is a default *Windows Software Restriction* policy 
-  that prevents the installation of some software packages like this one. 
-  You'll have to change the Registry to circumvent this:
-
-  - Click ``Start``, and search for ``regedit`` and launch the 
-    ``Registry Editor``
-  - Navigate to the following folder:
-    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
-  - If there is an entry for ``DisableMSI``, set the Value data to 0.
-  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
-    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
-    default you get 0 as the Value Data)
-
-|
-- **The installer has encountered an unexpected error installing this package. 
-  This may indicate a problem with this package. The error code is 2503.**
-
-  We have observed this when trying to install a package without enough 
-  administrator permissions. Even when you are logged in as an Administrator, 
-  you have to explictily install this package with administrator privileges:
-
-  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
-  - Right click on it and choose ``Run as administrator``
-  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
-  - Run ``msiexec /i VCForPython27.msi``
-
-
-Amazon Linux AMI
-~~~~~~~~~~~~~~~~
-
-The packages ``pip`` and ``setuptools`` that come shipped by default are very 
-outdated. Thus, you first need to run:
-
-.. code-block:: bash
-
-   pip install --upgrade pip setuptools
-
-The default installation is also missing the C and C++ compilers, which you 
-install by:
-
-.. code-block:: bash
-
-   sudo yum install gcc gcc-c++
-
-Then you should follow the regular installation instructions outlined at the 
-beginning of this page.
-
+   $ pip install enigma-catalyst matplotlib
 
 Troubleshooting ``pip`` Install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -287,99 +294,150 @@ Troubleshooting ``pip`` Install
       sudo apt-get install python-dev
 
 
-.. _conda:
+.. _linux:
 
-Installing with ``conda``
--------------------------
+GNU/Linux Requirements
+----------------------
 
-Another way to install Catalyst is via the ``conda`` package manager, which
-comes as part of Continuum Analytics' `Anaconda
-<http://continuum.io/downloads>`_ distribution.
+On `Debian-derived`_ Linux distributions, you can acquire all the necessary
+binary dependencies from ``apt`` by running:
 
-The primary advantage of using Conda over ``pip`` is that conda natively
-understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Catalyst and its 
-dependencies without requiring the use of a second tool to acquire Catalyst's 
-non-Python dependencies.
+.. code-block:: bash
 
-  For Windows, you will need the *Microsoft Visual C++ Compiler for Python 
-  2.7*. Follow the instructions on the :ref:`Windows` section and come back 
-  here.
+   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
 
-For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
-you can install MiniConda, which is a smaller footprint (fewer packages and 
-smaller size) than its big brother Anaconda, but it still contains all the 
-main packages needed. To install MiniConda, you can follow these steps:
+On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
+following should be sufficient to acquire the necessary additional
+dependencies:
 
-1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
-   for your Operating System.
-2. Install MiniConda. See the `Installation Instructions 
-   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
-3. Ensure the correct installation by running ``conda list`` in a Terminal 
-   window, which should print the list of packages installed with Conda.
+.. code-block:: bash
 
-Once either Conda or MiniConda has been set up you can install Catalyst:
+   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
 
-1. Download the file `python2.7-environment.yml 
-   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
-2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
-   saved the above ``python2.7-environment.yml`` file.
-3. Install using this file. This step can take about 5-10 minutes to install.
+On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
-   .. code-block:: bash
+.. code-block:: bash
 
-      conda env create -f python2.7-environment.yml
+   $ pacman -S lapack gcc gcc-fortran pkg-config
 
-4. Activate the environment (which you need to do every time you start a new 
-   session to run Catalyst):
+.. Commenting it out until Catalyst fully supports Python 3.X
+..
+.. There are also AUR packages available for installing `Python 3.4
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-   **Linux or OSX:**
+.. 
 
-   .. code-block:: bash
+..   $ pacman -S python2
 
-      source activate catalyst
+Amazon Linux AMI Notes
+~~~~~~~~~~~~~~~~~~~~~~
 
-   **Windows:**
+The packages ``pip`` and ``setuptools`` that come shipped by default are very 
+outdated. Thus, you first need to run:
 
-   .. code-block:: bash
+.. code-block:: bash
 
-      activate catalyst
+   pip install --upgrade pip setuptools
 
-Congratulations! You now have Catalyst installed.
+The default installation is also missing the C and C++ compilers, which you 
+install by:
 
-Troubleshooting ``conda`` Install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. code-block:: bash
 
-If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
-above failed for any reason, you can try setting up the environment manually 
-with the following steps:
+   sudo yum install gcc gcc-c++
 
-1. Create the environment:
+Then you should follow the regular installation instructions outlined at the 
+beginning of this page.
 
-   .. code-block:: bash
 
-      conda create --name catalyst python=2.7 scipy zlib
+.. _MacOS:
 
-2. Activate the environment:
+MacOS Requirements
+------------------
 
-   **Linux or OSX:**
+The version of Python shipped with MacOS by default is generally out of date, 
+and has a number of quirks because it's used directly by the operating system.
+For these reasons, many developers choose to install and use a separate Python
+installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
+to `Installing Python on MacOS <http://docs.python-guide.org/en/latest/>`_, 
+which explains how to install Python with the `Homebrew`_ manager.
 
-   .. code-block:: bash
+Assuming you've installed Python with Homebrew, you'll also likely need the
+following brew packages:
 
-      source activate catalyst
+.. code-block:: bash
 
-   **Windows:**
+   $ brew install freetype pkg-config gcc openssl
 
-   .. code-block:: bash
+MacOS + virtualenv + matplotlib
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-      activate catalyst
+A note about using matplotlib in virtual enviroments on MacOS: it may be 
+necessary to run
 
-3. Install the Catalyst inside the environment:
+.. code-block:: bash
 
-   .. code-block:: bash
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
 
-      pip install enigma-catalyst matplotlib
+in order to override the default ``MacOS`` backend for your system, which 
+may not be accessible from inside the virtual environment. This will allow 
+Catalyst to open matplotlib charts from within a virtual environment, which 
+is useful for displaying the performance of your backtests.  To learn more 
+about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+.. _windows:
+
+Windows Requirements
+--------------------
+
+In Windows, you will first need to install the `Microsoft Visual C++ Compiler 
+for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
+package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python 2.7 packages. If it's not already in your 
+system, download it and install it before proceeding to the next step.
+
+Once you have the above compiler installed, the easiest and best supported way 
+to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't 
+any problems installing the compiler, jump to the :ref:`Conda <conda>` section, 
+otherwise keep on reading to troubleshoot the C++ compiler installtion.
+
+Some problems we have encountered installing the **Visual C++ Compiler** 
+mentioned above are as follows:
+
+- **The system administrator has set policies to prevent this installation**.
+  
+  In some systems, there is a default *Windows Software Restriction* policy 
+  that prevents the installation of some software packages like this one. 
+  You'll have to change the Registry to circumvent this:
+
+  - Click ``Start``, and search for ``regedit`` and launch the 
+    ``Registry Editor``
+  - Navigate to the following folder:
+    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
+  - If the last folder does not exist, create it by right-clicking on the 
+    parent folder and choosing -> ``New`` -> ``Key`` and typing ``Installer``
+  - If there is an entry for ``DisableMSI``, set the Value data to 0.
+  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
+    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
+    default you get 0 as the Value Data)
+
+|
+- **The installer has encountered an unexpected error installing this package. 
+  This may indicate a problem with this package. The error code is 2503.**
+
+  We have observed this when trying to install a package without enough 
+  administrator permissions. Even when you are logged in as an Administrator, 
+  you have to explictily install this package with administrator privileges:
+
+  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
+  - Right click on it and choose ``Run as administrator``
+  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
+  - Run ``msiexec /i VCForPython27.msi``
 
 Getting Help
 ------------

docs/source/live-trading.rst

@@ -106,6 +106,10 @@ What differs are the arguments provided to the catalyst client or
 Here is the breakdown of the new arguments:
 
 - ``live``: Boolean flag which enables live trading.
+- ``capital_base``: The amount of base_currency assigned to the strategy.
+  It has to be lower or equal to the amount of base currency available for
+  trading on the exchange. For illustration, order_target_percent(asset, 1)
+  will order the capital_base amount specified here of the specified asset.
 - ``exchange_name``: The name of the targeted exchange
   (supported values: *bitfinex*, *bittrex*).
 - ``algo_namespace``: A arbitrary label assigned to your algorithm for
@@ -113,6 +117,8 @@ Here is the breakdown of the new arguments:
 - ``base_currency``: The base currency used to calculate the
   statistics of your algorithm. Currently, the base currency of all
   trading pairs of your algorithm must match this value.
+- ``simulate_orders``: Enables the paper trading mode, in which orders are
+  simulated in Catalyst instead of processed on the exchange.
 
 Here is a complete algorithm for reference:
 `Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_

docs/source/releases.rst

@@ -2,6 +2,60 @@
 Release Notes
 =============
 
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-12-12
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with fetching assets with daily frequency
+
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with fetching assets with daily frequency
+- Changed Poloniex interface (should solve :issue:`95` and :issue:`94`)
+- Solved issue with overriding commission and slippage (:issue:`87`)
+- Fixed inefficiency with Bittrex current prices (:issue:`76`)
+
+Build
+~~~~~
+- Integrated with CCXT
+- Added paper trading capability (`simulate_orders=True` param in live mode)
+- More granular commissions (:issue:`82`)
+- Added market orders in live mode (:issue:`81`)
+
+Version 0.3.9
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed sortino warning issues (:issue:`77`)
+- Adjusted computation of last candle of data.history (:issue:`71`)
+
+Build
+~~~~~
+- Added capital_base parameter to live mode to limit cash (:issue:`79`)
+- Added support for csv ingestion (:issue:`65`)
+- Improved cash display in running stats (:issue:`80`)
+
+
+Version 0.3.8
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed a warning filter issue introduced with the latest release
+
 Version 0.3.7
 ^^^^^^^^^^^^^
 **Release Date**: 2017-11-14
@@ -50,7 +104,7 @@ Bug Fixes
 - Fixed issue with sell orders in backtesting
 - Fixed data frequency issues with data.history() in backtesting
 - Fixed an issue with can_trade()
-- Reduced the commission and slippage values to account for lower volume 
+- Reduced the commission and slippage values to account for lower volume
   transactions
 
 Build
@@ -62,17 +116,17 @@ Documentation
 ~~~~~~~~~~~~~
 
 - Improved installation notes for Windows C++ compiler and Conda
-- Addition of 
+- Addition of
   `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
-- Addition of 
+- Addition of
   `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
-- Addition of 
+- Addition of
   `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
-- Addition of 
+- Addition of
   `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
-- Addition of `Development Guidelines 
+- Addition of `Development Guidelines
   <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
-- Addition of 
+- Addition of
   `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
 - Updated code docstrings
 
@@ -123,10 +177,10 @@ Bug Fixes
 ~~~~~~~~~
 
 - Fixed OS-dependent path issue in data bundle
-- Changed handling of empty ``auth.json``, instead of throwing an error for 
+- Changed handling of empty ``auth.json``, instead of throwing an error for
   missing file
 - Updated ``etc/python2.7-environment.yml`` to work with Catalyst version 0.3
-- Updated ``catalyst/examples/buy_and_hodl.py``  and 
+- Updated ``catalyst/examples/buy_and_hodl.py``  and
   ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
 
 
@@ -146,18 +200,18 @@ Version 0.2.dev5
 ^^^^^^^^^^^^^^^^
 **Release Date**: 2017-10-03
 
-- Fixes bug in data.history function that was formatting 'volume' data as 
-  integers, now they are returned as floats with up to 9 decimals of precision. 
+- Fixes bug in data.history function that was formatting 'volume' data as
+  integers, now they are returned as floats with up to 9 decimals of precision.
   Data bundles redone.
 
-Version 0.2.dev4 
+Version 0.2.dev4
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-09-20
 
-- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal 
+- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal
   places. Pricing resolution of daily data remains set to 9 decimal places.
-- The current data bundle takes 340MB compressed for download, and 460MB 
+- The current data bundle takes 340MB compressed for download, and 460MB
   uncompressed on disk for Catalyst to use.
 
 Version 0.2.dev3
@@ -167,14 +221,14 @@ Version 0.2.dev3
 
 - 1-minute resolution OHLCV data bundle for backtesting from Poloniex exchange
 - Implementation of trading of fractional crypto assets (i.e. 0.01 BTC)
-- Minimum trade size of a coin can be configured on a per-coin basis, defaults 
-  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger 
+- Minimum trade size of a coin can be configured on a per-coin basis, defaults
+  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger
   amounts, which will impact live trading)
 - Increased pricing resolution from 3 to 9 decimal places
-- The current data bundle takes 40MB compressed for download, and 99MB 
+- The current data bundle takes 40MB compressed for download, and 99MB
   uncompressed on disk for Catalyst to use.
 
-Version 0.2.dev2 
+Version 0.2.dev2
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-09-07
@@ -190,15 +244,15 @@ Version 0.2.dev1
 
   - Comprehensive trading functionality against exchanges Bitfinex and Bittrex.
   - Support for all trading pairs available on each exchange.
-  - Multiple algorithms can trade simultaneously against a single exchange 
+  - Multiple algorithms can trade simultaneously against a single exchange
     using the same account.
-  - Each algorithm has a persisted state (i.e. algorithm can be stopped and 
-    restarted preserving the state without data loss) that tracks all open 
+  - Each algorithm has a persisted state (i.e. algorithm can be stopped and
+    restarted preserving the state without data loss) that tracks all open
     orders, executed transactions and portfolio positions.
 
 - Minute by minute portfolio performance metrics.
 
-  - Daily summary performance statistics compatible with pyfolio, a Python 
+  - Daily summary performance statistics compatible with pyfolio, a Python
     library for performance and risk analysis of financial portfolios
 
 Version 0.1.dev9
@@ -206,13 +260,13 @@ Version 0.1.dev9
 
 **Release Date**: 2017-08-28
 
-- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex 
+- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex
   exchange directly
 - Change of bundle storage provider from Dropbox to AWS
 - Fix issue with 1/1000 scaling issue of prices in bundle
 
 Version 0.1.dev8
-^^^^^^^^^^^^^^^^ 
+^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-08-18
 
@@ -232,4 +286,3 @@ Version 0.1.dev6
 **Release Date**: 2017-07-13
 
 - Initial public release
-

docs/source/utilities.rst

@@ -0,0 +1,149 @@
+Utilities
+=========
+
+This section covers a variety of utilites that provide complimentary 
+functionality to your trading algorithms. These are code snippets that you can
+add to any algorithm to add the desired functionality.
+
+If you are looking for example trading algorithms, see the corresponding section.
+
+Output to CSV file
+~~~~~~~~~~~~~~~~~~
+
+Add this script to the analyze method to create and save a CSV file with the 
+results from the trading algorithm. This file will include the default 
+parameters of the results DataFrame plus any recorded variables and will be 
+saved in the same location where your trading algorithm is saved. The exact 
+script that you need to use depends on the interface that you are using to run 
+your trading algorithm, which could be the CLI or a Python Interpreter.
+
+1. Script to use with CLI:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import sys
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(basename(sys.argv[3]))[0]
+          results.to_csv(filename + '.csv')
+
+2. Script to use with Python Interpreter:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(os.path.basename(__file__))[0]
+          results.to_csv(filename + '.csv')
+
+Extracting market data
+~~~~~~~~~~~~~~~~~~~~~~
+
+Use this script to save the price and volume data of one cryptoasset in a CSV 
+file, which will be saved in the same location and with the same name as your 
+Python file. To get custom data, simply modify the asset's symbol and the dates. 
+Run this script directly from your development environment: python scriptname.py, 
+where the contents of 'scriptname.py' are as follows. Two different version are 
+provided as an example for daily- and minute-resolution data respectively:
+
+Simpler case for daily data
+
+.. code-block:: python
+
+   import os
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+   def initialize(context):
+      # Portfolio assets list
+      context.asset = symbol('btc_usdt') # Bitcoin on Poloniex
+                          
+   def handle_data(context, data):
+      # Variables to record for a given asset: price and volume
+      price = data.current(context.asset, 'price')
+      volume = data.current(context.asset, 'volume')
+      record(price=price, volume=volume)
+      
+   def analyze(context=None, results=None):
+      
+      # Generate DataFrame with Price and Volume only
+      data = results[['price','volume']]
+      
+      # Save results in CSV file
+      filename = os.path.splitext(os.path.basename(__file__))[0]
+      data.to_csv(filename + '.csv')
+      
+   ''' Bitcoin data is available on Poloniex since 2015-3-1. 
+       Dates vary for other tokens. In the example below, we choose the 
+       full month of July of 2017.
+   '''
+   start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data, 
+                           analyze=analyze, 
+                           start=start, 
+                           end=end, 
+                           exchange_name='poloniex', 
+                           capital_base=10000, 
+                           base_currency = 'usdt')
+
+More versatile case for minute data
+
+.. code-block:: python
+
+   import os
+   import csv
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+
+   def initialize(context):
+   # Portfolio assets list
+   context.asset     = symbol('btc_usdt') # Bitcoin on Poloniex
+
+   # Creates a .CSV file with the same name as this script to store results
+   context.csvfile   = open(os.path.splitext(
+                            os.path.basename(__file__))[0]+'.csv', 'w+')
+   context.csvwriter = csv.writer(context.csvfile)
+
+   def handle_data(context, data):
+   # Variables to record for a given asset: price and volume
+   # Other options include 'open', 'high', 'open', 'close'
+   # Please note that 'price' equals 'close'
+   date   = context.blotter.current_dt     # current time in each iteration
+   price  = data.current(context.asset, 'price')
+   volume = data.current(context.asset, 'volume')
+
+   # Writes one line to CSV on each iteration with the chosen variables    
+   context.csvwriter.writerow([date,price,volume])
+
+   def analyze(context=None, results=None):
+   # Close open file properly at the end
+   context.csvfile.close()
+
+
+   # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
+   start = datetime(2017, 7, 30, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data,
+                           analyze=analyze, 
+                           start=start, 
+                           end=end,
+                           exchange_name='poloniex', 
+                           data_frequency='minute',
+                           base_currency ='usdt', 
+                           capital_base=10000 )

docs/source/videos.rst

@@ -22,5 +22,37 @@ Where things go smoothly:
 
 |
 Where things don't:
+
+.. raw:: html
    
-   Coming up next!
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/qLkQcWlUBy8" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Backtesting a Strategy
+----------------------
+
+This is the first video of a two-part series on using Catalyst for algorithmic 
+trading. This video implements a simple momentum strategy based on 
+`mean reversion <example-algos.html#mean-reversion>`_: when the cryptoasset
+goes up quickly, we’re going to buy; when it goes down quickly, we’re going to 
+sell. Hopefully, we’ll ride the waves.
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/JOBRwst9jUY" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Live Trading a Strategy
+-----------------------
+
+This is the second part of the two-part series on using Catalyst for algorithmic
+trading. Having backtested `our strategy <example-algos.html#mean-reversion>`_ 
+in the previous video, we now take it to trade live against the Bittrex exchange.
+
+.. raw:: html
+
+	<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
+|
+|

docs/source/welcome.rst

@@ -1,43 +0,0 @@
-.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
-|
-Catalyst is an algorithmic trading library for crypto-assets written in Python.
-It allows trading strategies to be easily expressed and backtested against 
-historical data (with daily and minute resolution), providing analytics and 
-insights regarding a particular strategy's performance. Catalyst also supports
-live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
-and Poloniex) with more being added over time. Catalyst empowers users to share 
-and curate data and build profitable, data-driven investment strategies. Please 
-visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
-refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
-further technical details.
-
-Catalyst builds on top of the well-established 
-`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
-minimize structural changes to the general API to maximize compatibility with 
-existing trading algorithms, developer knowledge, and tutorials. Join us on 
-`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
-for questions around Catalyst, algorithmic trading and technical support.
-
-Features
-========
-
--  Ease of use: Catalyst tries to get out of your way so that you can 
-   focus on algorithm development. See 
-   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
-   provided.
--  Support for several of the top crypto-exchanges by trading volume:
-   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
-   and `Poloniex <https://www.poloniex.com>`_. 
--  Secure: You and only you have access to each exchange API keys for your accounts.
--  Input of historical pricing data of all crypto-assets by exchange, 
-   with daily and minute resolution. See 
-   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
--  Backtesting and live-trading functionality, with a seamless transition
-   between the two modes.
--  Output of performance statistics are based on Pandas DataFrames to 
-   integrate nicely into the existing PyData eco-system.
--  Statistic and machine learning libraries like matplotlib, scipy, 
-   statsmodels, and sklearn support development, analysis, and 
-   visualization of state-of-the-art trading systems.
--  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
-   performance across trading algorithms.

etc/python2.7-environment.yml

@@ -20,6 +20,7 @@ dependencies:
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
+  - ccxt==1.10.319
   - click==6.7
   - contextlib2==0.5.5
   - cycler==0.10.0

etc/requirements.txt

@@ -80,3 +80,6 @@ empyrical==0.2.1
 
 tables==3.3.0
 
+#Catalyst dependencies
+ccxt==1.10.283
+boto3==1.4.8

tests/exchange/test_bcolz.py

@@ -116,7 +116,7 @@ class TestBcolzWriter(object):
 
         df = self.generate_df(exchange_name, freq, start, end)
 
-        print df.index[0],df.index[-1]
+        print(df.index[0], df.index[-1])
 
         writer = BcolzExchangeBarWriter(
             rootdir=self.root_dir,
@@ -140,7 +140,7 @@ class TestBcolzWriter(object):
 
         dx = get_df_from_arrays(arrays, periods)
 
-        assert_equals(df.equals(df), True)
+        assert_equals(df.equals(dx), True)
         pass
 
     def test_bcolz_bitfinex_daily_write_read(self):

tests/exchange/test_bitfinex.py

@@ -4,10 +4,12 @@ from base import BaseExchangeTestCase
 from catalyst.exchange.bitfinex.bitfinex import Bitfinex
 from catalyst.exchange.exchange_utils import get_exchange_auth
 from catalyst.finance.execution import (LimitOrder)
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('test_bitfinex')
 
 
+@deprecated
 class TestBitfinex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
@@ -34,7 +36,7 @@ class TestBitfinex(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        orders = self.exchange.get_open_orders()
+        # orders = self.exchange.get_open_orders()
         pass
 
     def test_get_order(self):
@@ -47,18 +49,17 @@ class TestBitfinex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
-            freq='1T',
-            assets=self.exchange.get_asset('neo_btc')
-        )
+        # ohlcv_neo = self.exchange.get_candles(
+        #    freq='1T',
+        #    assets=self.exchange.get_asset('neo_btc'))
         pass
 
     def test_tickers(self):
         log.info('retrieving tickers')
-        tickers = self.exchange.tickers([
-            self.exchange.get_asset('eth_btc'),
-            self.exchange.get_asset('etc_btc')
-        ])
+        # tickers = self.exchange.tickers([
+        #     self.exchange.get_asset('eth_btc'),
+        #     self.exchange.get_asset('etc_btc')
+        # ])
         pass
 
     def test_get_account(self):
@@ -67,11 +68,11 @@ class TestBitfinex(BaseExchangeTestCase):
 
     def test_get_balances(self):
         log.info('testing exchange balances')
-        balances = self.exchange.get_balances()
+        # balances = self.exchange.get_balances()
         pass
 
     def test_orderbook(self):
         log.info('testing order book for bitfinex')
-        asset = self.exchange.get_asset('eth_btc')
-        orderbook = self.exchange.get_orderbook(asset)
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset)
         pass

tests/exchange/test_bittrex.py

@@ -1,13 +1,15 @@
-import pandas as pd
+# import pandas as pd
 from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.utils.deprecate import deprecated
 
 log = Logger('test_bittrex')
 
 
+@deprecated
 class TestBittrex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
@@ -33,8 +35,8 @@ class TestBittrex(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        asset = self.exchange.get_asset('neo_btc')
-        orders = self.exchange.get_open_orders(asset)
+        # asset = self.exchange.get_asset('neo_btc')
+        # orders = self.exchange.get_open_orders(asset)
         pass
 
     def test_get_order(self):
@@ -51,21 +53,21 @@ class TestBittrex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
-            freq='5T',
-            assets=self.exchange.get_asset('neo_btc'),
-            bar_count=20,
-            end_dt=pd.to_datetime('2017-10-20', utc=True)
-        )
-        ohlcv_neo_ubq = self.exchange.get_candles(
-            freq='1D',
-            assets=[
-                self.exchange.get_asset('neo_btc'),
-                self.exchange.get_asset('ubq_btc')
-            ],
-            bar_count=14,
-            end_dt=pd.to_datetime('2017-10-20', utc=True)
-        )
+        # ohlcv_neo = self.exchange.get_candles(
+        #     freq='5T',
+        #     assets=self.exchange.get_asset('neo_btc'),
+        #     bar_count=20,
+        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
+        # )
+        # ohlcv_neo_ubq = self.exchange.get_candles(
+        #     freq='1D',
+        #     assets=[
+        #         self.exchange.get_asset('neo_btc'),
+        #         self.exchange.get_asset('ubq_btc')
+        #     ],
+        #     bar_count=14,
+        #     end_dt=pd.to_datetime('2017-10-20', utc=True)
+        # )
         pass
 
     def test_tickers(self):
@@ -79,7 +81,7 @@ class TestBittrex(BaseExchangeTestCase):
 
     def test_get_balances(self):
         log.info('testing wallet balances')
-        balances = self.exchange.get_balances()
+        # balances = self.exchange.get_balances()
         pass
 
     def test_get_account(self):
@@ -88,6 +90,6 @@ class TestBittrex(BaseExchangeTestCase):
 
     def test_orderbook(self):
         log.info('testing order book for bittrex')
-        asset = self.exchange.get_asset('eth_btc')
-        orderbook = self.exchange.get_orderbook(asset)
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset)
         pass

tests/exchange/test_bundle.py

@@ -1,11 +1,10 @@
-import hashlib
+# import hashlib
 import os
 import tempfile
 from logging import getLogger
 
 import pandas as pd
 
-from catalyst import get_calendar
 from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
     get_start_dt, get_df_from_arrays
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
@@ -22,22 +21,22 @@ log = getLogger('test_exchange_bundle')
 
 class TestExchangeBundle:
     def test_spot_value(self):
-        data_frequency = 'daily'
-        exchange_name = 'poloniex'
+        # data_frequency = 'daily'
+        # exchange_name = 'poloniex'
 
-        exchange = get_exchange(exchange_name)
-        exchange_bundle = ExchangeBundle(exchange)
-        assets = [
-            exchange.get_asset('btc_usdt')
-        ]
-        dt = pd.to_datetime('2017-10-14', utc=True)
+        # exchange = get_exchange(exchange_name)
+        # exchange_bundle = ExchangeBundle(exchange)
+        # assets = [
+        #     exchange.get_asset('btc_usdt')
+        # ]
+        # dt = pd.to_datetime('2017-10-14', utc=True)
 
-        values = exchange_bundle.get_spot_values(
-            assets=assets,
-            field='close',
-            dt=dt,
-            data_frequency=data_frequency
-        )
+        # values = exchange_bundle.get_spot_values(
+        #     assets=assets,
+        #     field='close',
+        #     dt=dt,
+        #     data_frequency=data_frequency
+        # )
         pass
 
     def test_ingest_minute(self):
@@ -215,7 +214,7 @@ class TestExchangeBundle:
         # encounter these problems as I have been focusing on minute data.
         reader = exchange_bundle.get_reader(data_frequency)
         for asset in assets:
-            # Since this pair was loaded last. It should be there in daily mode.
+            # Since this pair was loaded last. It should be here in daily mode.
             arrays = reader.load_raw_arrays(
                 sids=[asset.sid],
                 fields=['close'],
@@ -252,7 +251,6 @@ class TestExchangeBundle:
         ensure_directory(path)
 
         exchange_bundle = ExchangeBundle(exchange)
-        calendar = get_calendar('OPEN')
 
         # We are using a BcolzMinuteBarWriter even though the data is daily
         # Each day has a maximum of one bar
@@ -304,26 +302,25 @@ class TestExchangeBundle:
         pass
 
     def test_minute_bundle(self):
-        exchange_name = 'poloniex'
-        data_frequency = 'minute'
+        # exchange_name = 'poloniex'
+        # data_frequency = 'minute'
 
-        exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neos_btc')
-
-        path = get_bcolz_chunk(
-            exchange_name=exchange_name,
-            symbol=asset.symbol,
-            data_frequency=data_frequency,
-            period='2017-5',
-        )
+        # exchange = get_exchange(exchange_name)
+        # asset = exchange.get_asset('neos_btc')
 
+        # path = get_bcolz_chunk(
+        #     exchange_name=exchange_name,
+        #     symbol=asset.symbol,
+        #     data_frequency=data_frequency,
+        #     period='2017-5',
+        # )
         pass
 
     def test_hash_symbol(self):
-        symbol = 'etc_btc'
-        sid = int(
-            hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
-        ) % 10 ** 6
+        # symbol = 'etc_btc'
+        # sid = int(
+        #     hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+        # ) % 10 ** 6
         pass
 
     def test_validate_data(self):
@@ -438,7 +435,7 @@ class TestExchangeBundle:
         pass
 
     def main_bundle_to_csv(self):
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
@@ -448,7 +445,7 @@ class TestExchangeBundle:
         end_dt = pd.to_datetime('2016-6-1', utc=True)
         self._bundle_to_csv(
             asset=asset,
-            exchange=exchange,
+            exchange_name=exchange.name,
             data_frequency=data_frequency,
             filename='{}_{}_{}'.format(
                 exchange_name, data_frequency, asset.symbol
@@ -460,7 +457,7 @@ class TestExchangeBundle:
     def bundle_to_csv(self):
         exchange_name = 'poloniex'
         data_frequency = 'minute'
-        period = '2017-09'
+        period = '2017-01'
         symbol = 'eth_btc'
 
         exchange = get_exchange(exchange_name)
@@ -474,16 +471,16 @@ class TestExchangeBundle:
         )
         self._bundle_to_csv(
             asset=asset,
-            exchange=exchange,
+            exchange_name=exchange.name,
             data_frequency=data_frequency,
             path=path,
             filename=period
         )
         pass
 
-    def _bundle_to_csv(self, asset, exchange, data_frequency, filename,
+    def _bundle_to_csv(self, asset, exchange_name, data_frequency, filename,
                        path=None, start_dt=None, end_dt=None):
-        bundle = ExchangeBundle(exchange)
+        bundle = ExchangeBundle(exchange_name)
         reader = bundle.get_reader(data_frequency, path=path)
 
         if start_dt is None:
@@ -514,14 +511,39 @@ class TestExchangeBundle:
         df = get_df_from_arrays(arrays, periods)
 
         folder = os.path.join(
-            tempfile.gettempdir(), 'catalyst', exchange.name, asset.symbol
+            tempfile.gettempdir(), 'catalyst', exchange_name, asset.symbol
         )
         ensure_directory(folder)
 
         path = os.path.join(folder, filename + '.csv')
 
         log.info('creating csv file: {}'.format(path))
-        print('HEAD\n{}'.format(df.head(10)))
-        print('TAIL\n{}'.format(df.tail(10)))
+        print('HEAD\n{}'.format(df.head(100)))
+        print('TAIL\n{}'.format(df.tail(100)))
         df.to_csv(path)
         pass
+
+    def test_ingest_csv(self):
+        data_frequency = 'minute'
+        exchange_name = 'bittrex'
+        path = '/Users/fredfortier/Dropbox/Enigma/Data/bittrex_bat_eth.csv'
+
+        exchange_bundle = ExchangeBundle(exchange_name)
+        exchange_bundle.ingest_csv(path, data_frequency)
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset('bat_eth')
+
+        start_dt = pd.to_datetime('2017-6-3', utc=True)
+        end_dt = pd.to_datetime('2017-8-3 19:24', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange_name=exchange.name,
+            data_frequency=data_frequency,
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
+        pass

tests/exchange/test_ccxt.py

@@ -0,0 +1,93 @@
+import pandas as pd
+from logbook import Logger
+from base import BaseExchangeTestCase
+
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.finance.order import Order
+from catalyst.exchange.exchange_utils import get_exchange_auth
+
+log = Logger('test_ccxt')
+
+
+class TestCCXT(BaseExchangeTestCase):
+    @classmethod
+    def setup(self):
+        exchange_name = 'gdax'
+        auth = get_exchange_auth(exchange_name)
+        self.exchange = CCXT(
+            exchange_name=exchange_name,
+            key=auth['key'],
+            secret=auth['secret'],
+            base_currency='eth',
+            portfolio=None
+        )
+
+    def test_order(self):
+        log.info('creating order')
+        asset = self.exchange.get_asset('neo_eth')
+        order_id = self.exchange.order(
+            asset=asset,
+            limit_price=0.07,
+            amount=1,
+        )
+        log.info('order created {}'.format(order_id))
+        assert order_id is not None
+        pass
+
+    def test_open_orders(self):
+        # log.info('retrieving open orders')
+        # asset = self.exchange.get_asset('neo_eth')
+        # orders = self.exchange.get_open_orders(asset)
+        pass
+
+    def test_get_order(self):
+        log.info('retrieving order')
+        order = self.exchange.get_order('2631386', 'neo_eth')
+        # order = self.exchange.get_order('2631386')
+        assert isinstance(order, Order)
+        pass
+
+    def test_cancel_order(self, ):
+        log.info('cancel order')
+        self.exchange.cancel_order('2631386', 'neo_eth')
+        pass
+
+    def test_get_candles(self):
+        log.info('retrieving candles')
+        candles = self.exchange.get_candles(
+            freq='5T',
+            assets=[self.exchange.get_asset('eth_btc')],
+            bar_count=200,
+            start_dt=pd.to_datetime('2017-01-01', utc=True)
+        )
+
+        for asset in candles:
+            df = pd.DataFrame(candles[asset])
+            df.set_index('last_traded', drop=True, inplace=True)
+        pass
+
+    def test_tickers(self):
+        log.info('retrieving tickers')
+        tickers = self.exchange.tickers([
+            self.exchange.get_asset('eth_btc'),
+        ])
+        assert len(tickers) == 1
+        pass
+
+    def test_get_balances(self):
+        log.info('testing wallet balances')
+        # balances = self.exchange.get_balances()
+        pass
+
+    def test_get_account(self):
+        log.info('testing account data')
+        pass
+
+    def test_orderbook(self):
+        log.info('testing order book for bittrex')
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
+        pass
+
+    def test_get_fees(self):
+        pass

tests/exchange/test_data_portal.py

@@ -1,16 +1,15 @@
 import pandas as pd
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
-    DataPortalExchangeLive
 from logbook import Logger
-from test_utils import rnd_history_date_days, rnd_bar_count
 
 from catalyst import get_calendar
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_common_assets
-from catalyst.exchange.factory import get_exchange, get_exchanges
+from catalyst.exchange.exchange_data_portal import (
+    DataPortalExchangeBacktest,
+    DataPortalExchangeLive
+)
+from catalyst.exchange.exchange_utils import get_common_assets
+from catalyst.exchange.factory import get_exchanges
+from test_utils import rnd_history_date_days, rnd_bar_count
 
 log = Logger('test_bitfinex')
 
@@ -38,31 +37,31 @@ class TestExchangeDataPortal:
         )
 
     def test_get_history_window_live(self):
-        asset_finder = self.data_portal_live.asset_finder
+        # asset_finder = self.data_portal_live.asset_finder
 
-        assets = [
-            asset_finder.lookup_symbol('eth_btc', self.bitfinex),
-            asset_finder.lookup_symbol('eth_btc', self.bittrex)
-        ]
-        now = pd.Timestamp.utcnow()
-        data = self.data_portal_live.get_history_window(
-            assets,
-            now,
-            10,
-            '1m',
-            'price')
+        # assets = [
+        #     asset_finder.lookup_symbol('eth_btc', self.bitfinex),
+        #     asset_finder.lookup_symbol('eth_btc', self.bittrex)
+        # ]
+        # now = pd.Timestamp.utcnow()
+        # data = self.data_portal_live.get_history_window(
+        #     assets,
+        #     now,
+        #     10,
+        #     '1m',
+        #     'price')
         pass
 
     def test_get_spot_value_live(self):
-        asset_finder = self.data_portal_live.asset_finder
+        # asset_finder = self.data_portal_live.asset_finder
 
-        assets = [
-            asset_finder.lookup_symbol('eth_btc', self.bitfinex),
-            asset_finder.lookup_symbol('eth_btc', self.bittrex)
-        ]
-        now = pd.Timestamp.utcnow()
-        value = self.data_portal_live.get_spot_value(
-            assets, 'price', now, '1m')
+        # assets = [
+        #     asset_finder.lookup_symbol('eth_btc', self.bitfinex),
+        #     asset_finder.lookup_symbol('eth_btc', self.bittrex)
+        # ]
+        # now = pd.Timestamp.utcnow()
+        # value = self.data_portal_live.get_spot_value(
+        #     assets, 'price', now, '1m')
         pass
 
     def test_get_history_window_backtest(self):
@@ -113,3 +112,6 @@ class TestExchangeDataPortal:
         )
 
         log.info('found history window: {}'.format(data))
+
+    def test_validate_resample(self):
+        pass

tests/exchange/test_poloniex.py

@@ -1,13 +1,17 @@
-from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.exchange.poloniex.poloniex import Poloniex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+import pandas as pd
+
+from catalyst.utils.deprecate import deprecated
+from test_utils import output_df
 
 log = Logger('test_poloniex')
 
 
+@deprecated
 class TestPoloniex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
@@ -33,8 +37,8 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        asset = self.exchange.get_asset('neos_btc')
-        orders = self.exchange.get_open_orders(asset)
+        # asset = self.exchange.get_asset('neos_btc')
+        # orders = self.exchange.get_open_orders(asset)
         pass
 
     def test_get_order(self):
@@ -51,18 +55,20 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
+        assets = self.exchange.get_asset('eth_btc')
+        ohlcv = self.exchange.get_candles(
+            # end_dt=pd.to_datetime('2017-11-01', utc=True),
+            end_dt=None,
             freq='5T',
-            assets=self.exchange.get_asset('eth_btc')
-        )
-        ohlcv_neo_ubq = self.exchange.get_candles(
-            freq='5T',
-            assets=[
-                self.exchange.get_asset('neos_btc'),
-                self.exchange.get_asset('via_btc')
-            ],
-            bar_count=14
+            assets=assets,
+            bar_count=200
         )
+        df = pd.DataFrame(ohlcv)
+        df.set_index('last_traded', drop=True, inplace=True)
+        log.info(df.tail(25))
+
+        path = output_df(df, assets, '5min_candles')
+        log.info('saved candles: {}'.format(path))
         pass
 
     def test_tickers(self):
@@ -76,7 +82,7 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_get_balances(self):
         log.info('testing wallet balances')
-        balances = self.exchange.get_balances()
+        # balances = self.exchange.get_balances()
         pass
 
     def test_get_account(self):
@@ -85,7 +91,6 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_orderbook(self):
         log.info('testing order book for poloniex')
-        asset = self.exchange.get_asset('eth_btc')
-
-        orderbook = self.exchange.get_orderbook(asset)
+        # asset = self.exchange.get_asset('eth_btc')
+        # orderbook = self.exchange.get_orderbook(asset)
         pass

tests/exchange/test_server_bundle.py

@@ -1,21 +1,16 @@
 import os
-import tarfile
 import importlib
+
 import pandas as pd
-
-from catalyst import get_calendar
-
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
-from catalyst.data.minute_bars import BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
-
 import matplotlib
 import matplotlib.pyplot as plt
 from matplotlib.finance import candlestick2_ohlc
-from matplotlib.finance import volume_overlay
+# from matplotlib.finance import volume_overlay
 import matplotlib.ticker as ticker
 
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
+from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
 from catalyst.exchange.factory import get_exchange
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
@@ -51,7 +46,7 @@ class ValidateChunks(object):
         if data_frequency == 'daily':
             end = end - pd.Timedelta(hours=23, minutes=59)
 
-        print start, end, data_frequency
+        print(start, end, data_frequency)
 
         arrays = reader.load_raw_arrays(self.columns, start, end,
                                         [asset.sid, ])
@@ -85,8 +80,8 @@ class ValidateChunks(object):
             matplotlib.transforms.Bbox([[0.125, 0.1], [0.9, 0.26]]))
 
         # Plot the volume overlay
-        bc = volume_overlay(ax2, df['open'], df['close'], df['volume'],
-                            colorup='g', alpha=0.5, width=1)
+        # bc = volume_overlay(ax2, df['open'], df['close'], df['volume'],
+        #                     colorup='g', alpha=0.5, width=1)
 
         ax.xaxis.set_major_locator(ticker.MaxNLocator(6))
 

tests/exchange/test_utils.py

@@ -1,17 +1,65 @@
+import os
+import tempfile
 from datetime import timedelta
 from random import randint
 
 import pandas as pd
+from catalyst.assets._assets import TradingPair
+
+from catalyst.utils.paths import ensure_directory
 
 
-def rnd_history_date_days(max_days=30):
-    now = pd.Timestamp.utcnow()
+def rnd_history_date_days(max_days=30, last_dt=None):
+    if last_dt is None:
+        last_dt = pd.Timestamp.utcnow()
+
     days = randint(0, max_days)
 
-    return now - timedelta(days=days)
+    return last_dt - timedelta(days=days)
+
+
+def rnd_history_date_minutes(max_minutes=1440):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_minutes)
+
+    return now - timedelta(minutes=days)
 
 
 def rnd_bar_count(max_bars=21):
-    now = pd.Timestamp.utcnow()
-
+    # now = pd.Timestamp.utcnow()
     return randint(0, max_bars)
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path