Merge branch 'develop'

DOC: fix import and support python3 for beginner tutorial

.gitignore

@@ -40,6 +40,7 @@ develop-eggs
 coverage.xml
 htmlcov
 nosetests.xml
+.python-version
 
 # C Extensions
 *.o

Dockerfile

@@ -11,10 +11,10 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
 # You can also run an algo using the docker exec command. For example:

Dockerfile-dev

@@ -5,7 +5,7 @@
 #
 # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile
+#    docker build -t quantopian/catalyst -f Dockerfile .
 #
 # To run the container:
 #
@@ -15,10 +15,10 @@
 #
 #    https://127.0.0.1
 #
-# default password is jupyter.  to provide another, see:
+# Default password is 'jupyter'. To provide another, see:
 #    http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
 #
-# once generated, you can pass the new value via `docker run --env` the first time
+# Once generated, you can pass the new value via `docker run --env` the first time
 # you start the container.
 #
 # You can also run an algo using the docker exec command. For example:

README.rst

@@ -1,3 +1,72 @@
-All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
-can be found in the 
+    :target: https://enigmampc.github.io/catalyst
-`documentation website <https://enigmampc.github.io/catalyst>`_.
+    :align: center
+    :alt: Enigma | Catalyst
+
+|version tag|
+|version status|
+|discord|
+|twitter|
+
+|
+
+Catalyst is an algorithmic trading library for crypto-assets written in Python.
+It allows trading strategies to be easily expressed and backtested against 
+historical data (with daily and minute resolution), providing analytics and 
+insights regarding a particular strategy's performance. Catalyst also supports
+live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
+and Poloniex) with more being added over time. Catalyst empowers users to share 
+and curate data and build profitable, data-driven investment strategies. Please 
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
+refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
+further technical details.
+
+Catalyst builds on top of the well-established 
+`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
+minimize structural changes to the general API to maximize compatibility with 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
+for questions around Catalyst, algorithmic trading and technical support.
+
+Overview
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.
+-  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
+   performance across trading algorithms.
+
+Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
+
+
+
+
+.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
+   :target: https://pypi.python.org/pypi/enigma-catalyst
+
+.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
+   :target: https://discordapp.com/invite/SJK32GY
+
+.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
+   :target: https://twitter.com/enigmampc
+
+

catalyst/__init__.py

@@ -29,11 +29,14 @@ from ._version import get_versions
 from . algorithm import TradingAlgorithm
 from . import api
 
+from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
+
+__version__ = get_versions()['version']
+del get_versions
 
 # PERF: Fire a warning if calendars were instantiated during catalyst import.
 # Having calendars doesn't break anything per-se, but it makes catalyst imports
 # noticeably slower, which becomes particularly noticeable in the Zipline CLI.
-from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
 if global_calendar_dispatcher._calendars:
     import warnings
     warnings.warn(
@@ -44,10 +47,6 @@ if global_calendar_dispatcher._calendars:
 del global_calendar_dispatcher
 
 
-__version__ = get_versions()['version']
-del get_versions
-
-
 def load_ipython_extension(ipython):
     from .__main__ import catalyst_magic
     ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -69,7 +68,6 @@ if os.name == 'nt':
     _()
     del _
 
-
 __all__ = [
     'TradingAlgorithm',
     'api',
@@ -80,7 +78,3 @@ __all__ = [
     'run_algorithm',
     'utils',
 ]
-
-from ._version import get_versions
-__version__ = get_versions()['version']
-del get_versions

catalyst/__main__.py

@@ -9,8 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_utils import delete_algo_folder
+from catalyst.exchange.utils.exchange_utils import delete_algo_folder
-from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -194,9 +193,7 @@ def ipython_only(option):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the targeted exchange.',
-    help='The name of the targeted exchange (supported: bitfinex,'
-         ' bittrex, poloniex).',
 )
 @click.option(
     '-n',
@@ -258,8 +255,9 @@ def run(ctx,
         ctx.fail("must specify a base currency with '-c' in backtest mode")
 
     if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'"
+        ctx.fail("must specify a capital base with '--capital-base'")
-                 " in backtest mode")
+
+    click.echo('Running in backtesting mode.')
 
     perf = _run(
         initialize=None,
@@ -284,7 +282,11 @@ def run(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=False
+        analyze_live=None,
+        live_graph=False,
+        simulate_orders=True,
+        auth_aliases=None,
+        stats_output=None,
     )
 
     if output == '-':
@@ -336,6 +338,12 @@ def catalyst_magic(line, cell=None):
     type=click.File('r'),
     help='The file that contains the algorithm to run.',
 )
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
 @click.option(
     '-t',
     '--algotext',
@@ -374,9 +382,7 @@ def catalyst_magic(line, cell=None):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the targeted exchange.',
-    help='The name of the targeted exchange (supported: bitfinex,'
-         ' bittrex, poloniex).',
 )
 @click.option(
     '-n',
@@ -389,15 +395,38 @@ def catalyst_magic(line, cell=None):
     help='The base currency used to calculate statistics '
          '(e.g. usd, btc, eth).',
 )
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
 @click.option(
     '--live-graph/--no-live-graph',
     is_flag=True,
     default=False,
     help='Display live graph.',
 )
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
 @click.pass_context
 def live(ctx,
          algofile,
+         capital_base,
          algotext,
          define,
          output,
@@ -406,7 +435,10 @@ def live(ctx,
          exchange_name,
          algo_namespace,
          base_currency,
-         live_graph):
+         end,
+         live_graph,
+         auth_aliases,
+         simulate_orders):
     """Trade live with the given algorithm.
     """
     if (algotext is not None) == (algofile is not None):
@@ -417,11 +449,22 @@ def live(ctx,
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
+
     if algo_namespace is None:
         ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
     if base_currency is None:
         ctx.fail("must specify a base currency '-c' in live execution mode")
 
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.')
+
+    else:
+        click.echo('Running in live trading mode.')
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -431,12 +474,12 @@ def live(ctx,
         algotext=algotext,
         defines=define,
         data_frequency=None,
-        capital_base=None,
+        capital_base=capital_base,
         data=None,
         bundle=None,
         bundle_timestamp=None,
         start=None,
-        end=None,
+        end=end,
         output=output,
         print_algo=print_algo,
         local_namespace=local_namespace,
@@ -445,7 +488,11 @@ def live(ctx,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        analyze_live=None,
+        simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
+        stats_output=None,
     )
 
     if output == '-':
@@ -460,9 +507,7 @@ def live(ctx,
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the exchange bundle to ingest.',
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
 )
 @click.option(
     '-f',
@@ -520,7 +565,8 @@ def live(ctx,
     default=False,
     help='Report potential anomalies found in data bundles.'
 )
-def ingest_exchange(exchange_name, data_frequency, start, end,
+@click.pass_context
+def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
                     include_symbols, exclude_symbols, csv, show_progress,
                     verbose, validate):
     """
@@ -565,9 +611,7 @@ def clean_algo(ctx, algo_namespace):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the exchange bundle to ingest.',
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
 )
 @click.option(
     '-f',
@@ -606,9 +650,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
 @click.option(
     '-x',
     '--exchange-name',
-    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the exchange bundle to ingest.',
-    help='The name of the exchange bundle to ingest (supported: bitfinex,'
-         ' bittrex, poloniex).',
 )
 @click.option(
     '-c',

catalyst/algorithm.py

@@ -124,7 +124,6 @@ from catalyst.utils.events import (
 from catalyst.utils.factory import create_simulation_parameters
 from catalyst.utils.math_utils import (
     tolerant_equals,
-    round_if_near_integer,
     round_nearest
 )
 from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
@@ -1485,7 +1484,6 @@ class TradingAlgorithm(object):
         """
         Converts the number of shares to the smallest tradable lot size for
         the asset being ordered.
-        
         """
         return round_nearest(amount, asset.min_trade_size)
 
@@ -1523,6 +1521,7 @@ class TradingAlgorithm(object):
                              self.updated_portfolio(),
                              self.get_datetime(),
                              self.trading_client.current_data)
+
     @staticmethod
     def __convert_order_params_for_blotter(limit_price, stop_price, style):
         """

catalyst/assets/_assets.pyx

@@ -17,6 +17,7 @@
 """
 Cythonized Asset object.
 """
+
 import hashlib
 
 cimport cython
@@ -38,7 +39,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
-from catalyst.exchange.exchange_utils import get_sid
+from catalyst.exchange.utils.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
@@ -405,6 +406,9 @@ cdef class TradingPair(Asset):
     cdef readonly float taker
     cdef readonly int trading_state
     cdef readonly object data_source
+    cdef readonly float max_trade_size
+    cdef readonly float lot
+    cdef readonly int decimals
 
     _kwargnames = frozenset({
         'sid',
@@ -423,10 +427,13 @@ cdef class TradingPair(Asset):
         'end_minute',
         'exchange_symbol',
         'min_trade_size',
+        'max_trade_size',
+        'lot',
         'maker',
         'taker',
         'trading_state',
-        'data_source'
+        'data_source',
+        'decimals'
     })
     def __init__(self,
                  object symbol,
@@ -443,8 +450,11 @@ cdef class TradingPair(Asset):
                  object auto_close_date=None,
                  object exchange_full=None,
                  float min_trade_size=0.0001,
+                 float max_trade_size=1000000,
                  float maker=0.0015,
                  float taker=0.0025,
+                 float lot=0,
+                 int decimals = 8,
                  int trading_state=0,
                  object data_source='catalyst'):
         """
@@ -509,9 +519,12 @@ cdef class TradingPair(Asset):
         :param auto_close_date:
         :param exchange_full:
         :param min_trade_size:
+        :param max_trade_size:
         :param maker:
         :param taker:
         :param data_source
+        :param decimals
+        :param lot
         """
 
         symbol = symbol.lower()
@@ -535,6 +548,9 @@ cdef class TradingPair(Asset):
         if end_date is None:
             end_date = pd.Timestamp.utcnow() + timedelta(days=365)
 
+        if lot == 0 and min_trade_size > 0:
+            lot = min_trade_size
+
         super().__init__(
             sid,
             exchange,
@@ -556,6 +572,9 @@ cdef class TradingPair(Asset):
         self.exchange_symbol = exchange_symbol
         self.trading_state = trading_state
         self.data_source = data_source
+        self.max_trade_size = max_trade_size
+        self.lot = lot
+        self.decimals = decimals
 
     def __repr__(self):
         return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
@@ -582,6 +601,7 @@ cdef class TradingPair(Asset):
         """
         Convert to a python dict.
         """
+        #TODO: missing fields
         super_dict = super(TradingPair, self).to_dict()
         super_dict['end_daily'] = self.end_daily
         super_dict['end_minute'] = self.end_minute
@@ -610,6 +630,7 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
+        #TODO: make sure that all fields set there
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
@@ -620,7 +641,12 @@ cdef class TradingPair(Asset):
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
-                                 self.min_trade_size))
+                                 self.min_trade_size,
+                                 self.max_trade_size,
+                                 self.lot,
+                                 self.decimals,
+                                 self.taker,
+                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -7,8 +7,7 @@ import logbook
     For example, if you want to see the DEBUG messages, run:
     $ export CATALYST_LOG_LEVEL=10
 '''
-# LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
+LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
-LOG_LEVEL = logbook.DEBUG
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'

catalyst/curate/poloniex.py

@@ -1,9 +1,16 @@
-import json, time, csv
+import csv
+import json
+import os
+import shutil
+import time
 from datetime import datetime
-import pandas as pd
-import os, time, shutil, requests, logbook
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
+import logbook
+import pandas as pd
+import requests
+
+from catalyst.exchange.utils.exchange_utils import \
+    get_exchange_symbols_filename
 
 DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
 DT_END = pd.to_datetime('today').value // 10 ** 9
@@ -13,6 +20,7 @@ CONN_RETRIES    = 2
 logbook.StderrHandler().push_application()
 log = logbook.Logger(__name__)
 
+
 class PoloniexCurator(object):
     '''
     OHLCV data feed generator for crypto data. Based on Poloniex market data
@@ -30,7 +38,6 @@ class PoloniexCurator(object):
                             CSV_OUT_FOLDER))
                 log.exception(e)
 
-
     def get_currency_pairs(self):
         '''
         Retrieves and returns all currency pairs from the exchange
@@ -54,8 +61,6 @@ class PoloniexCurator(object):
                     len(self.currency_pairs)
                     ))
 
-
-
     def _retrieve_tradeID_date(self, row):
         '''
         Helper function that reads tradeID and date fields from CSV readline
@@ -65,7 +70,6 @@ class PoloniexCurator(object):
                            infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
-
     def retrieve_trade_history(self, currencyPair, start=DT_START,
                                end=DT_END, temp=None):
         '''
@@ -90,18 +94,27 @@ class PoloniexCurator(object):
                 f.seek(0, os.SEEK_END)
                 if(f.tell() > 2):                   # Check file size is not 0
                     f.seek(0)                       # Go to start to read
-                    last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
+                    last_tradeID, end_file = self._retrieve_tradeID_date(
+                                                    f.readline())
                     f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
                     while f.read(1) != b"\n":       # Until EOL is found...
-                        f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
+                        # ...jump back the read byte plus one more.
-                    first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
+                        f.seek(-2, os.SEEK_CUR)
+                    first_tradeID, start_file = self._retrieve_tradeID_date(
+                                                    f.readline())
 
-                    if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
+                    if(end_file + 3600 * 6 > DT_END
-                        or (currencyPair == 'BTC_HUC' and first_tradeID ==     2) 
+                        and (first_tradeID == 1
-                        or (currencyPair == 'BTC_RIC' and first_tradeID ==     2) 
+                             or (currencyPair == 'BTC_HUC'
-                        or (currencyPair == 'BTC_XCP' and first_tradeID ==     2)
+                                 and first_tradeID == 2)
-                        or (currencyPair == 'BTC_NAV' and first_tradeID ==  4569)
+                             or (currencyPair == 'BTC_RIC'
-                        or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_XCP'
+                                 and first_tradeID == 2)
+                             or (currencyPair == 'BTC_NAV'
+                                 and first_tradeID == 4569)
+                             or (currencyPair == 'BTC_POT'
+                                 and first_tradeID == 23511))):
                         return
 
         except Exception as e:
@@ -129,7 +142,6 @@ class PoloniexCurator(object):
                     start=str(newstart),
                     end=str(end)
                 )
-        print url
 
         attempts = 0
         success = 0
@@ -137,14 +149,14 @@ class PoloniexCurator(object):
             try:
                 response = requests.get(url)
             except Exception as e:
-                log.error('Failed to retrieve trade history data for {}'.format(
+                log.error('Failed to retrieve trade history data'
-                            currencyPair
+                          'for {}'.format(currencyPair))
-                            ))
                 log.exception(e)
                 attempts += 1
             else:
                 try:
-                    if isinstance(response.json(), dict) and response.json()['error']:
+                    if(isinstance(response.json(), dict)
+                       and response.json()['error']):
                         log.error('Failed to to retrieve trade history data '
                                   'for {}: {}'.format(
                                     currencyPair,
@@ -161,7 +173,6 @@ class PoloniexCurator(object):
         if not success:
             return None
 
-
         '''
         If we get to transactionId == 1, and we already have that on
         disk, we got to the end of TradeHistory for this coin.
@@ -182,7 +193,8 @@ class PoloniexCurator(object):
                    for this currencyPair
         '''
         try:
-            if( 'end_file' in locals() and end_file + 3600 < end):
+            if(temp is not None
+                    or ('end_file' in locals() and end_file + 3600 < end)):
                 if (temp is None):
                     temp = os.tmpfile()
                 tempcsv = csv.writer(temp)
@@ -196,11 +208,12 @@ class PoloniexCurator(object):
                         item['rate'],
                         item['amount'],
                         item['total'],
-                        item['globalTradeID']        
+                        item['globalTradeID'],
                     ])
                 if(response.json()[-1]['tradeID'] > last_tradeID):
                     end = pd.to_datetime(response.json()[-1]['date'],
-                                infer_datetime_format=True).value // 10 ** 9
+                                         infer_datetime_format=True
+                                         ).value // 10**9
                     self.retrieve_trade_history(currencyPair, start,
                                                 end, temp=temp)
                 else:
@@ -240,8 +253,6 @@ class PoloniexCurator(object):
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
-
-
     def generate_ohlcv(self, df):
         '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
@@ -251,15 +262,13 @@ class PoloniexCurator(object):
         vol = df['total'].to_frame('volume')           # set Vol aside
         df.drop('total', axis=1, inplace=True)         # Drop volume data
         ohlc = df.resample('T').ohlc()                 # Resample OHLC 1min
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Raname columns by dropping 'rate'
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Rename cols
-        closes = ohlc['close'].fillna(method='pad')      # Pad fwd missing 'close'
+        closes = ohlc['close'].fillna(method='pad')    # Pad fwd missing close
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))    # Fill N/A with last close
+        ohlc = ohlc.apply(lambda x: x.fillna(closes))  # Fill NA w/ last close
         vol = vol.resample('T').sum().fillna(0)        # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)            # Concatenate OHLC + Vol
+        ohlcv = pd.concat([ohlc, vol], axis=1)         # Concat OHLC + Vol
         return ohlcv
 
-
-
     def write_ohlcv_file(self, currencyPair):
         '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
@@ -305,12 +314,10 @@ class PoloniexCurator(object):
                             item.volume,
                         ])
             except Exception as e:
-                log.error('Error opening {}'.format(csv_fn))
+                log.error('Error opening {}'.format(csv_1min))
                 log.exception(e)
             log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
 
-
-
     def onemin_to_dataframe(self, currencyPair, start, end):
         '''
         Returns a data frame for a given currencyPair from data on disk
@@ -321,13 +328,11 @@ class PoloniexCurator(object):
                                         'high',
                                         'low',
                                         'close',
-                                                'volume']
+                                        'volume'])
-                                )
         df['date'] = pd.to_datetime(df['date'], unit='s')
         df.set_index('date', inplace=True)
         return df[start:end]
 
-
     def generate_symbols_json(self, filename=None):
         '''
         Generates a symbols.json file with corresponding start_date
@@ -342,13 +347,15 @@ class PoloniexCurator(object):
             for currencyPair in self.currency_pairs:
                 start = None
                 csv_fn = '{}crypto_trades-{}.csv'.format(
-                                CSV_OUT_FOLDER, currencyPair)
+                                                    CSV_OUT_FOLDER,
+                                                    currencyPair)
                 with open(csv_fn, 'r') as f:
                     f.seek(0, os.SEEK_END)
                     if(f.tell() > 2):               # Check file size is not 0
                         f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
                         while f.read(1) != b"\n":   # Until EOL is found...
-                            f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
+                            # ...jump back the read byte plus one more.
+                            f.seek(-2, os.SEEK_CUR)
                         start = pd.to_datetime(f.readline().split(',')[1],
                                                infer_datetime_format=True)
 
@@ -373,4 +380,3 @@ if __name__ == '__main__':
         pc.retrieve_trade_history(currencyPair)
         log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
-

catalyst/data/bundles/__init__.py

@@ -1,6 +1,5 @@
 # These imports are necessary to force module-scope register calls to happen.
 from . import quandl  # noqa
-from . import poloniex
 from .core import (
     UnknownBundle,
     bundles,

catalyst/data/bundles/base.py

@@ -13,10 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from itertools import count
 import tarfile
-from time import time, sleep
+from time import sleep
 
 from abc import abstractmethod, abstractproperty
 import logbook
@@ -37,6 +36,7 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 
+
 class BaseBundle(object):
     def __init__(self, asset_filter=[]):
         self._asset_filter = asset_filter
@@ -128,7 +128,7 @@ class BaseBundle(object):
             retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
 
             if is_compile:
-                # User has instructed local compilation and ingestion of bundle.
+                # User has instructed local compilation & ingestion of bundle.
                 # Fetch raw metadata for all symbols.
                 raw_metadata = self._fetch_metadata_frame(
                     api_key,
@@ -157,9 +157,9 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # Post-process metadata using cached symbol frames, and write to
+                # Post-process metadata using cached symbol frames, and write
-                # disk.  This metadata must be written before any attempt to write
+                # to disk.  This metadata must be written before any attempt
-                # minute data.
+                # to write minute data.
                 metadata = self._post_process_metadata(
                     raw_metadata,
                     cache,
@@ -184,10 +184,11 @@ class BaseBundle(object):
                         show_progress=show_progress,
                     )
 
-                # For legacy purposes, this call is required to ensure the database
+                # For legacy purposes, this call is required to ensure the
-                # contains an appropriately initialized file structure.  We don't
+                # database contains an appropriately initialized file
-                # forsee a usecase for adjustments at this time, but may later
+                # structure.  We don't forsee a usecase for adjustments at
-                # choose to expose this functionality in the future.
+                # this time, but may later choose to expose this functionality
+                # in the future.
                 adjustment_writer.write(
                     splits=(
                         pd.concat(self.splits, ignore_index=True)
@@ -269,10 +270,10 @@ class BaseBundle(object):
                             page_number,
                         )
                         break
-                    except ValueError as e:
+                    except ValueError:
                         raw = pd.DataFrame([])
                         break
-                    except Exception as e:
+                    except Exception:
                         log.exception(
                             'Failed to load metadata from {}. '
                             'Retrying.'.format(self.name)
@@ -283,7 +284,6 @@ class BaseBundle(object):
                         'attempts.'.format(page_number, retries)
                     )
 
-
             if raw.empty:
                 # Empty DataFrame signals completion.
                 break
@@ -318,16 +318,16 @@ class BaseBundle(object):
             show_percent=False,
         ) as symbols_map:
             for asset_id, symbol in symbols_map:
-                # Attempt to load data from disk, the cache should have an entry
+                # Attempt to load data from disk, the cache should have an
-                # for each symbol at this point of the execution. If one does
+                # entry for each symbol at this point of the execution. If one
-                # not exist, we should fail.
+                # does not exist, we should fail.
                 key = '{sym}.daily.frame'.format(sym=symbol)
                 try:
                     raw_data = cache[key]
                 except KeyError:
                     raise ValueError(
-                      'Unable to find cached data for symbol: {0}'.format(symbol)
+                      'Unable to find cached data for symbol:'
-                    )
+                      ' {0}'.format(symbol))
 
                 # Perform and require post-processing of metadata.
                 final_symbol_metadata = self.post_process_symbol_metadata(
@@ -363,8 +363,8 @@ class BaseBundle(object):
             # returns the cached data unaltered.  The `should_sleep` flag
             # indicates that an API call was attempted, and that we should be
             # ensure aren't exceeding our rate limit before proceeding to the
-            # next symbol. If the raw_data is updated, it is cached before being
+            # next symbol. If the raw_data is updated, it is cached before
-            # returned.
+            # being returned.
             raw_data, should_sleep = self._maybe_update_symbol_frame(
                 start_time,
                 api_key,
@@ -468,7 +468,6 @@ class BaseBundle(object):
                     data_frequency,
                 )
                 raw_data.index = pd.to_datetime(raw_data.index, utc=True)
-                #raw_data.index = raw_data.index.tz_localize('UTC')
 
                 # Filter incoming data to fit start and end sessions.
                 raw_data = raw_data[
@@ -482,7 +481,7 @@ class BaseBundle(object):
 
                 return raw_data
 
-            except Exception as e:
+            except Exception:
                 log.exception(
                     'Exception raised fetching {name} data. Retrying.'
                     .format(name=self.name)

catalyst/data/bundles/base_pricing.py

@@ -16,6 +16,7 @@
 from catalyst.data.bundles.base import BaseBundle
 from catalyst.utils.memoize import lazyval
 
+
 class BasePricingBundle(BaseBundle):
     @lazyval
     def md_dtypes(self):
@@ -38,6 +39,7 @@ class BasePricingBundle(BaseBundle):
             ('volume', 'float64'),
         ]
 
+
 class BaseCryptoPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):
@@ -55,6 +57,7 @@ class BaseCryptoPricingBundle(BasePricingBundle):
     def dividends(self):
         return []
 
+
 class BaseEquityPricingBundle(BasePricingBundle):
     @lazyval
     def calendar_name(self):

catalyst/data/bundles/core.py

@@ -37,6 +37,7 @@ from catalyst.utils.cli import maybe_show_progress
 
 ONE_MEGABYTE = 1024 * 1024
 
+
 def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
     return pth.data_path(
         asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -135,6 +136,7 @@ def ingestions_for_bundle(bundle, environ=None):
         reverse=True,
     )
 
+
 def download_with_progress(url, chunk_size, **progress_kwargs):
     """
     Download streaming data from a URL, printing progress information to the
@@ -705,4 +707,5 @@ def _make_bundle_core():
     )
 
 
-bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
+bundles, register_bundle, register, unregister, ingest, load, clean = \
+    _make_bundle_core()

catalyst/data/bundles/poloniex.py

@@ -14,19 +14,17 @@
 # limitations under the License.
 
 import sys
-
+from six.moves.urllib.parse import urlencode
-from datetime import datetime
 
 import pandas as pd
 
-from six.moves.urllib.parse import urlencode
-
 from catalyst.data.bundles.core import register_bundle
 from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
 from catalyst.utils.memoize import lazyval
 
 from catalyst.curate.poloniex import PoloniexCurator
 
+
 class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def name(self):
@@ -46,7 +44,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def tar_url(self):
         return (
-            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
+            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
+            'poloniex/poloniex-bundle.tar.gz'
         )
 
     @lazyval
@@ -72,7 +71,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         )
 
         raw = raw[raw['isFrozen'] == 0]
-
         return raw
 
     def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
@@ -98,7 +96,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                                frequency):
 
         # TODO: replace this with direct exchange call
-        # The end date and frequency should be used to calculate the number of bars
+        # The end date and frequency should be used to
+        # calculate the number of bars
         if(frequency == 'minute'):
             pc = PoloniexCurator()
             raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
@@ -116,8 +115,9 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             )
             raw.set_index('date', inplace=True)
 
-        # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
+        # BcolzDailyBarReader introduces a 1/1000 factor in the way
-        # on disk, which we compensate here to get the right pricing amounts
+        # pricing is stored on disk, which we compensate here to get
+        # the right pricing amounts
         # ref: data/us_equity_pricing.py
         scale = 1
         raw.loc[:, 'open'] /= scale
@@ -139,7 +139,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
 
         return self._format_polo_query(query_params)
 
-
     def _format_data_url(self,
                          api_key,
                          symbol,
@@ -171,6 +170,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
             query=urlencode(query_params),
         )
 
+
 '''
 As a second parameter, you can pass an array of currency pairs
 that will be processed as an asset_filter to only process that
@@ -180,9 +180,7 @@ register_bundle(PoloniexBundle, ['USDT_BTC',])
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-
 if 'ingest' in sys.argv and '-c' in sys.argv:
     register_bundle(PoloniexBundle)
 else:
     register_bundle(PoloniexBundle, create_writers=False)
-

catalyst/data/bundles/quandl.py

@@ -16,7 +16,6 @@
 from datetime import datetime
 
 import pandas as pd
-
 from six.moves.urllib.parse import urlencode
 
 from catalyst.data.bundles.core import register_bundle
@@ -26,25 +25,16 @@ from catalyst.utils.memoize import lazyval
 """
 Module for building a complete daily dataset from Quandl's WIKI dataset.
 """
-from itertools import count
-import tarfile
-from time import time, sleep
-from datetime import datetime
-
 from logbook import Logger
-import pandas as pd
-from six.moves.urllib.parse import urlencode
-
-from catalyst.utils.calendars import register_calendar_alias
-from catalyst.utils.cli import maybe_show_progress
-
-from . import core as bundles
 
 from catalyst.constants import LOG_LEVEL
+from catalyst.utils.calendars import register_calendar_alias
+
 
 log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
+
 class QuandlBundle(BaseEquityPricingBundle):
     @lazyval
     def name(self):
@@ -109,8 +99,8 @@ class QuandlBundle(BaseEquityPricingBundle):
         # Filter out invalid symbols
         raw = raw[~raw.symbol.isin(self._excluded_symbols)]
 
-        # cut out all the other stuff in the name column
+        # cut out all the other stuff in the name column. We need to
-        # we need to escape the paren because it is actually splitting on a regex
+        # escape the paren because it is actually splitting on a regex
         raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
 
         return raw
@@ -175,7 +165,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['sid'] = asset_id
         self.splits.append(df)
 
-
     def _update_dividends(self, asset_id, raw_data):
         divs = raw_data.ex_dividend
         df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -186,7 +175,6 @@ class QuandlBundle(BaseEquityPricingBundle):
         df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
         self.dividends.append(df)
 
-
     def _format_metadata_url(self, api_key, page_number):
         """Build the query RL for the quandl WIKI metadata.
         """
@@ -200,10 +188,10 @@ class QuandlBundle(BaseEquityPricingBundle):
             query_params = [('api_key', api_key)] + query_params
 
         return (
-            'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
+            'https://www.quandl.com/api/v3/datasets.csv?'
+            + urlencode(query_params)
         )
 
-
     def _format_wiki_url(self,
                          api_key,
                          symbol,
@@ -229,5 +217,6 @@ class QuandlBundle(BaseEquityPricingBundle):
             )
         )
 
+
 register_calendar_alias('QUANDL', 'NYSE')
 register_bundle(QuandlBundle)

catalyst/data/dispatch_bar_reader.py

@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
         if self._last_available_dt is not None:
             return self._last_available_dt
         else:
-            return min(r.last_available_dt for r in self._readers.values())
+            return min(r.last_available_dt for r in list(self._readers.values()))
 
     @lazyval
     def first_trading_day(self):
-        return max(r.first_trading_day for r in self._readers.values())
+        return max(r.first_trading_day for r in list(self._readers.values()))
 
     def get_value(self, sid, dt, field):
         asset = self._asset_finder.retrieve_asset(sid)
@@ -133,11 +133,13 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
 
         return results
 
+
 class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):
         return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
 
+
 class AssetDispatchSessionBarReader(AssetDispatchBarReader):
 
     def _dt_window_size(self, start_dt, end_dt):

catalyst/data/loader.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import datetime
 import os
 from collections import OrderedDict
 
@@ -23,6 +22,7 @@ from pandas_datareader.data import DataReader
 from six import iteritems
 from six.moves.urllib_error import HTTPError
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.utils.calendars import get_calendar
 from . import treasuries, treasuries_can
 from .benchmarks import get_benchmark_returns
@@ -32,8 +32,6 @@ from ..utils.paths import (
     data_root,
 )
 
-from catalyst.constants import LOG_LEVEL
-
 logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
@@ -103,7 +101,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         trading_day = get_calendar('OPEN').trading_day
 
     # TODO: consider making configurable
-    bm_symbol = 'btc_usdt'
+    bm_symbol = 'btc_usd'
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
@@ -129,11 +127,13 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # before this date.
     '''
     if(bundle_data):
-        # If we are using the bundle to retrieve the cryptobenchmark, find the last
+        # If we are using the bundle to retrieve the cryptobenchmark, find
-        # date for which there is trading data in the bundle
+        # the last date for which there is trading data in the bundle
-        asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
+        asset = bundle_data.asset_finder.lookup_symbol(
+                    symbol=bm_symbol,as_of_date=None)
         ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
-        last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
+        last_date = pd.to_datetime(
+                    bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
@@ -142,10 +142,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.factory import get_exchange
+        from catalyst.exchange.utils.factory import get_exchange
         exchange = get_exchange(
-            exchange_name='poloniex', base_currency='usdt'
+            exchange_name='bitfinex', base_currency='usd'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 
@@ -164,8 +165,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     br.loc[start_dt] = 0
     br = br.sort_index()
 
-    # Override first_date for treasury data since we have it for many more years
+    # Override first_date for treasury data since we have it for many more
-    # and is independent of crypto data
+    # years and is independent of crypto data
     first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
     tc = ensure_treasury_data(
         bm_symbol,
@@ -301,14 +302,14 @@ def ensure_crypto_benchmark_data(symbol,
 
     if (bundle == 'poloniex'):
         '''
-        If we're using the Poloniex bundle, we'll get the benchmark from the bundle
+        If we're using the Poloniex bundle, we'll get the benchmark from the
-        instead of downloading it from Poloniex every time we need it.
+        bundle instead of downloading it from Poloniex every time we need it.
-        Poloniex has a captcha for API queries originating from outside the US that 
+        Poloniex has a captcha for API queries originating from outside the US
-        prevents users abroad from getting Catalyst to work
+        that prevents users abroad from getting Catalyst to work
         '''
         logger.info(
-            (
+            ('Retrieving benchmark data from bundle for {symbol!r}'
-                'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
@@ -331,10 +332,11 @@ def ensure_crypto_benchmark_data(symbol,
 
     else:
         # This is how it used to be: downloading the benchmark everytime.
-        # Leaving this code here to be repurposed in the future for other bundles.
+        # Leaving this code here to be repurposed in the future for
+        # other bundles.
         logger.info(
-            (
+            ('Downloading benchmark data for {symbol!r}'
-                'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
+             ' from {first_date} to {last_date}'),
             symbol=symbol, first_date=first_date, last_date=last_date)
 
         raise DeprecationWarning('poloniex bundle deprecated')
@@ -431,67 +433,6 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
     return data
 
 
-def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
-                          environ=None):
-    """
-    Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
-
-    Parameters
-    ----------
-    symbol : str
-        The symbol for the benchmark to load.
-    first_date : pd.Timestamp
-        First required date for the cache.
-    last_date : pd.Timestamp
-        Last required date for the cache.
-    now : pd.Timestamp
-        The current time.  This is used to prevent repeated attempts to
-        re-download data that isn't available due to scheduling quirks or other
-        failures.
-    trading_day : pd.CustomBusinessDay
-        A trading day delta.  Used to find the day before first_date so we can
-        get the close of the day prior to first_date.
-
-    We attempt to download data unless we already have data stored at the data
-    cache for `symbol` whose first entry is before or on `first_date` and whose
-    last entry is on or after `last_date`.
-
-    If we perform a download and the cache criteria are not satisfied, we wait
-    at least one hour before attempting a redownload.  This is determined by
-    comparing the current time to the result of os.path.getmtime on the cache
-    path.
-    """
-    filename = get_benchmark_filename(symbol)
-    data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
-                             environ)
-    if data is not None:
-        return data
-
-    # If no cached data was found or it was missing any dates then download the
-    # necessary data.
-    logger.info(
-        ('Downloading benchmark data for {symbol!r} '
-         'from {first_date} to {last_date}'),
-        symbol=symbol,
-        first_date=first_date - trading_day,
-        last_date=last_date
-    )
-
-    try:
-        data = get_benchmark_returns(
-            symbol,
-            first_date - trading_day,
-            last_date,
-        )
-        data.to_csv(get_data_filepath(filename, environ))
-    except (OSError, IOError, HTTPError):
-        logger.exception('Failed to cache the new benchmark returns')
-        raise
-    if not has_data_for_dates(data, first_date, last_date):
-        logger.warn("Still don't have expected data after redownload!")
-    return data
-
-
 def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
     """
     Ensure we have treasury data from treasury module associated with

catalyst/data/minute_bars.py

@@ -341,11 +341,9 @@ class BcolzMinuteBarMetadata(object):
             'end_session': str(self.end_session.date()),
             # Write these values for backwards compatibility
             'first_trading_day': str(self.start_session.date()),
-            'market_opens': (
+            'market_opens': (market_opens.values.astype('datetime64[m]').
-                market_opens.values.astype('datetime64[m]').
                              astype(np.int64).tolist()),
-            'market_closes': (
+            'market_closes': (market_closes.values.astype('datetime64[m]').
-                market_closes.values.astype('datetime64[m]').
                               astype(np.int64).tolist()),
         }
         with open(self.metadata_path(rootdir), 'w+') as fp:
@@ -1256,8 +1254,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 values = carray[start_idx:end_idx + 1]
                 if indices_to_exclude is not None:
                     for excl_start, excl_stop in indices_to_exclude[::-1]:
-                        excl_slice = np.s_[
+                        excl_slice = np.s_[excl_start - start_idx:excl_stop
-                                     excl_start - start_idx:excl_stop - start_idx + 1]
+                                           - start_idx + 1]
                         values = np.delete(values, excl_slice)
 
                 where = values != 0
@@ -1321,8 +1319,7 @@ class H5MinuteBarUpdateWriter(object):
     def __init__(self, path, complevel=None, complib=None):
         self._complevel = complevel if complevel \
                             is not None else self._COMPLEVEL
-        self._complib = complib if complib \
+        self._complib = complib if complib is not None else self._COMPLIB
-                                   is not None else self._COMPLIB
         self._path = path
 
     def write(self, frames):

catalyst/data/us_equity_pricing.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division     # Python2 req to have division of ints yield float
+from __future__ import division  # Python2 req for division of ints yield float
 
 from errno import ENOENT
 from functools import partial
@@ -120,7 +120,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
-PRICE_ADJUSTMENT_FACTOR = 1000000000    # Provides 9 decimals resolution. Also affects _equities.pyx L220
+# Provides 9 decimals resolution. Also affects _equities.pyx L220
+PRICE_ADJUSTMENT_FACTOR = 1000000000
 
 
 def check_uint32_safe(value, colname):
@@ -130,6 +131,7 @@ def check_uint32_safe(value, colname):
             "for uint32" % (value, colname)
         )
 
+
 def check_uint64_safe(value, colname):
     if value >= UINT64_MAX:
         raise ValueError(
@@ -439,11 +441,13 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed = (raw_data[list(OHLC)]
+                     * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
-        processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
+        processed['volume'] = (raw_data.volume
+                               * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         return ctable.fromdataframe(processed)
 
 

catalyst/examples/arbitrage_with_interface.py

@@ -6,7 +6,7 @@ from catalyst.api import (
     symbol,
     get_open_orders
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'arbitrage_eth_btc'
@@ -263,13 +263,20 @@ def analyze(context, stats):
     pass
 
 
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'live'
+    if MODE == 'live':
         run_algorithm(
+            capital_base=0.1,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
             exchange_name='poloniex,bitfinex',
             live=True,
             algo_namespace=algo_namespace,
-    quote_currency='btc',
+            base_currency='btc',
-    live_graph=False
+            live_graph=False,
+            simulate_orders=True,
+            stats_output=None,
         )

catalyst/examples/buy_and_hodl.py

@@ -23,7 +23,7 @@ from catalyst.api import (order_target_value, symbol, record,
 
 
 def initialize(context):
-    context.ASSET_NAME = 'btc_usd'
+    context.ASSET_NAME = 'btc_usdt'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
@@ -61,7 +61,6 @@ def handle_data(context, data):
             context.asset,
             target_hodl_value,
             limit_price=price * 1.1,
-            stop_price=price * 0.9,
         )
 
     record(
@@ -141,9 +140,9 @@ if __name__ == '__main__':
         initialize=initialize,
         handle_data=handle_data,
         analyze=analyze,
-        exchange_name='bitfinex',
+        exchange_name='poloniex',
         algo_namespace='buy_and_hodl',
-        base_currency='usd',
+        base_currency='usdt',
         start=pd.to_datetime('2015-03-01', utc=True),
         end=pd.to_datetime('2017-10-31', utc=True),
     )

catalyst/examples/buy_btc_simple.py

@@ -4,7 +4,8 @@
 
     Run this example, by executing the following from your terminal:
       catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
-	  catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
 
     If you want to run this code using another exchange, make sure that
     the asset is available on that exchange. For example, if you were to run
@@ -14,17 +15,35 @@
 
     and specify exchange poloniex as follows:
     catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
-	catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
+        --end 2017-9-30 -o buy_btc_simple_out.pickle
 
     To see which assets are available on each exchange, visit:
     https://www.enigma.co/catalyst/status
 '''
-
+from catalyst import run_algorithm
 from catalyst.api import order, record, symbol
+import pandas as pd
+
 
 def initialize(context):
-    context.asset = symbol('btc_usd')
+    context.asset = symbol('btc_usdt')
+
 
 def handle_data(context, data):
     order(context.asset, 1)
     record(btc=data.current(context.asset, 'price'))
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='poloniex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usdt',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_low_sell_high.py

@@ -1,15 +1,5 @@
-'''
-This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
-Install it first by running: 
-$ pip install TA-Lib
-
-If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
-it typically means that it can't find the underlying TA-Lib library and needs to be installed.
-See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install 
-the required dependencies.
-'''
-
 import talib
+import pandas as pd
 from logbook import Logger
 
 from catalyst.api import (
@@ -19,58 +9,52 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+from catalyst.utils.run_algo import run_algorithm
 
-algo_namespace = 'buy_low_sell_high_xrp'
+algo_namespace = 'buy_the_dip_live'
-log = Logger(algo_namespace)
+log = Logger('buy low sell high')
 
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USDT'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 5000
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.05
+    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.swallow_errors = True
 
     context.errors = []
     pass
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1D'
     )
-
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
     elif rsi <= 70:
-        buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -100,8 +84,8 @@ def _handle_data(context, data):
 
         if price < cost_basis:
             is_buy = True
-        elif position.amount > 0 and \
+        elif (position.amount > 0
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
+              and price > cost_basis * (1 + context.PROFIT_TARGET)):
             profit = (price * position.amount) - (cost_basis * position.amount)
             log.info('closing position, taking profit: {}'.format(profit))
             order_target_percent(
@@ -138,11 +122,11 @@ def _handle_data(context, data):
 
 def handle_data(context, data):
     log.info('handling bar {}'.format(data.current_dt))
-    try:
+    # try:
     _handle_data(context, data)
-    except Exception as e:
+    # except Exception as e:
-        log.warn('aborting the bar on error {}'.format(e))
+    #     log.warn('aborting the bar on error {}'.format(e))
-        context.errors.append(e)
+    #     context.errors.append(e)
 
     log.info('completed bar {}, total execution errors {}'.format(
         data.current_dt,
@@ -156,3 +140,32 @@ def handle_data(context, data):
 def analyze(context, stats):
     log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
     pass
+
+
+if __name__ == '__main__':
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.001,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='binance',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='btc',
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace='buy_and_hodl',
+            base_currency='usdt',
+            start=pd.to_datetime('2015-03-01', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )

catalyst/examples/buy_low_sell_high_live.py

@@ -1,168 +0,0 @@
-import talib
-from logbook import Logger
-
-import pandas as pd
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_the_dip_live'
-log = Logger('buy low sell high')
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.ASSET_NAME = 'btc_usdt'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.TARGET_POSITIONS = 30
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='1d'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.2
-    else:
-        buy_increment = 0.1
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(
-        price=price,
-        rsi=rsi,
-    )
-
-    orders = get_open_orders(context.asset)
-    if orders:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            log.info('the rsi is too high to consider buying {}'.format(rsi))
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
-        )
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-    pass
-
-
-run_algorithm(
-    capital_base=100000,
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='poloniex',
-    start=pd.to_datetime('2017-5-01', utc=True),
-    end=pd.to_datetime('2017-10-16', utc=True),
-    base_currency='usdt',
-    data_frequency='daily'
-)
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='poloniex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc'
-# )

catalyst/examples/dual_moving_average.py

@@ -1,16 +1,17 @@
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from logbook import Logger
-import matplotlib.pyplot as plt
 
 from catalyst import run_algorithm
-from catalyst.api import (order, record, symbol, order_target_percent,
+from catalyst.api import (record, symbol, order_target_percent,
                           get_open_orders)
-from catalyst.exchange.stats_utils import extract_transactions
+from catalyst.exchange.utils.stats_utils import extract_transactions
 
 NAMESPACE = 'dual_moving_average'
 log = Logger(NAMESPACE)
 
+
 def initialize(context):
     context.i = 0
     context.asset = symbol('ltc_usd')
@@ -31,10 +32,16 @@ def handle_data(context, data):
     # moving average with the appropriate parameters. We choose to use
     # minute bars for this simulation -> freq="1m"
     # Returns a pandas dataframe.
-    short_mavg = data.history(context.asset, 'price',
+    short_mavg = data.history(context.asset,
-                        bar_count=short_window, frequency="1m").mean()
+                              'price',
-    long_mavg = data.history(context.asset, 'price',
+                              bar_count=short_window,
-                        bar_count=long_window, frequency="1m").mean()
+                              frequency="1m",
+                              ).mean()
+    long_mavg = data.history(context.asset,
+                             'price',
+                             bar_count=long_window,
+                             frequency="1m",
+                             ).mean()
 
     # Let's keep the price of our asset in a more handy variable
     price = data.current(context.asset, 'price')
@@ -77,7 +84,8 @@ def handle_data(context, data):
 def analyze(context, perf):
 
     # Get the base_currency that was passed as a parameter to the simulation
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    exchange = list(context.exchanges.values())[0]
+    base_currency = exchange.base_currency.upper()
 
     # First chart: Plot portfolio value using base_currency
     ax1 = plt.subplot(411)
@@ -89,7 +97,9 @@ def analyze(context, perf):
 
     # Second chart: Plot asset price, moving averages and buys/sells
     ax2 = plt.subplot(412, sharex=ax1)
-    perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
+    perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
+        ax=ax2,
+        label='Price')
     ax2.legend_.remove()
     ax2.set_ylabel('{asset}\n({base})'.format(
         asset=context.asset.symbol,

catalyst/examples/dual_vwap.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2017 Enigma MPC, Inc.
-# Copyright 2014 Quantopian, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from catalyst.api import (
-    order_target_percent,
-    record,
-    symbol,
-    get_open_orders,
-    set_max_leverage,
-    schedule_function,
-    date_rules,
-    attach_pipeline,
-    pipeline_output,
-)
-
-from catalyst.pipeline import Pipeline
-from catalyst.pipeline.data import CryptoPricing
-from catalyst.pipeline.factors.crypto import VWAP
-
-
-def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
-    context.TARGET_INVESTMENT_RATIO = 0.8
-    context.SHORT_WINDOW = 30
-    context.LONG_WINDOW = 100
-
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
-    context.i = 0
-    context.asset = symbol(context.ASSET_NAME)
-
-    set_max_leverage(1.0)
-
-    attach_pipeline(make_pipeline(context), 'vwap_pipeline')
-
-    schedule_function(
-        rebalance,
-        time_rules=times_rules.every_minute(),
-    )
-
-
-def before_trading_start(context, data):
-    context.pipeline_data = pipeline_output('vwap_pipeline')
-
-def make_pipeline(context):
-    return Pipeline(
-        columns={
-            'price': CryptoPricing.open.latest,
-            'volume': CryptoPricing.volume.latest,
-            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
-            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
-        }
-    )
-
-def rebalance(context, data):
-    context.i += 1
-
-    # skip first LONG_WINDOW bars to fill windows
-    if context.i < context.LONG_WINDOW:
-        return
-
-    # get pipeline data for asset of interest
-    pipeline_data = context.pipeline_data
-    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
-
-    # retrieve long and short moving averages from pipeline
-    short_mavg = pipeline_data.short_mavg
-    long_mavg = pipeline_data.long_mavg
-    price = pipeline_data.price
-    volume = pipeline_data.volume
-
-    # check that order has not already been placed
-    open_orders = get_open_orders()
-    if context.asset not in open_orders:
-        # check that the asset of interest can currently be traded
-        if data.can_trade(context.asset):
-            # adjust portfolio based on comparison of long and short vwap
-            if short_mavg > long_mavg:
-                order_target_percent(
-                    context.asset,
-                    context.TARGET_INVESTMENT_RATIO,
-                )
-            elif short_mavg < long_mavg:
-                order_target_percent(
-                    context.asset,
-                    0.0,
-                )
-
-    record(
-        price=price,
-        cash=context.portfolio.cash,
-        leverage=context.account.leverage,
-        short_mavg=short_mavg,
-        long_mavg=long_mavg,
-        volume=volume,
-    )
-    
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(611)
-    results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio value (USD)')
-
-    ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
-
-    trans = results.ix[[t != [] for t in results.transactions]]
-    amounts = [t[0]['amount'] for t in trans.transactions]
-
-    buys = trans.ix[
-        [t[0]['amount'] > 0 for t in trans.transactions]
-    ]
-    sells = trans.ix[
-        [t[0]['amount'] < 0 for t in trans.transactions]
-    ]
-
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        context.TICK_SIZE * results.price[sells.index],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(613, sharex=ax1)
-    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
-    ax3.set_ylabel('Leverage (USD)')
-
-    ax4 = plt.subplot(614, sharex=ax1)
-    results[['cash']].plot(ax=ax4)
-    ax4.set_ylabel('Cash (USD)')
-
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]] = results[[
-        'treasury_period_return',
-        'algorithm_period_return',
-        'benchmark_period_return',
-    ]]
-
-    ax5 = plt.subplot(615, sharex=ax1)
-    results[[
-        'treasury',
-        'algorithm',
-        'benchmark',
-    ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    ax6 = plt.subplot(616, sharex=ax1)
-    results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mBTC/day)')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()

catalyst/examples/mean_reversion_simple.py

@@ -12,7 +12,7 @@ from logbook import Logger
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record, order_target_percent, get_open_orders
-from catalyst.exchange.stats_utils import extract_transactions
+from catalyst.exchange.utils.stats_utils import extract_transactions
 # We give a name to the algorithm which Catalyst will use to persist its state.
 # In this example, Catalyst will create the `.catalyst/data/live_algos`
 # directory. If we stop and start the algorithm, Catalyst will resume its
@@ -33,16 +33,19 @@ def initialize(context):
     # parameters or values you're going to use.
 
     # In our example, we're looking at Neo in Ether.
-    context.neo_eth = symbol('neo_eth')
+    context.market = symbol('eth_btc')
     context.base_price = None
     context.current_day = None
 
     context.RSI_OVERSOLD = 55
-    context.RSI_OVERBOUGHT = 82
+    context.RSI_OVERBOUGHT = 60
-    context.CANDLE_SIZE = '5T'
+    context.CANDLE_SIZE = '15T'
 
     context.start_time = time.time()
 
+    context.set_commission(maker=0.001, taker=0.002)
+    context.set_slippage(spread=0.001)
+
 
 def handle_data(context, data):
     # This handle_data function is where the real work is done.  Our data is
@@ -59,14 +62,14 @@ def handle_data(context, data):
         context.current_day = today
 
     # We're computing the volume-weighted-average-price of the security
-    # defined above, in the context.neo_eth variable.  For this example, we're 
+    # defined above, in the context.market variable.  For this example, we're
     # using three bars on the 15 min bars.
 
     # The frequency attribute determine the bar size. We use this convention
     # for the frequency alias:
     # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
     prices = data.history(
-        context.neo_eth,
+        context.market,
         fields='close',
         bar_count=50,
         frequency=context.CANDLE_SIZE
@@ -81,7 +84,7 @@ def handle_data(context, data):
     # We need a variable for the current price of the security to compare to
     # the average. Since we are requesting two fields, data.current()
     # returns a DataFrame with
-    current = data.current(context.neo_eth, fields=['close', 'volume'])
+    current = data.current(context.market, fields=['close', 'volume'])
     price = current['close']
 
     # If base_price is not set, we use the current value. This is the
@@ -95,34 +98,36 @@ def handle_data(context, data):
     # Now that we've collected all current data for this frame, we use
     # the record() method to save it. This data will be available as
     # a parameter of the analyze() function for further analysis.
+
     record(
-        price=price,
         volume=current['volume'],
+        price=price,
         price_change=price_change,
         rsi=rsi[-1],
         cash=cash
     )
-
     # We are trying to avoid over-trading by limiting our trades to
     # one per day.
     if context.traded_today:
         return
 
+    # TODO: retest with open orders
     # Since we are using limit orders, some orders may not execute immediately
     # we wait until all orders are executed before considering more trades.
-    orders = get_open_orders(context.neo_eth)
+    orders = context.blotter.open_orders
     if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
         return
 
     # Exit if we cannot trade
-    if not data.can_trade(context.neo_eth):
+    if not data.can_trade(context.market):
         return
 
     # Another powerful built-in feature of the Catalyst backtester is the
     # portfolio object.  The portfolio object tracks your positions, cash,
     # cost basis of specific holdings, and more.  In this line, we calculate
     # how long or short our position is at this minute.
-    pos_amount = context.portfolio.positions[context.neo_eth].amount
+    pos_amount = context.portfolio.positions[context.market].amount
 
     if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
         log.info(
@@ -133,7 +138,7 @@ def handle_data(context, data):
         # Set a style for limit orders,
         limit_price = price * 1.005
         order_target_percent(
-            context.neo_eth, 1, limit_price=limit_price
+            context.market, 1, limit_price=limit_price
         )
         context.traded_today = True
 
@@ -145,7 +150,7 @@ def handle_data(context, data):
         )
         limit_price = price * 0.995
         order_target_percent(
-            context.neo_eth, 0, limit_price=limit_price
+            context.market, 0, limit_price=limit_price
         )
         context.traded_today = True
 
@@ -156,7 +161,7 @@ def analyze(context=None, perf=None):
 
     import matplotlib.pyplot as plt
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -168,7 +173,7 @@ def analyze(context=None, perf=None):
     perf.loc[:, 'price'].plot(ax=ax2, label='Price')
 
     ax2.set_ylabel('{asset}\n({base})'.format(
-        asset=context.neo_eth.symbol, base=base_currency
+        asset=context.market.symbol, base=base_currency
     ))
 
     transaction_df = extract_transactions(perf)
@@ -239,9 +244,25 @@ def analyze(context=None, perf=None):
 
 if __name__ == '__main__':
     # The execution mode: backtest or live
-    MODE = 'live'
+    live = True
 
-    if MODE == 'backtest':
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+            # auth_aliases=dict(poloniex='auth2')
+        )
+
+    else:
         folder = os.path.join(
             tempfile.gettempdir(), 'catalyst', NAMESPACE
         )
@@ -249,7 +270,9 @@ if __name__ == '__main__':
 
         timestr = time.strftime('%Y%m%d-%H%M%S')
         out = os.path.join(folder, '{}.p'.format(timestr))
-        # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
         run_algorithm(
             capital_base=0.1,
             data_frequency='minute',
@@ -258,22 +281,9 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out
         )
         log.info('saved perf stats: {}'.format(out))
-
-    elif MODE == 'live':
-        run_algorithm(
-            capital_base=0.1,
-            initialize=initialize,
-            handle_data=handle_data,
-            analyze=analyze,
-            exchange_name='bittrex',
-            live=True,
-            algo_namespace=NAMESPACE,
-            base_currency='eth',
-            live_graph=False
-        )

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/examples/portfolio_optimization.py

@@ -15,11 +15,10 @@ import os
 import pytz
 import numpy as np
 import pandas as pd
-from scipy.optimize import minimize
 import matplotlib.pyplot as plt
 from datetime import datetime
 
-from catalyst.api import record, symbol, symbols, order_target_percent
+from catalyst.api import record, symbols, order_target_percent
 from catalyst.utils.run_algo import run_algorithm
 
 np.set_printoptions(threshold='nan', suppress=True)
@@ -72,7 +71,8 @@ def handle_data(context, data):
             weights /= np.sum(weights)
             w = np.asmatrix(weights)
             p_r = np.sum(np.dot(w, np.transpose(m))) * 365
-           p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
+            p_std = np.sqrt(np.dot(np.dot(w, cov_m),
+                                   np.transpose(w))) * np.sqrt(365)
 
             # store results in results array
             results_array[0, p] = p_r
@@ -87,11 +87,12 @@ def handle_data(context, data):
 
         # convert results array to Pandas DataFrame
         results_frame = pd.DataFrame(np.transpose(results_array),
-                          columns=['r','stdev','sharpe']+context.assets)
+                                     columns=['r', 'stdev', 'sharpe']
+                                             + context.assets)
         # locate position of portfolio with highest Sharpe Ratio
         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
         # locate positon of portfolio with minimum standard deviation
-       min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+        # min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
 
         # order optimal weights for each asset
         for asset in context.assets:
@@ -99,28 +100,43 @@ def handle_data(context, data):
                 order_target_percent(asset, max_sharpe_port[asset])
 
         # create scatter plot coloured by Sharpe Ratio
-       plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
+        plt.scatter(results_frame.stdev,
+                    results_frame.r,
+                    c=results_frame.sharpe,
+                    cmap='RdYlGn')
         plt.xlabel('Volatility')
         plt.ylabel('Returns')
         plt.colorbar()
-       #plot red star to highlight position of portfolio with highest Sharpe Ratio
+        # plot red star to highlight position of portfolio
-       plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
+        # with highest Sharpe Ratio
+        plt.scatter(max_sharpe_port[1],
+                    max_sharpe_port[0],
+                    marker='o',
+                    color='b',
+                    s=200)
         # plot green star to highlight position of minimum variance portfolio
         plt.show()
         print(max_sharpe_port)
-       record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
+        record(pr=pr,
+               r=r,
+               m=m,
+               stds=stds,
+               max_sharpe_port=max_sharpe_port,
+               corr_m=corr_m)
     context.i += 1
 
 
 def analyze(context=None, results=None):
     # Form DataFrame with selected data
-   data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
+    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
+                    'portfolio_value']]
 
     # Save results in CSV file
     filename = os.path.splitext(os.path.basename(__file__))[0]
     data.to_csv(filename + '.csv')
 
 
+if __name__ == '__main__':
     # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
     start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
     end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)

catalyst/examples/rsi_profit_target.py

@@ -11,7 +11,6 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
-from catalyst.exchange.stats_utils import crossover, crossunder
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'rsi'
@@ -55,7 +54,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
             stop=None
         )
 
-    action = None
+    # action = None
     if context.position is not None:
         cost_basis = context.position['cost_basis']
         amount = context.position['amount']
@@ -80,7 +79,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
                 amount=-amount,
                 limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
             )
-            action = 0
+            # action = 0
             context.position = None
 
     else:
@@ -97,7 +96,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
                 amount=buy_amount,
                 stop=None
             )
-            action = 0
+            # action = 0
 
 
 def _handle_data_rsi_only(context, data):
@@ -115,7 +114,7 @@ def _handle_data_rsi_only(context, data):
         prices = data.history(
             context.asset,
             fields='price',
-            bar_count=17,
+            bar_count=20,
             frequency='30T'
         )
     except Exception as e:
@@ -176,7 +175,7 @@ def handle_data(context, data):
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
 
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results.loc[:, 'portfolio_value'].plot(ax=ax1)
@@ -250,17 +249,7 @@ def analyze(context=None, results=None):
     pass
 
 
-# run_algorithm(
+if __name__ == '__main__':
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='bittrex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc',
-#     live_graph=False
-# )
-
     # Backtest
     run_algorithm(
         capital_base=0.5,

catalyst/examples/simple_loop.py

@@ -1,32 +1,38 @@
-import talib
 import pandas as pd
+import talib
+from logbook import Logger, INFO
 
 from catalyst import run_algorithm
 from catalyst.api import symbol, record
-from catalyst.exchange.stats_utils import get_pretty_stats, \
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
     extract_transactions
 
+log = Logger('simple_loop', level=INFO)
+
 
 def initialize(context):
-    print('initializing')
+    log.info('initializing')
-    context.asset = symbol('neo_eth')
+    context.asset = symbol('eth_btc')
     context.base_price = None
 
 
 def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
+    log.info('handling bar: {}'.format(data.current_dt))
 
     price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
+    log.info('got price {price}'.format(price=price))
 
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15T'
+        frequency='30T'
     )
+    last_traded = prices.index[-1]
+    log.info('last candle date: {}'.format(last_traded))
+
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    print('got rsi: {}'.format(rsi))
+    log.info('got rsi: {}'.format(rsi))
 
     # If base_price is not set, we use the current value. This is the
     # price at the first bar which we reference to calculate price_change.
@@ -48,10 +54,10 @@ def handle_data(context, data):
 
 def analyze(context, perf):
     import matplotlib.pyplot as plt
-    print('the stats: {}'.format(get_pretty_stats(perf)))
+    log.info('the stats: {}'.format(get_pretty_stats(perf)))
 
     # The base currency of the algo exchange
-    base_currency = context.exchanges.values()[0].base_currency.upper()
+    base_currency = list(context.exchanges.values())[0].base_currency.upper()
 
     # Plot the portfolio value over time.
     ax1 = plt.subplot(611)
@@ -107,25 +113,32 @@ def analyze(context, perf):
     pass
 
 
+if __name__ == '__main__':
+    mode = 'backtest'
+
+    if mode == 'backtest':
         run_algorithm(
-    capital_base=250,
+            capital_base=1,
-    start=pd.to_datetime('2017-11-1 0:00', utc=True),
-    end=pd.to_datetime('2017-11-10 23:59', utc=True),
-    data_frequency='daily',
             initialize=initialize,
             handle_data=handle_data,
-    analyze=analyze,
+            analyze=None,
-    exchange_name='bitfinex',
+            exchange_name='poloniex',
             algo_namespace='simple_loop',
-    base_currency='usd'
+            base_currency='eth',
+            data_frequency='minute',
+            start=pd.to_datetime('2017-9-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=None,
+            exchange_name='binance',
+            live=True,
+            algo_namespace='simple_loop',
+            base_currency='eth',
+            live_graph=False,
+            simulate_orders=True
         )
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=None,
-#     exchange_name='binance',
-#     live=True,
-#     algo_namespace='simple_loop',
-#     base_currency='eth',
-#     live_graph=False,
-# )

catalyst/examples/simple_universe.py

@@ -2,73 +2,117 @@
 Requires Catalyst version 0.3.0 or above
 Tested on Catalyst version 0.3.3
 
-These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
+This example aims to provide an easy way for users to learn how to
-You simply need to specify the exchange and the market that you want to focus on.
+collect data from any given exchange and select a subset of the available
-You will all see how to create a universe and filter it base on the exchange and the market you desire.
+currency pairs for trading. You simply need to specify the exchange and
+the market (base_currency) that you want to focus on. You will then see
+how to create a universe of assets, and filter it based the market you
+desire.
 
-The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
+The example prints out the closing price of all the pairs for a given
-The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
+market in a given exchange every 30 minutes. The example also contains
-Use this as the backbone to create your own trading strategies.
+the OHLCV data with minute-resolution for the past seven days which
+could be used to create indicators. Use this code as the backbone to
+create your own trading strategy.
+
+The lookback_date variable is used to ensure data for a coin existed on
+the lookback period specified.
+
+To run, execute the following two commands in a terminal (inside catalyst
+environment). The first one retrieves all the pricing data needed for this
+script to run (only needs to be run once), and the second one executes this
+script with the parameters specified in the run_algorithm() call at the end
+of the file:
+
+catalyst ingest-exchange -x bitfinex -f minute
+
+python simple_universe.py
 
-Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
 """
+from datetime import timedelta
 
 import numpy as np
 import pandas as pd
-from datetime import timedelta
-from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
 
-from catalyst.api import (
+from catalyst import run_algorithm
-    symbols,
+from catalyst.api import (symbols, )
-)
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
 
 
 def initialize(context):
-    context.i = -1  # counts the minutes
+    context.i = -1  # minute counter
-    context.exchange = context.exchanges.values()[0].name.lower()  # exchange name
+    context.exchange = list(context.exchanges.values())[0].name.lower()
-    context.base_currency = context.exchanges.values()[0].base_currency.lower()  # market base currency
+    context.base_currency = list(context.exchanges.values())[0].base_currency.lower()
 
 
 def handle_data(context, data):
     context.i += 1
     lookback_days = 7  # 7 days
 
-    # current date formatted into a string
+    # current date & time in each iteration formatted into a string
-    today = data.current_dt
+    now = data.current_dt
-    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-    lookback_date = today - timedelta(days=lookback_days)  # subtract the amount of days specified in lookback
+    lookback_date = now - timedelta(days=lookback_days)
-    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  # get only the date as a string
+    # keep only the date as a string, discard the time
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
 
-    # update universe everyday
+    one_day_in_minutes = 1440  # 60 * 24 assumes data_frequency='minute'
-    new_day = 60 * 24  # assuming data_frequency='minute'
+    # update universe everyday at midnight
-    if not context.i % new_day:
+    if not context.i % one_day_in_minutes:
         context.universe = universe(context, lookback_date, date)
 
     # get data every 30 minutes
     minutes = 30
-    one_day_in_minutes = 1440  # 1440 assumes data_frequency='minute'
+
-    lookback = one_day_in_minutes / minutes * lookback_days  # get N lookback_days of history data
+    # get lookback_days of history data: that is 'lookback' number of bins
+    lookback = int(one_day_in_minutes / minutes * lookback_days)
     if not context.i % minutes and context.universe:
         # we iterate for every pair in the current universe
         for coin in context.coins:
             pair = str(coin.symbol)
 
-            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
+            # Get 30 minute interval OHLCV data. This is the standard data
-            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
+            # required for candlestick or indicators/signals. Return Pandas
-            opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
+            # DataFrames. 30T means 30-minute re-sampling of one minute data.
-            high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
+            # Adjust it to your desired time interval as needed.
-            low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
+            opened = fill(data.history(coin,
-            close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
+                                       'open',
-            volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
+                                       bar_count=lookback,
+                                       frequency='30T')).values
+            high = fill(data.history(coin,
+                                     'high',
+                                     bar_count=lookback,
+                                     frequency='30T')).values
+            low = fill(data.history(coin,
+                                    'low',
+                                    bar_count=lookback,
+                                    frequency='30T')).values
+            close = fill(data.history(coin,
+                                      'price',
+                                      bar_count=lookback,
+                                      frequency='30T')).values
+            volume = fill(data.history(coin,
+                                       'volume',
+                                       bar_count=lookback,
+                                       frequency='30T')).values
 
-            # close[-1] is the equivalent to current price
+            # close[-1] is the last value in the set, which is the equivalent
+            # to current price (as in the most recent value)
             # displays the minute price for each pair every 30 minutes
-            print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
+            print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
+                  '\tV:{v}'.format(
+                    now=now,
+                    pair=pair,
+                    o=opened[-1],
+                    h=high[-1],
+                    l=low[-1],
+                    c=close[-1],
+                    v=volume[-1],
+                  ))
 
-            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------------------------------
-            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
+            # --------------- Insert Your Strategy Here -------------------
-            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------------------------------
 
 
 def analyze(context=None, results=None):
@@ -78,23 +122,24 @@ def analyze(context=None, results=None):
 # Get the universe for a given exchange and a given base_currency market
 # Example: Poloniex BTC Market
 def universe(context, lookback_date, current_date):
-    json_symbols = get_exchange_symbols(context.exchange)  # get all the pairs for the exchange
+    # get all the pairs for the given exchange
-    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)  # convert into a dataframe
+    json_symbols = get_exchange_symbols(context.exchange)
-    universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
+    # convert into a DataFrame for easier processing
+    df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
+    df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
                                    axis=1)
-    universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
+    df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
                                      axis=1)
 
-    # Filter all the exchange pairs to only the ones for a give base currency
+    # Filter all the pairs to get only the ones for a given base_currency
-    universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
+    df = df[df['base_currency'] == context.base_currency]
 
-    # Filter all the pairs to ensure that pair existed in the current date range
+    # Filter all pairs to ensure that pair existed in the current date range
-    universe_df = universe_df[universe_df.start_date < lookback_date]
+    df = df[df.start_date < lookback_date]
-    universe_df = universe_df[universe_df.end_daily >= current_date]
+    df = df[df.end_daily >= current_date]
-    context.coins = symbols(*universe_df.symbol)  # convert all the pairs to symbols
+    context.coins = symbols(*df.symbol)  # convert all the pairs to symbols
 
-    # print(universe_df.symbol.tolist())
+    return df.symbol.tolist()
-    return universe_df.symbol.tolist()
 
 
 # Replace all NA, NAN or infinite values with its nearest value
@@ -102,7 +147,9 @@ def fill(series):
     if isinstance(series, pd.Series):
         return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
     elif isinstance(series, np.ndarray):
-        return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
+        return pd.Series(series).replace(
+                     [np.inf, -np.inf], np.nan
+                    ).ffill().bfill().values
     else:
         return series
 
@@ -112,18 +159,13 @@ if __name__ == '__main__':
     end_date = pd.to_datetime('2017-11-13', utc=True)
 
     performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=100.0,  # amount of base_currency, not always in dollars unless usd
+                                capital_base=100.0,  # amount of base_currency
                                 initialize=initialize,
                                 handle_data=handle_data,
                                 analyze=analyze,
-                                exchange_name='bitfinex',
+                                exchange_name='poloniex',
                                 data_frequency='minute',
                                 base_currency='btc',
                                 live=False,
                                 live_graph=False,
                                 algo_namespace='simple_universe')
-
-"""
-Run in Terminal (inside catalyst environment):
-python simple_universe.py
-"""

catalyst/examples/talib_simple.py

@@ -1,9 +1,11 @@
 # Run Command
-# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex
+# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
+#   -f talib_simple.py -x poloniex
 #
 # Description
-# Simple TALib Example showing how to use various indicators in you strategy
+# Simple TALib Example showing how to use various indicators
-# Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
+# in you strategy. Based loosly on
+# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
 
 import os
 
@@ -21,7 +23,7 @@ from catalyst.api import (
     order_target_percent,
     symbol,
 )
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
 
 algo_namespace = 'talib_sample'
 log = Logger(algo_namespace)

catalyst/exchange/asset_finder_exchange.py

@@ -1,99 +0,0 @@
-from logbook import Logger
-
-from catalyst.constants import LOG_LEVEL
-
-log = Logger('AssetFinderExchange', level=LOG_LEVEL)
-
-
-class AssetFinderExchange(object):
-    def __init__(self):
-        self._asset_cache = {}
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        return list()
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        # for sid in sids:
-        #     if sid in self._asset_cache:
-        #         log.debug('got asset from cache: {}'.format(sid))
-        #     else:
-        #         log.debug('fetching asset: {}'.format(sid))
-        return list()
-
-    def lookup_symbol(self, symbol, exchange, data_frequency=None,
-                      as_of_date=None, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
-
-        if data_frequency is not None:
-            key = ','.join([exchange.name, symbol, data_frequency])
-
-        else:
-            key = ','.join([exchange.name, symbol])
-
-        if key in self._asset_cache:
-            return self._asset_cache[key]
-        else:
-            asset = exchange.get_asset(symbol, data_frequency)
-            self._asset_cache[key] = asset
-            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,705 +0,0 @@
-import base64
-import datetime
-import hashlib
-import hmac
-import json
-import re
-import time
-
-import numpy as np
-import pandas as pd
-import pytz
-import requests
-import six
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
-
-# Trying to account for REST api instability
-# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-requests.adapters.DEFAULT_RETRIES = 20
-
-BITFINEX_URL = 'https://api.bitfinex.com'
-
-from catalyst.constants import LOG_LEVEL
-
-log = Logger('Bitfinex', level=LOG_LEVEL)
-warning_logger = Logger('AlgoWarning')
-
-
-class Bitfinex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.url = BITFINEX_URL
-        self.key = key
-        self.secret = secret.encode('UTF-8')
-        self.name = 'bitfinex'
-        self.color = 'green'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-
-        # The candle limit for each request
-        self.num_candles_limit = 1000
-
-        # Max is 90 but playing it safe
-        # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 80
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def _request(self, operation, data, version='v1'):
-        payload_object = {
-            'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 1000000),
-            # convert to string
-            'options': {}
-        }
-
-        if data is None:
-            payload_dict = payload_object
-        else:
-            payload_dict = payload_object.copy()
-            payload_dict.update(data)
-
-        payload_json = json.dumps(payload_dict)
-        if six.PY3:
-            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
-        else:
-            payload = base64.b64encode(payload_json)
-
-        m = hmac.new(self.secret, payload, hashlib.sha384)
-        m = m.hexdigest()
-
-        # headers
-        headers = {
-            'X-BFX-APIKEY': self.key,
-            'X-BFX-PAYLOAD': payload,
-            'X-BFX-SIGNATURE': m
-        }
-
-        if data is None:
-            request = requests.get(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ), data={},
-                headers=headers)
-        else:
-            request = requests.post(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ),
-                headers=headers)
-
-        return request
-
-    def _get_v2_symbol(self, asset):
-        pair = asset.symbol.split('_')
-        symbol = 't' + pair[0].upper() + pair[1].upper()
-        return symbol
-
-    def _get_v2_symbols(self, assets):
-        """
-        Workaround to support Bitfinex v2
-        TODO: Might require a separate asset dictionary
-
-        :param assets:
-        :return:
-        """
-
-        v2_symbols = []
-        for asset in assets:
-            v2_symbols.append(self._get_v2_symbol(asset))
-
-        return v2_symbols
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from a Bitfinex order dictionary
-        :param order_status:
-        :return: Order
-        """
-        if order_status['is_cancelled']:
-            status = ORDER_STATUS.CANCELLED
-        elif not order_status['is_live']:
-            log.info('found executed order {}'.format(order_status))
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['original_amount'])
-        filled = float(order_status['executed_amount'])
-
-        if order_status['side'] == 'sell':
-            amount = -amount
-            filled = -filled
-
-        price = float(order_status['price'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        if order_type.endswith('limit'):
-            limit_price = price
-        elif order_type.endswith('stop'):
-            stop_price = price
-
-        executed_price = float(order_status['avg_execution_price'])
-
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
-        commission = None
-
-        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        date = pytz.utc.localize(date)
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['id']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        log.debug('retrieving wallets balances')
-        try:
-            self.ask_request()
-            response = self._request('balances', None)
-            balances = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['message'])
-            )
-
-        std_balances = dict()
-        for balance in balances:
-            currency = balance['currency'].lower()
-            std_balances[currency] = float(balance['available'])
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Bitfinex
-
-        :param data_frequency:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
-         '1M'
-        """
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
-                               '360T', '720T', '1D', '7D', '14D', '30D']
-        if freq not in allowed_frequencies:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
-        if freq_match:
-            number = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-            if unit == 'T':
-                if number in [60, 180, 360, 720]:
-                    number = number / 60
-                    converted_unit = 'h'
-                else:
-                    converted_unit = 'm'
-            else:
-                converted_unit = unit
-
-            frequency = '{}{}'.format(number, converted_unit)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-        for asset in asset_list:
-            symbol = self._get_v2_symbol(asset)
-            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
-                url=self.url,
-                frequency=frequency,
-                symbol=symbol
-            )
-
-            if bar_count:
-                is_list = True
-                url += '/hist?limit={}'.format(int(bar_count))
-
-                def get_ms(date):
-                    epoch = datetime.datetime.utcfromtimestamp(0)
-                    epoch = epoch.replace(tzinfo=pytz.UTC)
-
-                    return (date - epoch).total_seconds() * 1000.0
-
-                if start_dt is not None:
-                    start_ms = get_ms(start_dt)
-                    url += '&start={0:f}'.format(start_ms)
-
-                if end_dt is not None:
-                    end_ms = get_ms(end_dt)
-                    url += '&end={0:f}'.format(end_ms)
-
-            else:
-                is_list = False
-                url += '/last'
-
-            try:
-                self.ask_request()
-                response = requests.get(url)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response.content:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            candles = response.json()
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(
-                    candle[0] / 1000.0)
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-                ohlc = dict(
-                    open=np.float64(candle[1]),
-                    high=np.float64(candle[3]),
-                    low=np.float64(candle[4]),
-                    close=np.float64(candle[2]),
-                    volume=np.float64(candle[5]),
-                    price=np.float64(candle[2]),
-                    last_traded=last_traded
-                )
-                return ohlc
-
-            if is_list:
-                ohlc_bars = []
-                # We can to list candles from old to new
-                for candle in reversed(candles):
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-            else:
-                ohlc = ohlc_from_candle(candles)
-                ohlc_map[asset] = ohlc
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-        if isinstance(style, ExchangeLimitOrder) \
-                or isinstance(style, ExchangeStopLimitOrder):
-            price = style.get_limit_price(is_buy)
-            order_type = 'limit'
-
-        elif isinstance(style, ExchangeStopOrder):
-            price = style.get_stop_price(is_buy)
-            order_type = 'stop'
-
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-        req = dict(
-            symbol=exchange_symbol,
-            amount=str(float(abs(amount))),
-            price="{:.20f}".format(float(price)),
-            side='buy' if is_buy else 'sell',
-            type='exchange ' + order_type,  # TODO: support margin trades
-            exchange=self.name,
-            is_hidden=False,
-            is_postonly=False,
-            use_all_available=0,
-            ocoorder=False,
-            buy_price_oco=0,
-            sell_price_oco=0
-        )
-
-        date = pd.Timestamp.utcnow()
-        try:
-            self.ask_request()
-            response = self._request('order/new', req)
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='unable to create Bitfinex order {}'.format(
-                    order_status['message'])
-            )
-
-        order_id = str(order_status['id'])
-        order = Order(
-            dt=date,
-            asset=asset,
-            amount=amount,
-            stop=style.get_stop_price(is_buy),
-            limit=style.get_limit_price(is_buy),
-            id=order_id
-        )
-
-        return order
-
-    def get_open_orders(self, asset=None):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        try:
-            self.ask_request()
-            response = self._request('orders', None)
-            order_statuses = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_statuses:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        orders = []
-        for order_status in order_statuses:
-            order, executed_price = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-        try:
-            self.ask_request()
-            response = self._request(
-                'order/status', {'order_id': int(order_id)})
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='Unable to retrieve order status: {}'.format(
-                    order_status['message'])
-            )
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-
-        try:
-            self.ask_request()
-            response = self._request('order/cancel', {'order_id': order_id})
-            status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self._get_v2_symbols(assets)
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            self.ask_request()
-            response = requests.get(
-                '{url}/v2/tickers?symbols={symbols}'.format(
-                    url=self.url,
-                    symbols=','.join(symbols),
-                )
-            )
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response.content:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response.content)
-            )
-
-        try:
-            tickers = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        ticks = dict()
-        for index, ticker in enumerate(tickers):
-            if not len(ticker) == 11:
-                raise ExchangeRequestError(
-                    error='Invalid ticker in response: {}'.format(ticker)
-                )
-
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker[1],
-                ask=ticker[3],
-                last_price=ticker[7],
-                low=ticker[10],
-                high=ticker[9],
-                volume=ticker[8],
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self._request('symbols', None)
-
-        for symbol in response.json():
-            if (source_dates):
-                start_date = self.get_symbol_start_date(symbol)
-            else:
-                try:
-                    start_date = cached_symbols[symbol]['start_date']
-                except KeyError as e:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[symbol]['end_daily']
-            except KeyError as e:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[symbol]['end_minute']
-            except KeyError as e:
-                end_minute = 'N/A'
-
-            symbol_map[symbol] = dict(
-                symbol=symbol[:-3] + '_' + symbol[-3:],
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-
-        print(symbol)
-        symbol_v2 = 't' + symbol.upper()
-
-        """
-            For each symbol we retrieve candles with Monhtly resolution
-            We get the first month, and query again with daily resolution
-            around that date, and we get the first date
-        """
-        url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
-            url=self.url,
-            symbol=symbol_v2
-        )
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        """
-            If we don't get any data back for our monthly-resolution query
-            it means that symbol started trading less than a month ago, so
-            arbitrarily set the ref. date to 15 days ago to be safe with
-            +/- 31 days
-        """
-        if (len(response.json())):
-            startmonth = response.json()[-1][0]
-        else:
-            startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
-
-        """
-            Query again with daily resolution setting the start and end around
-            the startmonth we got above. Avoid end dates greater than now: time.time()
-        """
-        url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
-            url=self.url,
-            symbol=symbol_v2,
-            start=startmonth - 3600 * 24 * 31 * 1000,
-            end=min(startmonth + 3600 * 24 * 31 * 1000,
-                    int(time.time() * 1000))
-        )
-
-        try:
-            self.ask_request()
-            response = requests.get(url)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d',
-                             time.gmtime(int(response.json()[-1][0] / 1000)))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        exchange_symbol = asset.exchange_symbol
-        try:
-            self.ask_request()
-            # TODO: implement limit
-            response = self._request(
-                'book/{}'.format(exchange_symbol), None)
-            data = response.json()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        # TODO: filter by type
-        result = dict()
-        for order_type in data:
-            result[order_type] = []
-
-            for entry in data[order_type]:
-                result[order_type].append(dict(
-                    rate=float(entry['price']),
-                    quantity=float(entry['amount'])
-                ))
-
-        return result

catalyst/exchange/bitfinex/symbols.json

@@ -1,127 +0,0 @@
-{
-  "neobtc": {
-    "symbol": "neo_btc",
-    "start_date": "2017-09-07",
-    "precision": 5
-  },
-  "neousd": {
-    "symbol": "neo_usd",
-    "start_date": "2017-09-07"
-  },
-  "neoeth": {
-    "symbol": "neo_eth",
-    "start_date": "2017-09-07"
-  },
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bchusd": {
-    "symbol": "bch_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2017-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcbtc": {
-    "symbol": "etc_btc",
-    "start_date": "2017-01-01"
-  },
-  "etcusd": {
-    "symbol": "etc_usd",
-    "start_date": "2017-01-01"
-  },
-  "rrtusd": {
-    "symbol": "rrt_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtbtc": {
-    "symbol": "rrt_btc",
-    "start_date": "2010-01-01"
-  },
-  "zecusd": {
-    "symbol": "zec_usd",
-    "start_date": "2010-01-01"
-  },
-  "zecbtc": {
-    "symbol": "zec_btc",
-    "start_date": "2010-01-01"
-  },
-  "xmrusd": {
-    "symbol": "xmr_usd",
-    "start_date": "2010-01-01"
-  },
-  "xmrbtc": {
-    "symbol": "xmr_btc",
-    "start_date": "2010-01-01"
-  },
-  "dshusd": {
-    "symbol": "dsh_usd",
-    "start_date": "2010-01-01"
-  },
-  "dshbtc": {
-    "symbol": "dsh_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccbtc": {
-    "symbol": "bcc_btc",
-    "start_date": "2010-01-01"
-  },
-  "bcubtc": {
-    "symbol": "bcu_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccusd": {
-    "symbol": "bcc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bcuusd": {
-    "symbol": "bcu_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpusd": {
-    "symbol": "xrp_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpbtc": {
-    "symbol": "xrp_btc",
-    "start_date": "2010-01-01"
-  },
-  "iotusd": {
-    "symbol": "iot_usd",
-    "start_date": "2010-01-01"
-  },
-  "iotbtc": {
-    "symbol": "iot_btc",
-    "start_date": "2010-01-01"
-  },
-  "ioteth": {
-    "symbol": "iot_eth",
-    "start_date": "2010-01-01"
-  },
-  "eosusd": {
-    "symbol": "eos_usd",
-    "start_date": "2010-01-01"
-  },
-  "eosbtc": {
-    "symbol": "eos_btc",
-    "start_date": "2010-01-01"
-  },
-  "eoseth": {
-    "symbol": "eos_eth",
-    "start_date": "2010-01-01"
-  }
-}

catalyst/exchange/bittrex/bittrex.py

@@ -1,416 +0,0 @@
-import json
-import time
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from six.moves import urllib
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-    ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
-    CreateOrderError
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-
-# TODO: consider using this: https://github.com/mondeja/bittrex_v2
-
-log = Logger('Bittrex', level=LOG_LEVEL)
-
-URL2 = 'https://bittrex.com/Api/v2.0'
-
-
-class Bittrex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret)
-        self.name = 'bittrex'
-        self.color = 'blue'
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-
-        self.num_candles_limit = 2000
-
-        # Not sure what the rate limit is but trying to play it safe
-        # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.minute_writer = None
-        self.minute_reader = None
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.bundle = ExchangeBundle(self.name)
-
-    @property
-    def account(self):
-        pass
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def get_balances(self):
-        balances = self.api.getbalances()
-        try:
-            log.debug('retrieving wallet balances')
-            self.ask_request()
-
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        std_balances = dict()
-        try:
-            for balance in balances:
-                currency = balance['Currency'].lower()
-                std_balances[currency] = balance['Available']
-
-        except TypeError:
-            raise ExchangeRequestError(error=balances)
-
-        return std_balances
-
-    def create_order(self, asset, amount, is_buy, style):
-        log.info('creating {} order'.format('buy' if is_buy else 'sell'))
-        exchange_symbol = self.get_symbol(asset)
-
-        if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
-            if isinstance(style, StopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-            try:
-                self.ask_request()
-                if is_buy:
-                    order_status = self.api.buylimit(exchange_symbol, amount,
-                                                     price)
-                else:
-                    order_status = self.api.selllimit(exchange_symbol,
-                                                      abs(amount), price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'uuid' in order_status:
-                order_id = order_status['uuid']
-                order = Order(
-                    dt=pd.Timestamp.utcnow(),
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                if order_status == 'INSUFFICIENT_FUNDS':
-                    log.warn('not enough funds to create order')
-                    return None
-                elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
-                    log.warn('Your order is too small, order at least 50K'
-                             ' Satoshi')
-                    return None
-                else:
-                    raise CreateOrderError(
-                        exchange=self.name,
-                        error=order_status
-                    )
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset):
-        symbol = self.get_symbol(asset)
-        try:
-            self.ask_request()
-            open_orders = self.api.getopenorders(symbol)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        orders = list()
-        for order_status in open_orders:
-            order = self._create_order(order_status)
-            orders.append(order)
-
-        return orders
-
-    def _create_order(self, order_status):
-        log.info(
-            'creating catalyst order from Bittrex {}'.format(order_status))
-        if order_status['CancelInitiated']:
-            status = ORDER_STATUS.CANCELLED
-        elif order_status['Closed'] is not None:
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        date = pd.to_datetime(order_status['Opened'], utc=True)
-        amount = order_status['Quantity']
-        filled = amount - order_status['QuantityRemaining']
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['Exchange']],
-            amount=amount,
-            stop=None,  # Not yet supported by Bittrex
-            limit=order_status['Limit'],
-            filled=filled,
-            id=order_status['OrderUuid'],
-            commission=order_status['CommissionPaid']
-        )
-        order.status = status
-
-        executed_price = order_status['PricePerUnit']
-
-        return order, executed_price
-
-    def get_order(self, order_id):
-        log.info('retrieving order {}'.format(order_id))
-        try:
-            self.ask_request()
-            order_status = self.api.getorder(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if order_status is None:
-            raise OrderNotFound(order_id=order_id, exchange=self.name)
-
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-        log.info('cancelling order {}'.format(order_id))
-
-        try:
-            self.ask_request()
-            status = self.api.cancel(order_id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise OrderCancelError(
-                order_id=order_id,
-                exchange=self.name,
-                error=status['message']
-            )
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Supported Intervals
-        -------------------
-        day, oneMin, fiveMin, thirtyMin, hour
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :param start_dt
-        :param end_dt
-        :return:
-        """
-
-        # TODO: this has no effect at the moment
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T':
-            frequency = 'oneMin'
-        elif freq == '5T':
-            frequency = 'fiveMin'
-        elif freq == '30T':
-            frequency = 'thirtyMin'
-        elif freq == '60T':
-            frequency = 'hour'
-        elif freq == '1D':
-            frequency = 'day'
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        for asset in asset_list:
-            end = int(time.mktime(end_dt.timetuple()))
-            url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_={end}'.format(
-                url=URL2,
-                symbol=self.get_symbol(asset),
-                frequency=frequency,
-                end=end
-            )
-
-            try:
-                data = json.loads(urllib.request.urlopen(url).read().decode())
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if data['message']:
-                raise ExchangeRequestError(
-                    error='Unable to fetch candles {}'.format(data['message'])
-                )
-
-            candles = data['result']
-
-            def ohlc_from_candle(candle):
-                ohlc = dict(
-                    open=candle['O'],
-                    high=candle['H'],
-                    low=candle['L'],
-                    close=candle['C'],
-                    volume=candle['V'],
-                    price=candle['C'],
-                    last_traded=pd.to_datetime(candle['T'], utc=True)
-                )
-                return ohlc
-
-            ordered_candles = list(reversed(candles))
-            ohlc_map = dict()
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
-            else:
-                # TODO: optimize
-                ohlc_bars = []
-                for candle in ordered_candles[:bar_count]:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def tickers(self, assets):
-        """
-        As of v1.1, Bittrex only allows one ticker at the time.
-        So we have to make multiple calls to fetch multiple assets.
-
-        :param assets:
-        :return:
-        """
-        log.info('retrieving tickers')
-
-        ticks = dict()
-        for asset in assets:
-            symbol = self.get_symbol(asset)
-            try:
-                self.ask_request()
-                ticker = self.api.getticker(symbol)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            # TODO: catch invalid ticker
-            ticks[asset] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker['Bid'],
-                ask=ticker['Ask'],
-                last_price=ticker['Last']
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def get_account(self):
-        log.info('retrieving account data')
-        pass
-
-    def generate_symbols_json(self, filename=None):
-        symbol_map = {}
-
-        fn, r = download_exchange_symbols(self.name)
-        with open(fn) as data_file:
-            cached_symbols = json.load(data_file)
-
-        markets = self.api.getmarkets()
-        for market in markets:
-            exchange_symbol = market['MarketName']
-            symbol = '{market}_{base}'.format(
-                market=self.sanitize_curency_symbol(market['MarketCurrency']),
-                base=self.sanitize_curency_symbol(market['BaseCurrency'])
-            )
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=pd.to_datetime(market['Created'],
-                                          utc=True).strftime("%Y-%m-%d"),
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_orderbook(self, asset, order_type='all', limit=100):
-        if order_type == 'all':
-            order_type = 'both'
-        elif order_type == 'bid':
-            order_type = 'buy'
-        elif order_type == 'ask':
-            order_type = 'sell'
-        else:
-            raise ValueError('invalid type')
-
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(
-            market=exchange_symbol,
-            type=order_type,
-            depth=100
-        )
-
-        result = dict()
-        for exchange_type in data:
-            if exchange_type == 'buy':
-                order_type = 'bids'
-            elif exchange_type == 'sell':
-                order_type = 'asks'
-
-            result[order_type] = []
-            for entry in data[exchange_type]:
-                result[order_type].append(dict(
-                    rate=entry['Rate'],
-                    quantity=entry['Quantity']
-                ))
-
-        return result

catalyst/exchange/bittrex/bittrex_api.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-from six.moves import urllib
-
-urlopen = urllib.request.urlopen
-
-
-class Bittrex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-        self.public = ['getmarkets', 'getcurrencies', 'getticker',
-                       'getmarketsummaries', 'getmarketsummary',
-                       'getorderbook', 'getmarkethistory']
-        self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
-                       'cancel', 'getopenorders']
-        self.account = ['getbalances', 'getbalance', 'getdepositaddress',
-                        'withdraw', 'getorder', 'getorderhistory',
-                        'getwithdrawalhistory', 'getdeposithistory']
-
-    def query(self, method, values={}):
-        if method in self.public:
-            url = 'https://bittrex.com/api/v1.1/public/'
-        elif method in self.market:
-            url = 'https://bittrex.com/api/v1.1/market/'
-        elif method in self.account:
-            url = 'https://bittrex.com/api/v1.1/account/'
-        else:
-            return 'Something went wrong, sorry.'
-
-        url += method + '?' + urllib.parse.urlencode(values)
-
-        if method not in self.public:
-            url += '&apikey=' + self.key
-            url += '&nonce=' + str(int(time.time()))
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 url.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'apisign': signature}
-        else:
-            headers = {}
-
-        req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(
-            req, context=ssl._create_unverified_context()).read())
-
-        if response["result"]:
-            return response["result"]
-        else:
-            return response["message"]
-
-    def getmarkets(self):
-        return self.query('getmarkets')
-
-    def getcurrencies(self):
-        return self.query('getcurrencies')
-
-    def getticker(self, market):
-        return self.query('getticker', {'market': market})
-
-    def getmarketsummaries(self):
-        return self.query('getmarketsummaries')
-
-    def getmarketsummary(self, market):
-        return self.query('getmarketsummary', {'market': market})
-
-    def getorderbook(self, market, type, depth=20):
-        return self.query('getorderbook',
-                          {'market': market, 'type': type, 'depth': depth})
-
-    def getmarkethistory(self, market, count=20):
-        return self.query('getmarkethistory',
-                          {'market': market, 'count': count})
-
-    def buylimit(self, market, quantity, rate):
-        return self.query('buylimit', {'market': market, 'quantity': quantity,
-                                       'rate': rate})
-
-    def buymarket(self, market, quantity):
-        return self.query('buymarket',
-                          {'market': market, 'quantity': quantity})
-
-    def selllimit(self, market, quantity, rate):
-        return self.query('selllimit', {'market': market, 'quantity': quantity,
-                                        'rate': rate})
-
-    def sellmarket(self, market, quantity):
-        return self.query('sellmarket',
-                          {'market': market, 'quantity': quantity})
-
-    def cancel(self, uuid):
-        return self.query('cancel', {'uuid': uuid})
-
-    def getopenorders(self, market):
-        return self.query('getopenorders', {'market': market})
-
-    def getbalances(self):
-        return self.query('getbalances')
-
-    def getbalance(self, currency):
-        return self.query('getbalance', {'currency': currency})
-
-    def getdepositaddress(self, currency):
-        return self.query('getdepositaddress', {'currency': currency})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'quantity': quantity,
-                           'address': address})
-
-    def getorder(self, uuid):
-        return self.query('getorder', {'uuid': uuid})
-
-    def getorderhistory(self, market, count):
-        return self.query('getorderhistory',
-                          {'market': market, 'count': count})
-
-    def getwithdrawalhistory(self, currency, count):
-        return self.query('getwithdrawalhistory',
-                          {'currency': currency, 'count': count})
-
-    def getdeposithistory(self, currency, count):
-        return self.query('getdeposithistory',
-                          {'currency': currency, 'count': count})

catalyst/exchange/bittrex/extensions-example.py

@@ -1,7 +0,0 @@
-from catalyst.data.bundles import register
-from catalyst.exchange.exchange_bundle import exchange_bundle
-
-symbols = (
-    'neo_btc',
-)
-register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -1,25 +1,34 @@
+import json
+import os
 import re
 from collections import defaultdict
 
 import ccxt
 import pandas as pd
-from ccxt import ExchangeNotAvailable
+import six
+from ccxt import InvalidOrder, NetworkError, \
+    ExchangeError
+from logbook import Logger
 from six import string_types
 
-from catalyst.finance.order import Order, ORDER_STATUS
-
 from catalyst.algorithm import MarketOrder
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange import Exchange, ExchangeLimitOrder
+from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
-    ExchangeNotFoundError
+    ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
-from catalyst.exchange.exchange_utils import mixin_market_params, \
+    UnsupportedHistoryFrequencyError
-    from_ms_timestamp
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
+from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
+    get_exchange_folder, get_catalyst_symbol, \
+    get_exchange_auth
+from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
+    get_epoch, \
+    get_periods_range
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.finance.transaction import Transaction
 
 log = Logger('CCXT', level=LOG_LEVEL)
 
@@ -34,8 +43,7 @@ SUPPORTED_EXCHANGES = dict(
 
 
 class CCXT(Exchange):
-    def __init__(self, exchange_name, key, secret, base_currency,
+    def __init__(self, exchange_name, key, secret, base_currency):
-                 portfolio=None):
         log.debug(
             'finding {} in CCXT exchanges:\n{}'.format(
                 exchange_name, ccxt.exchanges
@@ -53,29 +61,116 @@ class CCXT(Exchange):
                 'apiKey': key,
                 'secret': secret,
             })
+            self.api.enableRateLimit = True
 
         except Exception:
             raise ExchangeNotFoundError(exchange_name=exchange_name)
 
         self._symbol_maps = [None, None]
 
-        markets_symbols = self.api.load_markets()
-        log.debug('the markets:\n{}'.format(markets_symbols))
-
         self.name = exchange_name
 
-        self.markets = self.api.fetch_markets()
-        self.load_assets()
-
         self.base_currency = base_currency
-        self._portfolio = portfolio
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
         self.max_requests_per_minute = 60
+        self.low_balance_threshold = 0.1
         self.request_cpt = dict()
+        self._common_symbols = dict()
 
         self.bundle = ExchangeBundle(self.name)
+        self.markets = None
+        self._is_init = False
+
+    def init(self):
+        if self._is_init:
+            return
+
+        exchange_folder = get_exchange_folder(self.name)
+        filename = os.path.join(exchange_folder, 'cctx_markets.json')
+
+        if os.path.exists(filename):
+            timestamp = os.path.getmtime(filename)
+            dt = pd.to_datetime(timestamp, unit='s', utc=True)
+
+            if dt >= pd.Timestamp.utcnow().floor('1D'):
+                with open(filename) as f:
+                    self.markets = json.load(f)
+
+                log.debug('loaded markets for {}'.format(self.name))
+
+        if self.markets is None:
+            try:
+                markets_symbols = self.api.load_markets()
+                log.debug(
+                    'fetching {} markets:\n{}'.format(
+                        self.name, markets_symbols
+                    )
+                )
+
+                self.markets = self.api.fetch_markets()
+                with open(filename, 'w+') as f:
+                    json.dump(self.markets, f, indent=4)
+
+            except (ExchangeError, NetworkError) as e:
+                log.warn(
+                    'unable to fetch markets {}: {}'.format(
+                        self.name, e
+                    )
+                )
+                raise ExchangeRequestError(error=e)
+
+        self.load_assets()
+        self._is_init = True
+
+    @staticmethod
+    def find_exchanges(features=None, is_authenticated=False):
+        ccxt_features = []
+        if features is not None:
+            for feature in features:
+                if not feature.endswith('Bundle'):
+                    ccxt_features.append(feature)
+
+        exchange_names = []
+        for exchange_name in ccxt.exchanges:
+            if is_authenticated:
+                exchange_auth = get_exchange_auth(exchange_name)
+
+                has_auth = (exchange_auth['key'] != ''
+                            and exchange_auth['secret'] != '')
+
+                if not has_auth:
+                    continue
+
+            log.debug('loading exchange: {}'.format(exchange_name))
+            exchange = getattr(ccxt, exchange_name)()
+
+            if ccxt_features is None:
+                has_feature = True
+
+            else:
+                try:
+                    has_feature = all(
+                        [exchange.has[feature] for feature in ccxt_features]
+                    )
+
+                except Exception:
+                    has_feature = False
+
+            if has_feature:
+                try:
+                    log.info('initializing {}'.format(exchange_name))
+                    exchange_names.append(exchange_name)
+
+                except Exception as e:
+                    log.warn(
+                        'unable to initialize exchange {}: {}'.format(
+                            exchange_name, e
+                        )
+                    )
+
+        return exchange_names
 
     def account(self):
         return None
@@ -83,6 +178,30 @@ class CCXT(Exchange):
     def time_skew(self):
         return None
 
+    def get_candle_frequencies(self, data_frequency=None):
+        frequencies = []
+        try:
+            for timeframe in self.api.timeframes:
+                freq = CCXT.get_frequency(timeframe, raise_error=False)
+
+                # TODO: support all frequencies
+                if data_frequency == 'minute' and not freq.endswith('T'):
+                    continue
+
+                elif data_frequency == 'daily' and not freq.endswith('D'):
+                    continue
+
+                frequencies.append(freq)
+
+        except Exception as e:
+            log.warn(
+                'candle frequencies not available for exchange {}'.format(
+                    self.name
+                )
+            )
+
+        return frequencies
+
     def get_market(self, symbol):
         """
         The CCXT market.
@@ -104,48 +223,149 @@ class CCXT(Exchange):
         )
         return market
 
-    def get_symbol(self, asset_or_symbol):
+    def substitute_currency_code(self, currency, source='catalyst'):
+        if source == 'catalyst':
+            currency = currency.upper()
+
+            key = self.api.common_currency_code(currency)
+            self._common_symbols[key] = currency.lower()
+            return key
+
+        else:
+            if currency in self._common_symbols:
+                return self._common_symbols[currency]
+
+            else:
+                return currency.lower()
+
+    def get_symbol(self, asset_or_symbol, source='catalyst'):
         """
         The CCXT symbol.
 
         Parameters
         ----------
         asset_or_symbol
+        source
 
         Returns
         -------
 
         """
+
+        if source == 'ccxt':
+            if isinstance(asset_or_symbol, string_types):
+                parts = asset_or_symbol.split('/')
+                base_currency = self.substitute_currency_code(
+                    parts[0], source
+                )
+                quote_currency = self.substitute_currency_code(
+                    parts[1], source
+                )
+                return '{}_{}'.format(base_currency, quote_currency)
+
+            else:
+                return asset_or_symbol.symbol
+
+        else:
             symbol = asset_or_symbol if isinstance(
                 asset_or_symbol, string_types
             ) else asset_or_symbol.symbol
 
             parts = symbol.split('_')
-        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+            base_currency = self.substitute_currency_code(
+                parts[0], source
+            )
+            quote_currency = self.substitute_currency_code(
+                parts[1], source
+            )
+            return '{}/{}'.format(base_currency, quote_currency)
 
-    def get_catalyst_symbol(self, market_or_symbol):
+    @staticmethod
+    def map_frequency(value, source='ccxt', raise_error=True):
         """
-        The Catalyst symbol.
+        Map a frequency value between CCXT and Catalyst
 
         Parameters
         ----------
-        market_or_symbol
+        value: str
+        source: str
+        raise_error: bool
 
         Returns
         -------
 
+        Notes
+        -----
+        The Pandas offset aliases supported by Catalyst:
+        Alias	Description
+        W	weekly frequency
+        M	month end frequency
+        D	calendar day frequency
+        H	hourly frequency
+        T, min	minutely frequency
+
+        The CCXT timeframes:
+        '1m': '1minute',
+        '1h': '1hour',
+        '1d': '1day',
+        '1w': '1week',
+        '1M': '1month',
+        '1y': '1year',
         """
-        if isinstance(market_or_symbol, string_types):
+        match = re.match(
-            parts = market_or_symbol.split('/')
+            r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value, re.M | re.I
-            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+        )
+        if match:
+            candle_size = int(match.group(1)) \
+                if match.group(1) else 1
+
+            unit = match.group(2)
 
         else:
-            return '{}_{}'.format(
+            raise ValueError('Unable to parse frequency or timeframe')
-                market_or_symbol['base'].lower(),
-                market_or_symbol['quote'].lower(),
-            )
 
-    def get_timeframe(self, freq):
+        if source == 'ccxt':
+            if unit == 'd':
+                result = '{}D'.format(candle_size)
+
+            elif unit == 'm':
+                result = '{}T'.format(candle_size)
+
+            elif unit == 'h':
+                result = '{}H'.format(candle_size)
+
+            elif unit == 'w':
+                result = '{}W'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryTimeframeError(timeframe=value)
+
+        else:
+            if unit == 'D':
+                result = '{}d'.format(candle_size)
+
+            elif unit == 'min' or unit == 'T':
+                result = '{}m'.format(candle_size)
+
+            elif unit == 'H':
+                result = '{}h'.format(candle_size)
+
+            elif unit == 'W':
+                result = '{}w'.format(candle_size)
+
+            elif unit == 'M':
+                result = '{}M'.format(candle_size)
+
+            elif raise_error:
+                raise InvalidHistoryFrequencyError(frequency=value)
+
+        return result
+
+    @staticmethod
+    def get_timeframe(freq, raise_error=True):
         """
         The CCXT timeframe from the Catalyst frequency.
 
@@ -159,40 +379,79 @@ class CCXT(Exchange):
         str
 
         """
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        return CCXT.map_frequency(
-        if freq_match:
+            freq, source='catalyst', raise_error=raise_error
-            candle_size = int(freq_match.group(1)) \
+        )
-                if freq_match.group(1) else 1
 
-            unit = freq_match.group(2)
+    @staticmethod
+    def get_frequency(timeframe, raise_error=True):
+        """
+        Test Catalyst frequency from the CCXT timeframe
 
-        else:
+        Catalyst uses the Pandas offset alias convention:
-            raise InvalidHistoryFrequencyError(frequency=freq)
+        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
 
-        if unit.lower() == 'd':
+        Parameters
-            timeframe = '{}d'.format(candle_size)
+        ----------
+        timeframe
 
-        elif unit.lower() == 'm' or unit == 'T':
+        Returns
-            timeframe = '{}m'.format(candle_size)
+        -------
 
-        elif unit.lower() == 'h' or unit == 'T':
+        """
-            timeframe = '{}h'.format(candle_size)
+        return CCXT.map_frequency(
+            timeframe, source='ccxt', raise_error=raise_error
+        )
 
-        return timeframe
+    def get_candles(self, freq, assets, bar_count=1, start_dt=None,
-
-    def get_candles(self, freq, assets, bar_count=None, start_dt=None,
                     end_dt=None):
+        is_single = (isinstance(assets, TradingPair))
+        if is_single:
+            assets = [assets]
+
         symbols = self.get_symbols(assets)
-        timeframe = self.get_timeframe(freq)
+        timeframe = CCXT.get_timeframe(freq)
-        delta = start_dt - pd.to_datetime('1970-1-1', utc=True)
+
-        ms = int(delta.total_seconds()) * 1000
+        if timeframe not in self.api.timeframes:
+            freqs = [CCXT.get_frequency(t) for t in self.api.timeframes]
+            raise UnsupportedHistoryFrequencyError(
+                exchange=self.name,
+                freq=freq,
+                freqs=freqs,
+            )
+
+        if start_dt is not None and end_dt is not None:
+            raise ValueError(
+                'Please provide either start_dt or end_dt, not both.'
+            )
+
+        elif end_dt is not None:
+            # Make sure that end_dt really wants data in the past
+            # if it's close to now, we skip the 'since' parameters to
+            # lower the probability of error
+            bars_to_now = pd.date_range(
+                end_dt, pd.Timestamp.utcnow(), freq=freq
+            )
+            # See: https://github.com/ccxt/ccxt/issues/1360
+            if len(bars_to_now) > 1 or self.name in ['poloniex']:
+                dt_range = get_periods_range(
+                    end_dt=end_dt,
+                    periods=bar_count,
+                    freq=freq,
+                )
+                start_dt = dt_range[0]
+
+        since = None
+        if start_dt is not None:
+            delta = start_dt - get_epoch()
+            since = int(delta.total_seconds()) * 1000
 
         candles = dict()
-        for asset in assets:
+        for index, asset in enumerate(assets):
             ohlcvs = self.api.fetch_ohlcv(
-                symbol=symbols[0],
+                symbol=symbols[index],
                 timeframe=timeframe,
-                since=ms,
+                since=since,
                 limit=bar_count,
                 params={}
             )
@@ -200,19 +459,29 @@ class CCXT(Exchange):
             candles[asset] = []
             for ohlcv in ohlcvs:
                 candles[asset].append(dict(
-                    last_traded=pd.to_datetime(ohlcv[0], unit='ms', utc=True),
+                    last_traded=pd.to_datetime(
+                        ohlcv[0], unit='ms', utc=True
+                    ),
                     open=ohlcv[1],
                     high=ohlcv[2],
                     low=ohlcv[3],
                     close=ohlcv[4],
                     volume=ohlcv[5]
                 ))
+            candles[asset] = sorted(
+                candles[asset], key=lambda c: c['last_traded']
+            )
 
+        if is_single:
+            return six.next(six.itervalues(candles))
+
+        else:
             return candles
 
     def _fetch_symbol_map(self, is_local):
         try:
             return self.fetch_symbol_map(is_local)
+
         except ExchangeSymbolsNotFound:
             return None
 
@@ -274,7 +543,7 @@ class CCXT(Exchange):
         else:
             return None
 
-    def create_trading_pair(self, market, asset_def, is_local):
+    def create_trading_pair(self, market, asset_def=None, is_local=False):
         """
         Creating a TradingPair from market and asset data.
 
@@ -320,18 +589,24 @@ class CCXT(Exchange):
                    and asset_def['end_minute'] != 'N/A' else None
 
         else:
-            params['symbol'] = self.get_catalyst_symbol(market)
+            params['symbol'] = get_catalyst_symbol(market)
             # TODO: add as an optional column
             params['leverage'] = 1.0
 
         return TradingPair(**params)
 
     def load_assets(self):
+        log.debug('loading assets for {}'.format(self.name))
         self.assets = []
 
         for market in self.markets:
+            if 'id' not in market:
+                log.warn('invalid market: {}'.format(market))
+                continue
+
             asset_defs = self.get_asset_defs(market)
 
+            asset = None
             for asset_def in asset_defs:
                 if asset_def[0] is not None or not asset_defs[1]:
                     try:
@@ -342,8 +617,12 @@ class CCXT(Exchange):
                         )
                         self.assets.append(asset)
 
-                    except TypeError:
+                    except TypeError as e:
-                        pass
+                        log.warn('unable to add asset: {}'.format(e))
+
+            if asset is None:
+                asset = self.create_trading_pair(market=market)
+                self.assets.append(asset)
 
     def get_balances(self):
         try:
@@ -354,8 +633,12 @@ class CCXT(Exchange):
             for key in balances:
                 balances_lower[key.lower()] = balances[key]
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
-            log.debug('error retrieving balances: {}', e)
+            log.warn(
+                'unable to fetch balance {}: {}'.format(
+                    self.name, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         return balances_lower
@@ -375,21 +658,61 @@ class CCXT(Exchange):
             The Catalyst order object
 
         """
-        if order_status['status'] == 'canceled':
+        order_id = order_status['id']
+        symbol = self.get_symbol(order_status['symbol'], source='ccxt')
+        asset = self.get_asset(symbol)
+
+        s = order_status['status']
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if s == 'canceled' or (s == 'closed' and filled == 0):
             status = ORDER_STATUS.CANCELLED
 
-        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
+        elif s == 'closed' and filled > 0:
-            log.debug('found executed order {}'.format(order_status))
+            if filled < amount:
+                log.warn(
+                    'order {id} is executed but only partially filled:'
+                    ' {filled} {symbol} out of {amount}'.format(
+                        id=order_status['status'],
+                        filled=order_status['filled'],
+                        symbol=asset.symbol,
+                        amount=order_status['amount'],
+                    )
+                )
+            else:
+                log.info(
+                    'order {id} executed in full: {filled} {symbol}'.format(
+                        id=order_id,
+                        filled=filled,
+                        symbol=asset.symbol,
+                    )
+                )
+
             status = ORDER_STATUS.FILLED
 
-        elif order_status['status'] == 'open':
+        elif s == 'open':
+            status = ORDER_STATUS.OPEN
+
+        elif filled > 0:
+            log.info(
+                'order {id} partially filled: {filled} {symbol} out of '
+                '{amount}, waiting for complete execution'.format(
+                    id=order_id,
+                    filled=filled,
+                    symbol=asset.symbol,
+                    amount=amount,
+                )
+            )
             status = ORDER_STATUS.OPEN
 
         else:
-            raise ValueError('invalid state for order')
+            log.warn(
-
+                'invalid state {} for order {}'.format(
-        amount = order_status['amount']
+                    s, order_id
-        filled = order_status['filled']
+                )
+            )
+            status = ORDER_STATUS.OPEN
 
         if order_status['side'] == 'sell':
             amount = -amount
@@ -399,25 +722,16 @@ class CCXT(Exchange):
         order_type = order_status['type']
 
         limit_price = price if order_type == 'limit' else None
-        stop_price = None  # TODO: add support
 
         executed_price = order_status['cost'] / order_status['amount']
         commission = order_status['fee']
         date = from_ms_timestamp(order_status['timestamp'])
 
-        # order_id = str(order_status['info']['clientOrderId'])
-        order_id = order_status['id']
-
-        # TODO: this won't work, redo the packages with a different key.
-        symbol = order_status['info']['symbol'] \
-            if 'symbol' in order_status['info'] \
-            else order_status['info']['Exchange']
-
         order = Order(
             dt=date,
-            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            asset=asset,
             amount=amount,
-            stop=stop_price,
+            stop=None,
             limit=limit_price,
             filled=filled,
             id=order_id,
@@ -445,27 +759,55 @@ class CCXT(Exchange):
             )
 
         side = 'buy' if amount > 0 else 'sell'
+        if hasattr(self.api, 'amount_to_lots'):
+            adj_amount = self.api.amount_to_lots(
+                symbol=symbol,
+                amount=abs(amount),
+            )
+            if adj_amount != abs(amount):
+                log.info(
+                    'adjusted order amount {} to {} based on lot size'.format(
+                        abs(amount), adj_amount,
+                    )
+                )
+        else:
+            adj_amount = abs(amount)
+
+        if adj_amount == 0:
+            raise CreateOrderError(
+                exchange=self.name,
+                e='order amount lower than the smallest lot: {}'.format(amount)
+            )
 
         try:
             result = self.api.create_order(
                 symbol=symbol,
                 type=order_type,
                 side=side,
-                amount=abs(amount),
+                amount=adj_amount,
                 price=price
             )
-        except ExchangeNotAvailable as e:
+        except InvalidOrder as e:
-            log.debug('unable to create order: {}'.format(e))
+            log.warn('the exchange rejected the order: {}'.format(e))
+            raise CreateOrderError(exchange=self.name, error=e)
+
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to create order {} / {}: {}'.format(
+                    self.name, symbol, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         if 'info' not in result:
             raise ValueError('cannot use order without info attribute')
 
+        final_amount = adj_amount if side == 'buy' else -adj_amount
         order_id = result['id']
         order = Order(
             dt=pd.Timestamp.utcnow(),
             asset=asset,
-            amount=amount,
+            amount=final_amount,
             stop=style.get_stop_price(is_buy),
             limit=style.get_limit_price(is_buy),
             id=order_id
@@ -481,29 +823,121 @@ class CCXT(Exchange):
                 limit=None,
                 params=dict()
             )
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch open orders {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
         orders = []
         for order_status in result:
-            order, executed_price = self._create_order(order_status)
+            order, _ = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 
         return orders
 
-    def _get_asset_from_order(self, order_id):
+    def _process_order_fallback(self, order):
-        open_orders = self.portfolio.open_orders
+        """
-        order = next(
+        Fallback method for exchanges which do not play nice with
-            (open_orders[id] for id in open_orders if id == order_id),
+        fetch-my-trades. Apparently, about 60% of exchanges will return
-            None
+        the correct executed values with this method. Others will support
-        )  # type: Order
+        fetch-my-trades.
-        return order.asset if order is not None else None
 
-    def get_order(self, order_id, asset_or_symbol=None):
+        Parameters
-        if asset_or_symbol is None and self.portfolio is not None:
+        ----------
-            asset_or_symbol = self._get_asset_from_order(order_id)
+        order: Order
 
+        Returns
+        -------
+        float
+
+        """
+        exc_order, price = self.get_order(
+            order.id, order.asset, return_price=True
+        )
+        order.status = exc_order.status
+
+        order.commission = exc_order.commission
+        if order.amount != exc_order.amount:
+            log.warn(
+                'executed order amount {} differs '
+                'from original'.format(
+                    exc_order.amount, order.amount
+                )
+            )
+            order.amount = exc_order.amount
+
+        if order.status == ORDER_STATUS.FILLED:
+            transaction = Transaction(
+                asset=order.asset,
+                amount=order.amount,
+                dt=pd.Timestamp.utcnow(),
+                price=price,
+                order_id=order.id,
+                commission=order.commission
+            )
+        return [transaction]
+
+    def process_order(self, order):
+        # TODO: move to parent class after tracking features in the parent
+        if not self.api.hasFetchMyTrades:
+            return self._process_order_fallback(order)
+
+        try:
+            all_trades = self.get_trades(order.asset)
+        except ExchangeRequestError as e:
+            log.warn(
+                'unable to fetch account trades, trying an alternate '
+                'method to find executed order {} / {}: {}'.format(
+                    order.id, order.asset.symbol, e
+                )
+            )
+            return self._process_order_fallback(order)
+
+        transactions = []
+        trades = [t for t in all_trades if t['order'] == order.id]
+        if not trades:
+            log.debug(
+                'order {} / {} not found in trades'.format(
+                    order.id, order.asset.symbol
+                )
+            )
+            return transactions
+
+        trades.sort(key=lambda t: t['timestamp'], reverse=False)
+        order.filled = 0
+        order.commission = 0
+        for trade in trades:
+            # status property will update automatically
+            filled = trade['amount'] * order.direction
+            order.filled += filled
+
+            commission = 0
+            if 'fee' in trade and 'cost' in trade['fee']:
+                commission = trade['fee']['cost']
+                order.commission += commission
+
+            order.check_triggers(
+                price=trade['price'],
+                dt=pd.to_datetime(trade['timestamp'], unit='ms', utc=True),
+            )
+            transaction = Transaction(
+                asset=order.asset,
+                amount=filled,
+                dt=pd.Timestamp.utcnow(),
+                price=trade['price'],
+                order_id=order.id,
+                commission=commission
+            )
+            transactions.append(transaction)
+
+        order.broker_order_id = ', '.join([t['id'] for t in trades])
+        return transactions
+
+    def get_order(self, order_id, asset_or_symbol=None, return_price=False):
         if asset_or_symbol is None:
             log.debug(
                 'order not found in memory, the request might fail '
@@ -515,18 +949,24 @@ class CCXT(Exchange):
             order_status = self.api.fetch_order(id=order_id, symbol=symbol)
             order, executed_price = self._create_order(order_status)
 
-        except Exception as e:
+            if return_price:
-            raise ExchangeRequestError(error=e)
-
                 return order, executed_price
 
+            else:
+                return order
+
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
     def cancel_order(self, order_param, asset_or_symbol=None):
         order_id = order_param.id \
             if isinstance(order_param, Order) else order_param
 
-        if asset_or_symbol is None and self.portfolio is not None:
-            asset_or_symbol = self._get_asset_from_order(order_id)
-
         if asset_or_symbol is None:
             log.debug(
                 'order not found in memory, cancelling order might fail '
@@ -537,7 +977,12 @@ class CCXT(Exchange):
                 if asset_or_symbol is not None else None
             self.api.cancel_order(id=order_id, symbol=symbol)
 
-        except Exception as e:
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to cancel order {} / {}: {}'.format(
+                    self.name, order_id, e
+                )
+            )
             raise ExchangeRequestError(error=e)
 
     def tickers(self, assets):
@@ -553,10 +998,26 @@ class CCXT(Exchange):
         list[dict[str, float]
 
         """
-        tickers = dict()
+        tickers = {}
         for asset in assets:
-            ccxt_symbol = self.get_symbol(asset)
+            symbol = self.get_symbol(asset)
-            ticker = self.api.fetch_ticker(ccxt_symbol)
+
+            # Test the CCXT throttling further to see if we need this
+            self.ask_request()
+
+            # TODO: use fetch_tickers() for efficiency
+            # I tried using fetch_tickers() but noticed some
+            # inconsistencies, see issue:
+            # https://github.com/ccxt/ccxt/issues/870
+            try:
+                ticker = self.api.fetch_ticker(symbol=symbol)
+            except (ExchangeError, NetworkError) as e:
+                log.warn(
+                    'unable to fetch ticker {} / {}: {}'.format(
+                        self.name, asset.symbol, e
+                    )
+                )
+                continue
 
             ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
 
@@ -564,9 +1025,17 @@ class CCXT(Exchange):
                 # TODO: any more exceptions?
                 ticker['last_price'] = ticker['last']
 
+            if 'baseVolume' in ticker and ticker['baseVolume'] is not None:
                 # Using the volume represented in the base currency
-            ticker['volume'] = ticker['baseVolume'] \
+                ticker['volume'] = ticker['baseVolume']
-                if 'baseVolume' in ticker else 0
+
+            elif 'info' in ticker and 'bidQty' in ticker['info'] \
+                    and 'askQty' in ticker['info']:
+                ticker['volume'] = float(ticker['info']['bidQty']) + \
+                                   float(ticker['info']['askQty'])
+
+            else:
+                ticker['volume'] = 0
 
             tickers[asset] = ticker
 
@@ -598,3 +1067,27 @@ class CCXT(Exchange):
                 ))
 
         return result
+
+    def get_trades(self, asset, my_trades=True, start_dt=None, limit=None):
+        if not my_trades:
+            raise NotImplemented(
+                'get_trades only supports "my trades"'
+            )
+
+        # TODO: is it possible to sort this? Limit is useless otherwise.
+        ccxt_symbol = self.get_symbol(asset)
+        try:
+            trades = self.api.fetch_my_trades(
+                symbol=ccxt_symbol,
+                since=start_dt,
+                limit=limit,
+            )
+        except (ExchangeError, NetworkError) as e:
+            log.warn(
+                'unable to fetch trades {} / {}: {}'.format(
+                    self.name, asset.symbol, e
+                )
+            )
+            raise ExchangeRequestError(error=e)
+
+        return trades

catalyst/exchange/exchange.py

@@ -5,26 +5,20 @@ from time import sleep
 
 import numpy as np
 import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
-from catalyst.algorithm import MarketOrder
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
-from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField
+    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
-from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
+    TickerNotFoundError, NotEnoughCashError
-    ExchangeLimitOrder, ExchangeStopOrder
+from catalyst.exchange.utils.datetime_utils import get_delta, \
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
+    get_periods_range, \
-from catalyst.exchange.exchange_utils import get_exchange_symbols, \
+    get_periods, get_start_dt, get_frequency
-    get_frequency, resample_history_df
+from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
-from catalyst.finance.order import ORDER_STATUS
+    resample_history_df, has_bundle
-from catalyst.finance.transaction import Transaction
+from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -36,7 +30,6 @@ class Exchange:
         self.name = None
         self.assets = []
         self._symbol_maps = [None, None]
-        self._portfolio = None
         self.minute_writer = None
         self.minute_reader = None
         self.base_currency = None
@@ -46,26 +39,7 @@ class Exchange:
         self.request_cpt = None
         self.bundle = ExchangeBundle(self.name)
 
-    @property
+        self.low_balance_threshold = None
-    def positions(self):
-        return self.portfolio.positions
-
-    @property
-    def portfolio(self):
-        """
-        The exchange portfolio
-
-        Returns
-        -------
-        ExchangePortfolio
-        """
-        if self._portfolio is None:
-            self._portfolio = ExchangePortfolio(
-                start_date=pd.Timestamp.utcnow()
-            )
-            self.synchronize_portfolio()
-
-        return self._portfolio
 
     @abstractproperty
     def account(self):
@@ -75,6 +49,9 @@ class Exchange:
     def time_skew(self):
         pass
 
+    def has_bundle(self, data_frequency):
+        return has_bundle(self.name, data_frequency)
+
     def is_open(self, dt):
         """
         Is the exchange open
@@ -177,7 +154,7 @@ class Exchange:
 
     def get_assets(self, symbols=None, data_frequency=None,
                    is_exchange_symbol=False,
-                   is_local=None):
+                   is_local=None, quote_currency=None):
         """
         The list of markets for the specified symbols.
 
@@ -201,14 +178,30 @@ class Exchange:
         if symbols is None:
             # Make a distinct list of all symbols
             symbols = list(set([asset.symbol for asset in self.assets]))
+
+            if quote_currency is not None:
+                for symbol in symbols[:]:
+                    suffix = '_{}'.format(quote_currency.lower())
+
+                    if not symbol.endswith(suffix):
+                        symbols.remove(symbol)
+
             is_exchange_symbol = False
 
         assets = []
         for symbol in symbols:
+            try:
                 asset = self.get_asset(
                     symbol, data_frequency, is_exchange_symbol, is_local
                 )
                 assets.append(asset)
+
+            except SymbolNotFoundOnExchange:
+                log.debug(
+                    'skipping non-existent market {} {}'.format(
+                        self.name, symbol
+                    )
+                )
         return assets
 
     def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
@@ -241,11 +234,15 @@ class Exchange:
         """
         asset = None
 
+        # TODO: temp mapping, fix to use a single symbol convention
+        og_symbol = symbol
+        symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol
         log.debug(
             'searching assets for: {} {}'.format(
                 self.name, symbol
             )
         )
+        # TODO: simplify and loose the loop
         for a in self.assets:
             if asset is not None:
                 break
@@ -256,8 +253,10 @@ class Exchange:
 
             elif data_frequency is not None:
                 applies = (
-                    (data_frequency == 'minute' and a.end_minute is not None)
+                    (
-                    or (data_frequency == 'daily' and a.end_daily is not None)
+                        data_frequency == 'minute' and a.end_minute is not None)
+                    or (
+                        data_frequency == 'daily' and a.end_daily is not None)
                 )
 
             else:
@@ -265,17 +264,24 @@ class Exchange:
 
             # The symbol provided may use the Catalyst or the exchange
             # convention
-            key = a.exchange_symbol if is_exchange_symbol else a.symbol
+            key = a.exchange_symbol if \
-            if not asset and key.lower() == symbol.lower() and applies:
+                is_exchange_symbol else self.get_symbol(a)
+            if not asset and key.lower() == symbol.lower():
+                if applies:
                     asset = a
 
+                else:
+                    raise NoDataAvailableOnExchange(
+                        symbol=key,
+                        exchange=self.name,
+                        data_frequency=data_frequency,
+                    )
+
         if asset is None:
-            supported_symbols = sorted([
+            supported_symbols = sorted([a.symbol for a in self.assets])
-                asset.symbol for asset in self.assets
-            ])
 
             raise SymbolNotFoundOnExchange(
-                symbol=symbol,
+                symbol=og_symbol,
                 exchange=self.name.title(),
                 supported_symbols=supported_symbols
             )
@@ -293,6 +299,16 @@ class Exchange:
             self._symbol_maps[index] = symbol_map
             return symbol_map
 
+    @abstractmethod
+    def init(self):
+        """
+        Load the asset list from the network.
+
+        Returns
+        -------
+
+        """
+
     @abstractmethod
     def load_assets(self, is_local=False):
         """
@@ -313,54 +329,6 @@ class Exchange:
         """
         pass
 
-    def check_open_orders(self):
-        """
-        Loop through the list of open orders in the Portfolio object.
-        For each executed order found, create a transaction and apply to the
-        Portfolio.
-
-        Returns
-        -------
-        list[Transaction]
-
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-                log.debug('found open order: {}'.format(order_id))
-
-                order, executed_price = self.get_order(order_id)
-                log.debug(
-                    'got updated order {} {}'.format(
-                        order, executed_price
-                    )
-                )
-                if order.status == ORDER_STATUS.FILLED:
-                    transaction = Transaction(
-                        asset=order.asset,
-                        amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
-                        price=executed_price,
-                        order_id=order.id,
-                        commission=order.commission
-                    )
-                    transactions.append(transaction)
-
-                    self.portfolio.execute_order(order, transaction)
-
-                elif order.status == ORDER_STATUS.CANCELLED:
-                    self.portfolio.remove_order(order)
-
-                else:
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta
-                        )
-                    )
-        return transactions
-
     def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
         """
         Public API method that returns a scalar value representing the value
@@ -446,6 +414,7 @@ class Exchange:
 
         return value
 
+    # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 data_frequency, field, previous_value=None):
         """
@@ -470,7 +439,7 @@ class Exchange:
         series = pd.Series(values, index=dates)
 
         periods = get_periods_range(
-            start_dt, end_dt, data_frequency
+            start_dt=start_dt, end_dt=end_dt, freq=data_frequency
         )
         # TODO: ensure that this working as expected, if not use fillna
         series = series.reindex(
@@ -478,7 +447,7 @@ class Exchange:
             method='ffill',
             fill_value=previous_value,
         )
-
+        series.sort_index(inplace=True)
         return series
 
     def get_history_window(self,
@@ -488,7 +457,7 @@ class Exchange:
                            frequency,
                            field,
                            data_frequency=None,
-                           ffill=True):
+                           is_current=False):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -515,10 +484,15 @@ class Exchange:
             The frequency of the data to query; i.e. whether the data is
             'daily' or 'minute' bars.
 
-        # TODO: fill how?
+        is_current: bool
-        ffill: boolean
+            Skip date filters when current data is requested (last few bars
-            Forward-fill missing values. Only has effect if field
+            until now).
-            is 'price'.
+
+        Notes
+        -----
+        Catalysts requires an end data with bar count both CCXT wants a
+        start data with bar count. Since we have to make calculations here,
+        we ensure that the last candle match the end_dt parameter.
 
         Returns
         -------
@@ -529,27 +503,37 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
-        adj_bar_count = candle_size * bar_count
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
             bar_count=bar_count,
-            start_dt=start_dt,
+            end_dt=end_dt if not is_current else None,
-            end_dt=end_dt
         )
 
         series = dict()
         for asset in candles:
+            first_candle = candles[asset][0]
             asset_series = self.get_series_from_candles(
                 candles=candles[asset],
-                start_dt=start_dt,
+                start_dt=first_candle['last_traded'],
                 end_dt=end_dt,
                 data_frequency=frequency,
                 field=field,
             )
+
+            # Checking to make sure that the dates match
+            delta = get_delta(candle_size, data_frequency)
+            adj_end_dt = end_dt - delta
+            last_traded = asset_series.index[-1]
+
+            if last_traded < adj_end_dt:
+                raise LastCandleTooEarlyError(
+                    last_traded=last_traded,
+                    end_dt=adj_end_dt,
+                    exchange=self.name,
+                )
+
             series[asset] = asset_series
 
         df = pd.DataFrame(series)
@@ -603,6 +587,7 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
+        # TODO: this function needs some work, we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
@@ -616,6 +601,7 @@ class Exchange:
                 data_frequency=data_frequency,
                 force_auto_ingest=force_auto_ingest
             )
+
         except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
             series = dict()
 
@@ -632,15 +618,14 @@ class Exchange:
                 # The get_history method supports multiple asset
                 # Use the original frequency to let each api optimize
                 # the size of result sets
-                trailing_bar_count = get_periods(
+                trailing_bars = get_periods(
                     trailing_dt, end_dt, freq
                 )
                 candles = self.get_candles(
                     freq=freq,
                     assets=asset,
-                    bar_count=trailing_bar_count,
+                    end_dt=end_dt,
-                    start_dt=start_dt,
+                    bar_count=trailing_bars if trailing_bars < 500 else 500,
-                    end_dt=end_dt
                 )
 
                 last_value = series[asset].iloc(0) if asset in series \
@@ -669,51 +654,100 @@ class Exchange:
 
         return df
 
-    def synchronize_portfolio(self):
+    def _check_low_balance(self, currency, balances, amount):
+        free = balances[currency]['free'] if currency in balances else 0.0
+
+        if free < amount:
+            return free, True
+
+        else:
+            return free, False
+
+    def sync_positions(self, positions, cash=None, check_balances=False):
         """
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
+        Parameters
+        ----------
+        positions:
+            The positions to synchronize.
+
+        check_balances:
+            Check balances amounts against the exchange.
+
         """
-        log.debug('synchronizing portfolio with exchange {}'.format(self.name))
+        free_cash = 0.0
+        if check_balances:
+            log.debug('fetching {} balances'.format(self.name))
             balances = self.get_balances()
-
+            log.debug(
-        base_position_available = balances[self.base_currency]['free'] \
+                'got free balances for {} currencies'.format(
-            if self.base_currency in balances else None
+                    len(balances)
-
+                )
-        if base_position_available is None:
+            )
-            raise BaseCurrencyNotFoundError(
+            if cash is not None:
-                base_currency=self.base_currency,
+                free_cash, is_lower = self._check_low_balance(
-                exchange=self.name.title()
+                    currency=self.base_currency,
+                    balances=balances,
+                    amount=cash,
+                )
+                if is_lower:
+                    raise NotEnoughCashError(
+                        currency=self.base_currency,
+                        exchange=self.name,
+                        free=free_cash,
+                        cash=cash,
                     )
 
-        portfolio = self._portfolio
+        positions_value = 0.0
-        portfolio.cash = base_position_available
+        if positions is not None:
-        log.debug('found base currency balance: {}'.format(portfolio.cash))
+            assets = set([position.asset for position in positions])
-
-        if portfolio.starting_cash is None:
-            portfolio.starting_cash = portfolio.cash
-
-        if portfolio.positions:
-            assets = list(portfolio.positions.keys())
             tickers = self.tickers(assets)
 
-            portfolio.positions_value = 0.0
+            for position in positions:
-            for asset in tickers:
+                asset = position.asset
-                # TODO: convert if the position is not in the base currency
+                if asset not in tickers:
-                ticker = tickers[asset]
+                    raise TickerNotFoundError(
-                position = portfolio.positions[asset]
+                        symbol=asset.symbol,
+                        exchange=self.name,
+                    )
 
+                ticker = tickers[asset]
+                log.debug(
+                    'updating {symbol} position, last traded on {dt} for '
+                    '{price}{currency}'.format(
+                        symbol=asset.symbol,
+                        dt=ticker['last_traded'],
+                        price=ticker['last_price'],
+                        currency=asset.quote_currency,
+                    )
+                )
                 position.last_sale_price = ticker['last_price']
                 position.last_sale_date = ticker['last_traded']
 
-                portfolio.positions_value += \
+                positions_value += \
                     position.amount * position.last_sale_price
-                portfolio.portfolio_value = \
-                    portfolio.positions_value + portfolio.cash
 
-    def order(self, asset, amount, limit_price=None, stop_price=None,
+                if check_balances:
-              style=None):
+                    free, is_lower = self._check_low_balance(
+                        currency=asset.base_currency,
+                        balances=balances,
+                        amount=position.amount,
+                    )
+
+                    if is_lower:
+                        log.warn(
+                            'detected lower balance for {} on {}: {} < {}, '
+                            'updating position amount'.format(
+                                asset.symbol, self.name, free, position.amount
+                            )
+                        )
+                        position.amount = free
+
+        return free_cash, positions_value
+
+    def order(self, asset, amount, style):
         """Place an order.
 
         Parameters
@@ -772,24 +806,11 @@ class Exchange:
             )
 
         is_buy = (amount > 0)
+        display_price = style.get_limit_price(is_buy)
 
-        if limit_price is not None and stop_price is not None:
-            style = ExchangeStopLimitOrder(
-                limit_price, stop_price, exchange=self.name
-            )
-
-        elif limit_price is not None:
-            style = ExchangeLimitOrder(limit_price, exchange=self.name)
-
-        elif stop_price is not None:
-            style = ExchangeStopOrder(stop_price, exchange=self.name)
-
-        else:
-            style = MarketOrder(exchange=self.name)
-
-        display_price = limit_price if limit_price is not None else stop_price
         log.debug(
-            'issuing {side} order of {amount} {symbol} for {type}: {price}'.format(
+            'issuing {side} order of {amount} {symbol} for {type}:'
+            ' {price}'.format(
                 side='buy' if is_buy else 'sell',
                 amount=amount,
                 symbol=asset.symbol,
@@ -798,12 +819,7 @@ class Exchange:
             )
         )
 
-        order = self.create_order(asset, amount, is_buy, style)
+        return self.create_order(asset, amount, is_buy, style)
-        if order:
-            self._portfolio.create_order(order)
-            return order.id
-        else:
-            return None
 
     # The methods below must be implemented for each exchange.
     @abstractmethod
@@ -886,6 +902,22 @@ class Exchange:
         """
         pass
 
+    @abstractmethod
+    def process_order(self, order):
+        """
+        Similar to get_order but looks only for executed orders.
+
+        Parameters
+        ----------
+        order: Order
+
+        Returns
+        -------
+        float
+            Avg execution price
+
+        """
+
     @abstractmethod
     def cancel_order(self, order_param, symbol_or_asset=None):
         """Cancel an open order.
@@ -900,8 +932,7 @@ class Exchange:
         pass
 
     @abstractmethod
-    def get_candles(self, freq, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None):
-                    start_dt=None, end_dt=None):
         """
         Retrieve OHLCV candles for the given assets
 
@@ -966,7 +997,7 @@ class Exchange:
     @abc.abstractmethod
     def get_orderbook(self, asset, order_type, limit):
         """
-        Retrieve the the orderbook for the given trading pair.
+        Retrieve the orderbook for the given trading pair.
 
         Parameters
         ----------
@@ -980,3 +1011,20 @@ class Exchange:
         list[dict[str, float]
         """
         pass
+
+    @abc.abstractmethod
+    def get_trades(self, asset, my_trades, start_dt, limit):
+        """
+        Retrieve a list of trades.
+
+        Parameters
+        ----------
+        my_trades: bool
+            List only my trades.
+        start_dt
+        limit
+
+        Returns
+        -------
+
+        """

catalyst/exchange/exchange_algorithm.py

@@ -10,47 +10,44 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import pickle
 import signal
 import sys
-from collections import deque
 from datetime import timedelta
 from os import listdir
 from os.path import isfile, join
-from time import sleep
-
-import logbook
-import pandas as pd
-from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
+import logbook
+import pandas as pd
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
-from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangePortfolioDataError,
+    OrderTypeNotSupported)
-    ExchangeTransactionError,
+from catalyst.exchange.exchange_execution import ExchangeLimitOrder
-    OrphanOrderError)
-from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
-    ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
-    get_algo_folder, get_algo_df, \
-    save_algo_df
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.exchange.utils.exchange_utils import (
+    save_algo_object,
+    get_algo_object,
+    get_algo_folder,
+    get_algo_df,
+    save_algo_df,
+    group_assets_by_exchange, )
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
+    stats_to_algo_folder
 from catalyst.finance.execution import MarketOrder
+from catalyst.finance.performance import PerformanceTracker
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
-from catalyst.utils.api_support import (
+from catalyst.utils.api_support import api_method
-    api_method,
+from catalyst.utils.input_validation import error_keywords, ensure_upper_case
-    disallowed_in_before_trading_start)
-from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
-    expect_types
 from catalyst.utils.math_utils import round_nearest
 from catalyst.utils.preprocess import preprocess
+from redo import retry
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
@@ -63,9 +60,104 @@ class ExchangeAlgorithmExecutor(AlgorithmSimulator):
 class ExchangeTradingAlgorithmBase(TradingAlgorithm):
     def __init__(self, *args, **kwargs):
         self.exchanges = kwargs.pop('exchanges', None)
+        self.simulate_orders = kwargs.pop('simulate_orders', None)
 
         super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
 
+        self.current_day = None
+
+        if self.simulate_orders is None \
+                and self.sim_params.arena == 'backtest':
+            self.simulate_orders = True
+
+        # Operations with retry features
+        self.attempts = dict(
+            get_transactions_attempts=5,
+            order_attempts=5,
+            synchronize_portfolio_attempts=5,
+            get_order_attempts=5,
+            get_open_orders_attempts=5,
+            cancel_order_attempts=5,
+            get_spot_value_attempts=5,
+            get_history_window_attempts=5,
+            retry_sleeptime=5,
+        )
+
+        self.blotter = ExchangeBlotter(
+            data_frequency=self.data_frequency,
+            # Default to NeverCancel in catalyst
+            cancel_policy=self.cancel_policy,
+            simulate_orders=self.simulate_orders,
+            exchanges=self.exchanges,
+            attempts=self.attempts,
+        )
+
+    @staticmethod
+    def __convert_order_params_for_blotter(limit_price, stop_price, style):
+        """
+        Helper method for converting deprecated limit_price and stop_price
+        arguments into ExecutionStyle instances.
+
+        This function assumes that either style == None or (limit_price,
+        stop_price) == (None, None).
+        """
+        if stop_price:
+            raise OrderTypeNotSupported(order_type='stop')
+
+        if style:
+            if limit_price is not None:
+                raise ValueError(
+                    'An order style and a limit price was included in the '
+                    'order. Please pick one to avoid any possible conflict.'
+                )
+
+            # Currently limiting order types or limit and market to
+            # be in-line with CXXT and many exchanges. We'll consider
+            # adding more order types in the future.
+            if not isinstance(style, ExchangeLimitOrder) or \
+                    not isinstance(style, MarketOrder):
+                raise OrderTypeNotSupported(
+                    order_type=style.__class__.__name__
+                )
+
+            return style
+
+        if limit_price:
+            return ExchangeLimitOrder(limit_price)
+        else:
+            return MarketOrder()
+
+    @api_method
+    def set_commission(self, maker=None, taker=None):
+        key = list(self.blotter.commission_models.keys())[0]
+        if maker is not None:
+            self.blotter.commission_models[key].maker = maker
+
+        if taker is not None:
+            self.blotter.commission_models[key].taker = taker
+
+    @api_method
+    def set_slippage(self, spread=None):
+        key = list(self.blotter.slippage_models.keys())[0]
+        if spread is not None:
+            self.blotter.slippage_models[key].spread = spread
+
+    def _calculate_order(self, asset, amount,
+                         limit_price=None, stop_price=None, style=None):
+        # Raises a ZiplineError if invalid parameters are detected.
+        self.validate_order_params(asset,
+                                   amount,
+                                   limit_price,
+                                   stop_price,
+                                   style)
+
+        # Convert deprecated limit_price and stop_price parameters to use
+        # ExecutionStyle objects.
+        style = self.__convert_order_params_for_blotter(limit_price,
+                                                        stop_price,
+                                                        style)
+        return amount, style
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies
@@ -140,28 +232,28 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         """
         tracker = self.perf_tracker
-        period = tracker.todays_performance
+        cum = tracker.cumulative_performance
 
-        pos_stats = period.position_tracker.stats()
+        pos_stats = cum.position_tracker.stats()
-        period_stats = calc_period_stats(pos_stats, period.ending_cash)
+        period_stats = calc_period_stats(pos_stats, cum.ending_cash)
 
         stats = dict(
             period_start=tracker.period_start,
             period_end=tracker.period_end,
             capital_base=tracker.capital_base,
             progress=tracker.progress,
-            ending_value=period.ending_value,
+            ending_value=cum.ending_value,
-            ending_exposure=period.ending_exposure,
+            ending_exposure=cum.ending_exposure,
-            capital_used=period.cash_flow,
+            capital_used=cum.cash_flow,
-            starting_value=period.starting_value,
+            starting_value=cum.starting_value,
-            starting_exposure=period.starting_exposure,
+            starting_exposure=cum.starting_exposure,
-            starting_cash=period.starting_cash,
+            starting_cash=cum.starting_cash,
-            ending_cash=period.ending_cash,
+            ending_cash=cum.ending_cash,
-            portfolio_value=period.ending_cash + period.ending_value,
+            portfolio_value=cum.ending_cash + cum.ending_value,
-            pnl=period.pnl,
+            pnl=cum.pnl,
-            returns=period.returns,
+            returns=cum.returns,
-            period_open=period.period_open,
+            period_open=start_dt,
-            period_close=period.period_close,
+            period_close=end_dt,
             gross_leverage=period_stats.gross_leverage,
             net_leverage=period_stats.net_leverage,
             short_exposure=pos_stats.short_exposure,
@@ -178,6 +270,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         # Merging latest recorded variables
         stats.update(self.recorded_vars)
 
+        period = tracker.todays_performance
         stats['positions'] = period.position_tracker.get_positions_list()
 
         # we want the key to be absent, not just empty
@@ -198,56 +291,21 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         return stats
 
+    def run(self, data=None, overwrite_sim_params=True):
+        data.attempts = self.attempts
+        return super(ExchangeTradingAlgorithmBase, self).run(
+            data, overwrite_sim_params
+        )
+
 
 class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
         self.frame_stats = list()
-        self.blotter = ExchangeBlotter(
+        self.state = {}
-            data_frequency=self.data_frequency,
-            # Default to NeverCancel in catalyst
-            cancel_policy=self.cancel_policy,
-        )
         log.info('initialized trading algorithm in backtest mode')
 
-    def _calculate_order(self, asset, amount,
-                         limit_price=None, stop_price=None, style=None):
-        # Raises a ZiplineError if invalid parameters are detected.
-        self.validate_order_params(asset,
-                                   amount,
-                                   limit_price,
-                                   stop_price,
-                                   style)
-
-        # Convert deprecated limit_price and stop_price parameters to use
-        # ExecutionStyle objects.
-        style = self.__convert_order_params_for_blotter(limit_price,
-                                                        stop_price,
-                                                        style)
-        return amount, style
-
-    @staticmethod
-    def __convert_order_params_for_blotter(limit_price, stop_price, style):
-        """
-        Helper method for converting deprecated limit_price and stop_price
-        arguments into ExecutionStyle instances.
-
-        This function assumes that either style == None or (limit_price,
-        stop_price) == (None, None).
-        """
-        if style:
-            assert (limit_price, stop_price) == (None, None)
-            return style
-        if limit_price and stop_price:
-            return ExchangeStopLimitOrder(limit_price, stop_price)
-        if limit_price:
-            return ExchangeLimitOrder(limit_price)
-        if stop_price:
-            return ExchangeStopOrder(stop_price)
-        else:
-            return MarketOrder()
-
     def is_last_frame_of_day(self, data):
         # TODO: adjust here to support more intervals
         next_frame_dt = data.current_dt + timedelta(minutes=1)
@@ -265,6 +323,8 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
             )
             self.frame_stats.append(frame_stats)
 
+        self.current_day = data.current_dt.floor('1D')
+
     def _create_stats_df(self):
         stats = pd.DataFrame(self.frame_stats)
         stats.set_index('period_close', inplace=True, drop=False)
@@ -289,10 +349,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
-        self.simulate_orders = kwargs.pop('simulate_orders', None)
+        self.stats_output = kwargs.pop('stats_output', None)
+        self._analyze_live = kwargs.pop('analyze_live', None)
+        self.end = kwargs.pop('end', None)
 
         self._clock = None
-        self.frame_stats = deque(maxlen=60)
+        self.frame_stats = list()
 
         self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
 
@@ -304,20 +366,51 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         self.is_running = True
 
-        self.retry_check_open_orders = 5
+        self.stats_minutes = 1
-        self.retry_synchronize_portfolio = 5
-        self.retry_get_open_orders = 5
-        self.retry_order = 2
-        self.retry_delay = 5
 
-        self.stats_minutes = 5
+        self._last_orders = []
+        self.trading_client = None
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
 
+        try:
             signal.signal(signal.SIGINT, self.signal_handler)
+        except ValueError:
+            log.warn("Can't initialize signal handler inside another thread."
+                     "Exit should be handled by the user.")
 
         log.info('initialized trading algorithm in live mode')
 
+    def interrupt_algorithm(self):
+        self.is_running = False
+
+        if self._analyze is None:
+            log.info('Exiting the algorithm.')
+
+        else:
+            log.info('Exiting the algorithm. Calling `analyze()` '
+                     'before exiting the algorithm.')
+
+            algo_folder = get_algo_folder(self.algo_namespace)
+            folder = join(algo_folder, 'daily_performance')
+            files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+            daily_perf_list = []
+            for item in files:
+                filename = join(folder, item)
+
+                with open(filename, 'rb') as handle:
+                    perf_period = pickle.load(handle)
+                    perf_period_dict = perf_period.to_dict()
+                    daily_perf_list.append(perf_period_dict)
+
+            stats = pd.DataFrame(daily_perf_list)
+            stats.set_index('period_close', drop=False, inplace=True)
+
+            self.analyze(stats)
+
+        sys.exit(0)
+
     def signal_handler(self, signal, frame):
         """
         Handles the keyboard interruption signal.
@@ -331,31 +424,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         -------
 
         """
-        self.is_running = False
-
-        if self._analyze is None:
         log.info('Interruption signal detected {}, exiting the '
                  'algorithm'.format(signal))
-
+        self.interrupt_algorithm()
-        else:
-            log.info('Interruption signal detected {}, calling `analyze()` '
-                     'before exiting the algorithm'.format(signal))
-
-            algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
-
-            daily_perf_list = []
-            for item in files:
-                filename = join(folder, item)
-                with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
-
-            stats = pd.DataFrame(daily_perf_list)
-
-            self.analyze(stats)
-
-        sys.exit(0)
 
     @property
     def clock(self):
@@ -378,13 +449,14 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         # This method is taken from TradingAlgorithm.
         # The clock has been replaced to use RealtimeClock
-        # TODO: should we apply a time skew? not sure to understand the utility.
+        # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
-        if self.live_graph:
+        if self.live_graph or self._analyze_live is not None:
             self._clock = LiveGraphClock(
                 self.sim_params.sessions,
-                context=self
+                context=self,
+                callback=self._analyze_live,
             )
         else:
             self._clock = SimpleClock(
@@ -393,100 +465,162 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
 
         return self._clock
 
-    def _create_generator(self, sim_params):
+    def _init_trading_client(self):
-        if self.perf_tracker is None:
+        """
-            self.perf_tracker = get_algo_object(
+        This replaces Ziplines `_create_generator` method. The main difference
+        is that we are restoring performance tracker objects if available.
+        This allows us to stop/start algos without loosing their state.
+
+        """
+        self.state = get_algo_object(
             algo_name=self.algo_namespace,
-                key='perf_tracker'
+            key='context.state',
+        )
+        if self.state is None:
+            self.state = {}
+
+        if self.perf_tracker is None:
+            # Note from the Zipline dev:
+            # HACK: When running with the `run` method, we set perf_tracker to
+            # None so that it will be overwritten here.
+            tracker = self.perf_tracker = PerformanceTracker(
+                sim_params=self.sim_params,
+                trading_calendar=self.trading_calendar,
+                env=self.trading_environment,
+            )
+            # Set the dt initially to the period start by forcing it to change.
+            self.on_dt_changed(self.sim_params.start_session)
+
+            new_position_tracker = tracker.position_tracker
+            tracker.position_tracker = None
+
+            # Unpacking the perf_tracker and positions if available
+            cum_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key='cumulative_performance',
+            )
+            if cum_perf is not None:
+                tracker.cumulative_performance = cum_perf
+                # Ensure single common position tracker
+                tracker.position_tracker = cum_perf.position_tracker
+
+            today = pd.Timestamp.utcnow().floor('1D')
+            todays_perf = get_algo_object(
+                algo_name=self.algo_namespace,
+                key=today.strftime('%Y-%m-%d'),
+                rel_path='daily_performance',
+            )
+            if todays_perf is not None:
+                # Ensure single common position tracker
+                if tracker.position_tracker is not None:
+                    todays_perf.position_tracker = tracker.position_tracker
+                else:
+                    tracker.position_tracker = todays_perf.position_tracker
+
+                tracker.todays_performance = todays_perf
+
+            if tracker.position_tracker is None:
+                # Use a new position_tracker if not is found in the state
+                tracker.position_tracker = new_position_tracker
+
+        if not self.initialized:
+            # Calls the initialize function of the algorithm
+            self.initialize(*self.initialize_args, **self.initialize_kwargs)
+            self.initialized = True
+
+        self.trading_client = ExchangeAlgorithmExecutor(
+            algo=self,
+            sim_params=self.sim_params,
+            data_portal=self.data_portal,
+            clock=self.clock,
+            benchmark_source=self._create_benchmark_source(),
+            restrictions=self.restrictions,
+            universe_func=self._calculate_universe,
         )
 
-        # Call the simulation trading algorithm for side-effects:
+    def get_generator(self):
-        # it creates the perf tracker
+        if self.trading_client is None:
-        TradingAlgorithm._create_generator(self, sim_params)
+            self._init_trading_client()
-        self.trading_client = ExchangeAlgorithmExecutor(
-            self,
-            sim_params,
-            self.data_portal,
-            self.clock,
-            self._create_benchmark_source(),
-            self.restrictions,
-            universe_func=self._calculate_universe
-        )
 
         return self.trading_client.transform()
 
     def updated_portfolio(self):
-        """
-        We skip the entire performance tracker business and update the
-        portfolio directly.
-
-        Returns
-        -------
-        ExchangePortfolio
-
-        """
-        # TODO: build cumulative portfolio
         return self.perf_tracker.get_portfolio(False)
 
     def updated_account(self):
         return self.perf_tracker.get_account(False)
 
-    def _synchronize_portfolio(self, attempt_index=0):
+    def synchronize_portfolio(self):
-        try:
+        """
+        Synchronizes the portfolio tracked by the algorithm to refresh
+        its current value.
+
+        This includes updating the last_sale_price of all tracked
+        positions, returning the available cash, and raising error
+        if the data goes out of sync.
+
+        Returns
+        -------
+        float
+            The amount of base currency available for trading.
+
+        float
+            The total value of all tracked positions.
+
+        """
+        check_balances = (not self.simulate_orders)
+        base_currency = None
+        tracker = self.perf_tracker.position_tracker
+        total_cash = 0.0
+        total_positions_value = 0.0
+
+        # Position keys correspond to assets
+        positions = self.portfolio.positions
+        assets = list(positions)
+        exchange_assets = group_assets_by_exchange(assets)
         for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
+            assets = exchange_assets[exchange_name] \
+                if exchange_name in exchange_assets else []
 
-                exchange.synchronize_portfolio()
+            exchange_positions = copy.deepcopy(
+                [positions[asset] for asset in assets if asset in positions]
+            )
 
-                # Applying the updated last_sales_price to the positions
+            exchange = self.exchanges[exchange_name]  # Type: Exchange
-                # in the performance tracker. This seems a bit redundant
+
-                # but it will make sense when we have multiple exchange portfolios
+            if base_currency is None:
-                # feeding into the same performance tracker.
+                base_currency = exchange.base_currency
-                tracker = self.perf_tracker.todays_performance.position_tracker
+
-                for asset in exchange.portfolio.positions:
+            # Don't check the cash if there are open orders. This could
-                    position = exchange.portfolio.positions[asset]
+            # results in false positives.
+            orders = []
+            for asset in self.blotter.open_orders:
+                asset_orders = self.blotter.open_orders[asset]
+                if asset_orders:
+                    orders += asset_orders
+
+            required_cash = self.portfolio.cash if not orders else None
+            cash, positions_value = exchange.sync_positions(
+                positions=exchange_positions,
+                check_balances=check_balances,
+                cash=required_cash,
+            )
+            total_cash += cash
+            total_positions_value += positions_value
+
+            # Applying modifications to the original positions
+            for position in exchange_positions:
                 tracker.update_position(
-                        asset=asset,
+                    asset=position.asset,
+                    amount=position.amount,
                     last_sale_date=position.last_sale_date,
-                        last_sale_price=position.last_sale_price
+                    last_sale_price=position.last_sale_price,
-                    )
-        except ExchangeRequestError as e:
-            log.warn(
-                'update portfolio attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_synchronize_portfolio:
-                sleep(self.retry_delay)
-                self._synchronize_portfolio(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='update-portfolio',
-                    attempts=attempt_index,
-                    error=e
                 )
 
-    def _check_open_orders(self, attempt_index=0):
+        if not check_balances:
-        try:
+            total_cash = self.portfolio.cash
-            orders = list()
-            for exchange_name in self.exchanges:
-                exchange = self.exchanges[exchange_name]
-                exchange_orders = exchange.check_open_orders()
 
-                orders += exchange_orders
+        return total_cash, total_positions_value
-
-            return orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self._check_open_orders(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
 
     def add_pnl_stats(self, period_stats):
         """
@@ -577,15 +711,42 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if not self.is_running:
             return
 
-        self._synchronize_portfolio()
+        if self.end is not None and self.end < data.current_dt:
+            log.info('Algorithm has reached specified end time. Finishing...')
+            self.interrupt_algorithm()
 
-        transactions = self._check_open_orders()
+        # Resetting the frame stats every day to minimize memory footprint
-        if len(transactions) > 0:
+        today = data.current_dt.floor('1D')
-            for transaction in transactions:
+        if self.current_day is not None and today > self.current_day:
-                self.perf_tracker.process_transaction(transaction)
+            self.frame_stats = list()
 
+        self.performance_needs_update = False
+        orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
+        if orders != self._last_orders:
+            self.performance_needs_update = True
+
+        # Saving current orders to detect changes in the next frame
+        self._last_orders = copy.deepcopy(orders)
+
+        if self.performance_needs_update:
             self.perf_tracker.update_performance()
+            self.performance_needs_update = False
 
+        if self.portfolio_needs_update:
+            cash, positions_value = retry(
+                action=self.synchronize_portfolio,
+                attempts=self.attempts['synchronize_portfolio_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.')
+            )
+            self.portfolio_needs_update = False
+
+        log.info(
+            'portfolio balances, cash: {}, positions: {}'.format(
+                cash, positions_value
+            )
+        )
         if self._handle_data:
             self._handle_data(self, data)
 
@@ -594,13 +755,45 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        self._save_algo_state(data)
+        self.current_day = data.current_dt.floor('1D')
+
+    def _save_algo_state(self, data):
+        today = data.current_dt.floor('1D')
         try:
+            self._save_stats_csv(self._process_stats(data))
+        except Exception as e:
+            log.warn('unable to calculate performance: {}'.format(e))
+
+        log.debug('saving cumulative performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='cumulative_performance',
+            obj=self.perf_tracker.cumulative_performance,
+        )
+        log.debug('saving todays performance object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key=today.strftime('%Y-%m-%d'),
+            obj=self.perf_tracker.todays_performance,
+            rel_path='daily_performance'
+        )
+        log.debug('saving context.state object')
+        save_algo_object(
+            algo_name=self.algo_namespace,
+            key='context.state',
+            obj=self.state)
+
+    def _process_stats(self, data):
+        today = data.current_dt.floor('1D')
+
         # Since the clock runs 24/7, I trying to disable the daily
         # Performance tracker and keep only minute and cumulative
         self.perf_tracker.update_performance()
 
         frame_stats = self.prepare_period_stats(
-                data.current_dt, data.current_dt + timedelta(minutes=1))
+            data.current_dt, data.current_dt + timedelta(minutes=1)
+        )
 
         # Saving the last hour in memory
         self.frame_stats.append(frame_stats)
@@ -609,141 +802,66 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if self.recorded_vars:
             self.add_custom_signals_stats(frame_stats)
             recorded_cols = list(self.recorded_vars.keys())
+
         else:
             recorded_cols = None
 
         self.add_exposure_stats(frame_stats)
 
-            print_df = pd.DataFrame(list(self.frame_stats))
         log.info(
-                'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
+            'statistics for the last {stats_minutes} minutes:\n'
+            '{stats}'.format(
                 stats_minutes=self.stats_minutes,
                 stats=get_pretty_stats(
-                        stats_df=print_df,
+                    stats=self.frame_stats,
                     recorded_cols=recorded_cols,
-                        num_rows=self.stats_minutes
+                    num_rows=self.stats_minutes,
                 )
             ))
 
-            today = pd.to_datetime('today', utc=True)
+        # Saving the daily stats in a format usable for performance
+        # analysis.
         daily_stats = self.prepare_period_stats(
             start_dt=today,
-                end_dt=pd.Timestamp.utcnow()
+            end_dt=data.current_dt
-            )
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key=today.strftime('%Y-%m-%d'),
-                obj=daily_stats,
-                rel_path='daily_perf'
         )
 
-        except Exception as e:
+        return recorded_cols
-            log.warn('unable to calculate performance: {}'.format(e))
 
-        # TODO: pickle does not seem to work in python 3
+    def _save_stats_csv(self, recorded_cols):
+        # Writing the stats output
+        csv_bytes = None
         try:
-            save_algo_object(
+            csv_bytes = stats_to_algo_folder(
-                algo_name=self.algo_namespace,
+                stats=self.frame_stats,
-                key='perf_tracker',
+                algo_namespace=self.algo_namespace,
-                obj=self.perf_tracker
+                recorded_cols=recorded_cols,
             )
         except Exception as e:
-            log.warn('unable to save minute perfs to disk: {}'.format(e))
+            log.warn('unable save stats locally: {}'.format(e))
 
         try:
-            for exchange_name in self.exchanges:
+            if self.stats_output is not None:
-                exchange = self.exchanges[exchange_name]
+                if 's3://' in self.stats_output:
-                save_algo_object(
+                    stats_to_s3(
-                    algo_name=self.algo_namespace,
+                        uri=self.stats_output,
-                    key='portfolio_{}'.format(exchange_name),
+                        stats=self.frame_stats,
-                    obj=exchange.portfolio
+                        algo_namespace=self.algo_namespace,
+                        recorded_cols=recorded_cols,
+                        bytes_to_write=csv_bytes
+                    )
+                else:
+                    raise ValueError(
+                        'Only S3 stats output is supported for now.'
                     )
         except Exception as e:
-            log.warn('unable to save portfolio to disk: {}'.format(e))
+            log.warn('unable save stats externally: {}'.format(e))
-
-    def _order(self,
-               asset,
-               amount,
-               limit_price=None,
-               stop_price=None,
-               style=None,
-               attempt_index=0):
-        try:
-            exchange = self.exchanges[asset.exchange]
-            return exchange.order(asset, amount, limit_price,
-                                  stop_price,
-                                  style)
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-                return self._order(
-                    asset, amount, limit_price, stop_price, style,
-                    attempt_index + 1)
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    @api_method
-    @disallowed_in_before_trading_start(OrderInBeforeTradingStart())
-    @expect_types(asset=TradingPair)
-    def order(self,
-              asset,
-              amount,
-              limit_price=None,
-              stop_price=None,
-              style=None):
-        """
-        We use the exchange specific portfolio to place orders.
-        The cumulative portfolio does not contain open orders but exchange
-        portfolios do.
-
-        Parameters
-        ----------
-        asset: TradingPair
-        amount: float
-        limit_price: float
-        stop_price: float
-        style: Style
-        order: Order
-            The catalyst order object or None
-        """
-        amount, style = self._calculate_order(asset, amount,
-                                              limit_price, stop_price,
-                                              style)
-
-        order_id = self._order(asset, amount, limit_price, stop_price, style)
-
-        exchange = self.exchanges[asset.exchange]
-        exchange_portfolio = exchange.portfolio
-        if order_id is not None:
-
-            if order_id in exchange_portfolio.open_orders:
-                order = exchange_portfolio.open_orders[order_id]
-                self.perf_tracker.process_order(order)
-                return order
-
-            else:
-                raise OrphanOrderError(
-                    order_id=order_id,
-                    exchange=exchange.name
-                )
-        else:
-            log.warn('unable to order {} {} on exchange {}'.format(
-                amount, asset.symbol, asset.exchange))
-            return None
 
     @api_method
     def batch_market_order(self, share_counts):
         raise NotImplementedError()
 
-    def _get_open_orders(self, asset=None, attempt_index=0):
+    def _get_open_orders(self, asset=None):
-        try:
         if asset:
             exchange = self.exchanges[asset.exchange]
             return exchange.get_open_orders(asset)
@@ -756,19 +874,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 open_orders.append(exchange_orders)
 
             return open_orders
-        except ExchangeRequestError as e:
-            log.warn(
-                'open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_open_orders:
-                sleep(self.retry_delay)
-                return self._get_open_orders(asset, attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='open-orders',
-                    attempts=attempt_index,
-                    error=e
-                )
 
     @error_keywords(sid='Keyword argument `sid` is no longer supported for '
                         'get_open_orders. Use `asset` instead.')
@@ -790,7 +895,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             If an asset is passed then this will return a list of the open
             orders for this asset.
         """
-        return self._get_open_orders(asset)
+        return retry(
+            action=self._get_open_orders,
+            attempts=self.attempts['get_open_orders_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching open orders again.'),
+            args=(asset,))
 
     @api_method
     def get_order(self, order_id, exchange_name):
@@ -810,7 +921,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             The execution price per share of the order
         """
         exchange = self.exchanges[exchange_name]
-        return exchange.get_order(order_id)
+        return retry(
+            action=exchange.get_order,
+            attempts=self.attempts['get_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('Fetching orders again.'),
+            args=(order_id,))
 
     @api_method
     def cancel_order(self, order_param, exchange_name):
@@ -827,4 +944,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         if isinstance(order_param, zp.Order):
             order_id = order_param.id
 
-        exchange.cancel_order(order_id)
+        retry(
+            action=exchange.cancel_order,
+            attempts=self.attempts['cancel_order_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('cancelling order again.'),
+            args=(order_id,))

catalyst/exchange/exchange_asset_finder.py

@@ -0,0 +1,179 @@
+import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.factory import find_exchanges
+from logbook import Logger
+
+log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
+
+
+class ExchangeAssetFinder(object):
+    def __init__(self, exchanges):
+        self.exchanges = exchanges
+
+    @property
+    def sids(self):
+        """
+        This seems to be used to pre-fetch assets.
+        I don't think that we need this for live-trading.
+        Leaving the list empty.
+        """
+        all_sids = []
+        for exchange_name in self.exchanges:
+            # This is what initializes each exchanges at the beginning
+            # of an algo
+            exchange = self.exchanges[exchange_name]
+            exchange.init()
+
+            all_sids += [asset.sid for asset in exchange.assets]
+
+        sids = list(set(all_sids))
+        return sids
+
+    def retrieve_asset(self, sid, default_none=False):
+        """
+        Retrieve the first Asset found for a given sid.
+        """
+        asset = None
+        for exchange_name in self.exchanges:
+            if asset is not None:
+                break
+
+            exchange = self.exchanges[exchange_name]
+            assets = [asset for asset in exchange.assets if asset.sid == sid]
+            if assets:
+                asset = assets[0]
+
+        return asset
+
+    def retrieve_all(self, sids, default_none=False):
+        """
+        Retrieve all assets in `sids`.
+
+        Parameters
+        ----------
+        sids : iterable of int
+            Assets to retrieve.
+        default_none : bool
+            If True, return None for failed lookups.
+            If False, raise `SidsNotFound`.
+
+        Returns
+        -------
+        assets : list[Asset or None]
+            A list of the same length as `sids` containing Assets (or Nones)
+            corresponding to the requested sids.
+
+        Raises
+        ------
+        SidsNotFound
+            When a requested sid is not found and default_none=False.
+        """
+        assets = []
+        for exchange_name in self.exchanges:
+            exchange = self.exchanges[exchange_name]
+            xas = [asset for asset in exchange.assets if asset.sid in sids]
+            assets += xas
+
+        return assets
+
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
+        """Lookup an asset by symbol.
+
+        Parameters
+        ----------
+        symbol : str
+            The ticker symbol to resolve.
+        as_of_date : datetime or None
+            Look up the last owner of this symbol as of this datetime.
+            If ``as_of_date`` is None, then this can only resolve the equity
+            if exactly one equity has ever owned the ticker.
+        fuzzy : bool, optional
+            Should fuzzy symbol matching be used? Fuzzy symbol matching
+            attempts to resolve differences in representations for
+            shareclasses. For example, some people may represent the ``A``
+            shareclass of ``BRK`` as ``BRK.A``, where others could write
+            ``BRK_A``.
+
+        Returns
+        -------
+        equity : Asset
+            The equity that held ``symbol`` on the given ``as_of_date``, or the
+            only equity to hold ``symbol`` if ``as_of_date`` is None.
+
+        Raises
+        ------
+        SymbolNotFound
+            Raised when no equity has ever held the given symbol.
+        MultipleSymbolsFound
+            Raised when no ``as_of_date`` is given and more than one equity
+            has held ``symbol``. This is also raised when ``fuzzy=True`` and
+            there are multiple candidates for the given ``symbol`` on the
+            ``as_of_date``.
+        """
+        log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
+
+        return exchange.get_asset(symbol, data_frequency)
+
+    def lifetimes(self, dates, include_start_date):
+        """
+        Compute a DataFrame representing asset lifetimes for the specified date
+        range.
+
+        Parameters
+        ----------
+        dates : pd.DatetimeIndex
+            The dates for which to compute lifetimes.
+        include_start_date : bool
+            Whether or not to count the asset as alive on its start_date.
+
+            This is useful in a backtesting context where `lifetimes` is being
+            used to signify "do I have data for this asset as of the morning of
+            this date?"  For many financial metrics, (e.g. daily close), data
+            isn't available for an asset until the end of the asset's first
+            day.
+
+        Returns
+        -------
+        lifetimes : pd.DataFrame
+            A frame of dtype bool with `dates` as index and an Int64Index of
+            assets as columns.  The value at `lifetimes.loc[date, asset]` will
+            be True iff `asset` existed on `date`.  If `include_start_date` is
+            False, then lifetimes.loc[date, asset] will be false when date ==
+            asset.start_date.
+
+        See Also
+        --------
+        numpy.putmask
+        catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
+        """
+        exchanges = find_exchanges(features=['minuteBundle'])
+        if not exchanges:
+            raise ValueError('exchange with minute bundles not found')
+
+        # TODO: find a way to support multiple exchanges
+        exchange = exchanges[0]
+        # Using a single exchange for now because are not unique for the
+        # same asset in different exchanges. I'd like to avoid binding
+        # pipeline to a single exchange.
+        exchange.init()
+
+        data = []
+        for dt in dates:
+            exists = []
+
+            for asset in exchange.assets:
+                if include_start_date:
+                    condition = (asset.start_date <= dt < asset.end_minute)
+
+                else:
+                    condition = (asset.start_date < dt < asset.end_minute)
+
+                exists.append(condition)
+
+            data.append(exists)
+
+        sids = [asset.sid for asset in exchange.assets]
+        df = pd.DataFrame(data, index=dates, columns=exchange.assets)
+
+        return df

catalyst/exchange/exchange_blotter.py

@@ -1,21 +1,20 @@
-from catalyst.assets._assets import TradingPair
+import numpy as np
+import pandas as pd
 from logbook import Logger
+from redo import retry
 
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange_errors import ExchangeRequestError
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
+from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.slippage import SlippageModel
-from catalyst.finance.transaction import create_transaction
+from catalyst.finance.transaction import create_transaction, Transaction
+from catalyst.utils.input_validation import expect_types
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
-# It seems like we need to accept greater slippage risk in cryptos
-# Orders won't often close at Equity levels.
-# TODO: should work with set_commission and set_slippage
-DEFAULT_SLIPPAGE_SPREAD = 0.0001
-DEFAULT_MAKER_FEE = 0.0015
-DEFAULT_TAKER_FEE = 0.0025
-
 
 class TradingPairFeeSchedule(CommissionModel):
     """
@@ -23,40 +22,55 @@ class TradingPairFeeSchedule(CommissionModel):
 
     Parameters
     ----------
-    fee : float, optional
+    maker : float, optional
-        The percentage fee.
+        The percentage maker fee.
+
+    taker: float, optional
+        The percentage taker fee.
     """
 
-    def __init__(self,
+    def __init__(self, maker=None, taker=None):
-                 maker_fee=DEFAULT_MAKER_FEE,
+        self.maker = maker
-                 taker_fee=DEFAULT_TAKER_FEE):
+        self.taker = taker
-        self.maker_fee = maker_fee
-        self.taker_fee = taker_fee
 
     def __repr__(self):
         return (
-            '{class_name}(maker_fee={maker_fee}, '
+            '{class_name}(maker={maker}, '
-            'taker_fee={taker_fee})'.format(
+            'taker={taker})'.format(
                 class_name=self.__class__.__name__,
-                maker_fee=self.maker_fee,
+                maker=self.maker,
-                taker_fee=self.taker_fee,
+                taker=self.taker,
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
 
-        :param order:
+        :param order: Order
-        :param transaction:
+        :param transaction: Transaction
 
         :return float:
             The total commission.
         """
         cost = abs(transaction.amount) * transaction.price
 
-        # Assuming just the taker fee for now
+        asset = order.asset
-        fee = cost * self.taker_fee
+        maker, taker = self.get_maker_taker(asset)
+
+        multiplier = taker
+        if order.limit is not None:
+            multiplier = maker \
+                if ((order.amount > 0 and order.limit < transaction.price)
+                    or (order.amount < 0 and order.limit > transaction.price)) \
+                   and order.limit_reached else taker
+
+        fee = cost * multiplier
         return fee
 
 
@@ -70,7 +84,7 @@ class TradingPairFixedSlippage(SlippageModel):
         spread / 2 will be added to buys and subtracted from sells.
     """
 
-    def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
+    def __init__(self, spread=0.0001):
         super(TradingPairFixedSlippage, self).__init__()
         self.spread = spread
 
@@ -81,7 +95,6 @@ class TradingPairFixedSlippage(SlippageModel):
 
     def simulate(self, data, asset, orders_for_asset):
         self._volume_for_bar = 0
-
         price = data.current(asset, 'close')
 
         dt = data.current_dt
@@ -91,12 +104,14 @@ class TradingPairFixedSlippage(SlippageModel):
 
             order.check_triggers(price, dt)
             if not order.triggered:
-                log.debug('order has not reached the trigger at current '
+                log.info(
-                          'price {}'.format(price))
+                    'order has not reached the trigger at current '
+                    'price {}'.format(price)
+                )
                 continue
 
             execution_price, execution_volume = self.process_order(data, order)
-
+            if execution_price is not None:
                 transaction = create_transaction(
                     order, dt, execution_price, execution_volume
                 )
@@ -121,6 +136,15 @@ class TradingPairFixedSlippage(SlippageModel):
 
 class ExchangeBlotter(Blotter):
     def __init__(self, *args, **kwargs):
+        self.simulate_orders = kwargs.pop('simulate_orders', False)
+        self.attempts = kwargs.pop('attempts', False)
+
+        self.exchanges = kwargs.pop('exchanges', None)
+        if not self.exchanges:
+            raise ValueError(
+                'ExchangeBlotter must have an `exchanges` attribute.'
+            )
+
         super(ExchangeBlotter, self).__init__(*args, **kwargs)
 
         # Using the equity models for now
@@ -132,3 +156,119 @@ class ExchangeBlotter(Blotter):
         self.commission_models = {
             TradingPair: TradingPairFeeSchedule()
         }
+
+    def exchange_order(self, asset, amount, style=None):
+        exchange = self.exchanges[asset.exchange]
+        return exchange.order(
+            asset, amount, style
+        )
+
+    @expect_types(asset=TradingPair)
+    def order(self, asset, amount, style, order_id=None):
+        log.debug('ordering {} {}'.format(amount, asset.symbol))
+        if amount == 0:
+            log.warn('skipping 0 amount orders')
+            return None
+
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).order(
+                asset, amount, style, order_id
+            )
+
+        else:
+            order = retry(
+                action=self.exchange_order,
+                attempts=self.attempts['order_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn('Ordering again.'),
+                args=(asset, amount, style),
+            )
+
+            self.open_orders[order.asset].append(order)
+            self.orders[order.id] = order
+            self.new_orders.append(order)
+
+            return order.id
+
+    def check_open_orders(self):
+        """
+        Loop through the list of open orders in the Portfolio object.
+        For each executed order found, create a transaction and apply to the
+        Portfolio.
+
+        Returns
+        -------
+        list[Transaction]
+
+        """
+        for asset in self.open_orders:
+            exchange = self.exchanges[asset.exchange]
+
+            for order in self.open_orders[asset]:
+                log.debug('found open order: {}'.format(order.id))
+
+                transactions = exchange.process_order(order)
+                # This is a temporary measure, we should really update all
+                # trades, not just when the order gets filled. I just think
+                # that this is safer until we have a robust way to track
+                # the trades already processed by the algo. We can't loose
+                # them if the algo shuts down.
+                if transactions and order.open_amount == 0:
+                    avg_price = np.average(
+                        a=[t.price for t in transactions],
+                        weights=[t.amount for t in transactions],
+                    )
+                    ostatus = 'filled' if order.open_amount == 0 else 'partial'
+                    log.info(
+                        '{} order {} / {}: {}, avg price: {}'.format(
+                            ostatus,
+                            order.id,
+                            asset.symbol,
+                            order.filled,
+                            avg_price,
+                        )
+                    )
+                    for transaction in transactions:
+                        yield order, transaction
+
+                elif order.status == ORDER_STATUS.CANCELLED:
+                    yield order, None
+
+                else:
+                    delta = pd.Timestamp.utcnow() - order.dt
+                    log.info(
+                        'order {order_id} still open after {delta}'.format(
+                            order_id=order.id,
+                            delta=delta
+                        )
+                    )
+
+    def get_exchange_transactions(self):
+        closed_orders = []
+        transactions = []
+        commissions = []
+
+        for order, txn in self.check_open_orders():
+            order.dt = txn.dt
+            transactions.append(txn)
+
+            if not order.open:
+                closed_orders.append(order)
+
+        return transactions, commissions, closed_orders
+
+    def get_transactions(self, bar_data):
+        if self.simulate_orders:
+            return super(ExchangeBlotter, self).get_transactions(bar_data)
+
+        else:
+            return retry(
+                action=self.get_exchange_transactions,
+                attempts=self.attempts['get_transactions_attempts'],
+                sleeptime=self.attempts['retry_sleeptime'],
+                retry_exceptions=(ExchangeRequestError,),
+                cleanup=lambda: log.warn(
+                    'Fetching exchange transactions again.'
+                )
+            )

catalyst/exchange/exchange_bundle.py

@@ -1,6 +1,6 @@
 import os
 import shutil
-from datetime import datetime, timedelta
+from datetime import timedelta
 from functools import partial
 from itertools import chain
 from operator import is_not
@@ -8,30 +8,29 @@ from operator import is_not
 import numpy as np
 import pandas as pd
 import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from pytz import UTC
-from six import itervalues
-
 from catalyst import get_calendar
+from catalyst.assets._assets import TradingPair
 from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
-    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
-from catalyst.exchange.exchange_utils import get_exchange_folder, \
+from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
-    save_exchange_symbols, mixin_market_params
+    get_bcolz_chunk, get_df_from_arrays, get_assets
+from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
+    get_period_label, get_month_start_end, get_year_start_end
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
+    save_exchange_symbols, mixin_market_params, get_catalyst_symbol
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
+from logbook import Logger
+from pytz import UTC
+from six import itervalues
 
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 
@@ -234,10 +233,12 @@ class ExchangeBundle:
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
                     name=asset.symbol,
-                    start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                    start_dt=asset.start_date.strftime(
+                        DATE_TIME_FORMAT),
                     end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                    dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
+                    dates=[date.strftime(
-                )
+                        DATE_TIME_FORMAT) for date in dates])
+
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
 
@@ -245,8 +246,7 @@ class ExchangeBundle:
                     raise EmptyValuesInBundleError(
                         name=asset.symbol,
                         end_minute=end_dt,
-                        dates=dates
+                        dates=dates, )
-                    )
 
                 else:
                     ohlcv_df.dropna(inplace=True)
@@ -291,8 +291,7 @@ class ExchangeBundle:
                 end_dt=end_dt.strftime(DATE_TIME_FORMAT),
                 threshold=threshold,
                 dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                       for date in dates]
+                       for date in dates])
-            )
 
             problems.append(problem)
 
@@ -462,7 +461,7 @@ class ExchangeBundle:
                 (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
-        if end is None or (last_entry is not None and end > last_entry):
+        if last_entry is not None and (end is None or end > last_entry):
             end = last_entry.replace(minute=59, hour=23) \
                 if data_frequency == 'minute' else last_entry
 
@@ -668,7 +667,7 @@ class ExchangeBundle:
 
         if self.exchange is None:
             # Avoid circular dependencies
-            from catalyst.exchange.factory import get_exchange
+            from catalyst.exchange.utils.factory import get_exchange
             self.exchange = get_exchange(self.exchange_name)
 
         problems = []
@@ -681,6 +680,7 @@ class ExchangeBundle:
                 last_traded=np.object_,
                 open=np.float64,
                 high=np.float64,
+                low=np.float64,
                 close=np.float64,
                 volume=np.float64
             ),
@@ -730,7 +730,7 @@ class ExchangeBundle:
                     if data_frequency == 'minute' else asset_def['end_minute']
 
             else:
-                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+                params['symbol'] = get_catalyst_symbol(market)
 
                 params['end_daily'] = end_dt \
                     if data_frequency == 'daily' else 'N/A'
@@ -755,9 +755,10 @@ class ExchangeBundle:
         )
 
         for symbol in assets:
+            # here the symbol is the market['id']
             asset = assets[symbol]
             ohlcv_df = df.loc[
-                (df.index.get_level_values(0) == symbol)
+                (df.index.get_level_values(0) == asset.symbol)
             ]  # type: pd.DataFrame
             ohlcv_df.index = ohlcv_df.index.droplevel(0)
 
@@ -805,7 +806,7 @@ class ExchangeBundle:
         else:
             if self.exchange is None:
                 # Avoid circular dependencies
-                from catalyst.exchange.factory import get_exchange
+                from catalyst.exchange.utils.factory import get_exchange
                 self.exchange = get_exchange(self.exchange_name)
 
             assets = get_assets(
@@ -963,15 +964,15 @@ class ExchangeBundle:
                                   data_frequency,
                                   trailing_bar_count=None,
                                   reset_reader=False):
+        if trailing_bar_count:
+            delta = get_delta(trailing_bar_count, data_frequency)
+            end_dt += delta
+
         start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
         start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
 
-        if trailing_bar_count:
-            delta = get_delta(trailing_bar_count, data_frequency)
-            end_dt += delta
-
         # This is an attempt to resolve some caching with the reader
         # when auto-ingesting data.
         # TODO: needs more work

catalyst/exchange/exchange_data_portal.py

@@ -1,30 +1,29 @@
 import abc
-from time import sleep
 
 import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
-    ExchangeBarDataError,
     PricingDataNotLoadedError)
-from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
+from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
+from catalyst.exchange.utils.datetime_utils import get_frequency
+from logbook import Logger
+from redo import retry
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
-
+        self.attempts = dict(
-        # TODO: put somewhere accessible by each algo
+            get_spot_value_attempts=5,
-        self.retry_get_history_window = 5
+            get_history_window_attempts=5,
-        self.retry_get_spot_value = 5
+            retry_sleeptime=5,
-        self.retry_delay = 5
+        )
 
         super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
 
@@ -35,16 +34,8 @@ class DataPortalExchangeBase(DataPortal):
                             frequency,
                             field,
                             data_frequency,
-                            ffill=True,
+                            ffill=True):
-                            attempt_index=0):
+        exchange_assets = group_assets_by_exchange(assets)
-        try:
-            exchange_assets = dict()
-            for asset in assets:
-                if asset.exchange not in exchange_assets:
-                    exchange_assets[asset.exchange] = list()
-
-                exchange_assets[asset.exchange].append(asset)
-
         if len(exchange_assets) > 1:
             df_list = []
             for exchange_name in exchange_assets:
@@ -77,27 +68,6 @@ class DataPortalExchangeBase(DataPortal):
                 data_frequency,
                 ffill)
 
-        except ExchangeRequestError as e:
-            log.warn(
-                'get history attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_history_window:
-                sleep(self.retry_delay)
-                return self._get_history_window(assets,
-                                                end_dt,
-                                                bar_count,
-                                                frequency,
-                                                field,
-                                                data_frequency,
-                                                ffill,
-                                                attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='history',
-                    attempts=attempt_index,
-                    error=e
-                )
-
     def get_history_window(self,
                            assets,
                            end_dt,
@@ -110,13 +80,19 @@ class DataPortalExchangeBase(DataPortal):
         if field == 'price':
             field = 'close'
 
-        return self._get_history_window(assets,
+        return retry(
+            action=self._get_history_window,
+            attempts=self.attempts['get_history_window_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching history again.'),
+            args=(assets,
                   end_dt,
                   bar_count,
                   frequency,
                   field,
                   data_frequency,
-                                        ffill)
+                  ffill))
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
@@ -130,9 +106,7 @@ class DataPortalExchangeBase(DataPortal):
                                     ffill=True):
         pass
 
-    def _get_spot_value(self, assets, field, dt, data_frequency,
+    def _get_spot_value(self, assets, field, dt, data_frequency):
-                        attempt_index=0):
-        try:
         if isinstance(assets, TradingPair):
             spot_values = self.get_exchange_spot_value(
                 assets.exchange, [assets], field, dt, data_frequency)
@@ -173,26 +147,17 @@ class DataPortalExchangeBase(DataPortal):
 
                 return spot_values
 
-        except ExchangeRequestError as e:
-            log.warn(
-                'get spot value attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_spot_value:
-                sleep(self.retry_delay)
-                return self._get_spot_value(assets, field, dt, data_frequency,
-                                            attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='spot',
-                    attempts=attempt_index,
-                    error=e
-                )
-
     def get_spot_value(self, assets, field, dt, data_frequency):
         if field == 'price':
             field = 'close'
 
-        return self._get_spot_value(assets, field, dt, data_frequency)
+        return retry(
+            action=self._get_spot_value,
+            attempts=self.attempts['get_spot_value_attempts'],
+            sleeptime=self.attempts['retry_sleeptime'],
+            retry_exceptions=(ExchangeRequestError,),
+            cleanup=lambda: log.warn('fetching spot value again.'),
+            args=(assets, field, dt, data_frequency))
 
     @abc.abstractmethod
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -242,6 +207,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
         """
         exchange = self.exchanges[exchange_name]
+
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -249,7 +215,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
             frequency,
             field,
             data_frequency,
-            ffill)
+            False)
         return df
 
     def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -325,6 +291,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         DataFrame
 
         """
+        # TODO: verify that the exchange supports the timeframe
         bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
@@ -343,7 +310,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
-            trailing_bar_count=trailing_bar_count
+            trailing_bar_count=trailing_bar_count,
         )
 
         df = resample_history_df(pd.DataFrame(series), freq, field)

catalyst/exchange/exchange_errors.py

@@ -100,6 +100,19 @@ class InvalidHistoryFrequencyError(ZiplineError):
     ).strip()
 
 
+class UnsupportedHistoryFrequencyError(ZiplineError):
+    msg = (
+        '{exchange} does not support candle frequency {freq}, please choose '
+        'from: {freqs}.'
+    ).strip()
+
+
+class InvalidHistoryTimeframeError(ZiplineError):
+    msg = (
+        'CCXT timeframe {timeframe} not supported by the exchange.'
+    ).strip()
+
+
 class MismatchingFrequencyError(ZiplineError):
     msg = (
         'Bar aggregate frequency {frequency} not compatible with '
@@ -143,7 +156,8 @@ class OrphanOrderError(ZiplineError):
 
 class OrphanOrderReverseError(ZiplineError):
     msg = (
-        'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
+        'Order {order_id} tracked by algorithm, but not found in exchange '
+        '{exchange}.'
     ).strip()
 
 
@@ -161,8 +175,8 @@ class SidHashError(ZiplineError):
 
 class BaseCurrencyNotFoundError(ZiplineError):
     msg = (
-        'Algorithm base currency {base_currency} not found in exchange '
+        'Algorithm base currency {base_currency} not found in account '
-        '{exchange}.'
+        'balances on {exchange}: {balances}'
     ).strip()
 
 
@@ -206,8 +220,9 @@ class EmptyValuesInBundleError(ZiplineError):
 
 class PricingDataBeforeTradingError(ZiplineError):
     msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
-           'starts on {first_trading_day}, but you are either trying to trade or '
+           'starts on {first_trading_day}, but you are either trying to trade '
-           'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
+           'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
+           ).strip()
 
 
 class PricingDataNotLoadedError(ZiplineError):
@@ -217,30 +232,93 @@ class PricingDataNotLoadedError(ZiplineError):
            '{data_frequency} -i {symbol_list}`. See catalyst documentation '
            'for details.').strip()
 
+
 class PricingDataValueError(ZiplineError):
     msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
            '[{start_dt} - {end_dt}]: {error}').strip()
 
 
 class DataCorruptionError(ZiplineError):
-    msg = ('Unable to validate data for {exchange} {symbols} in date range '
+    msg = (
+        'Unable to validate data for {exchange} {symbols} in date range '
         '[{start_dt} - {end_dt}]. The data is either corrupted or '
         'unavailable. Please try deleting this bundle:'
         '\n`catalyst clean-exchange -x {exchange}\n'
         'Then, ingest the data again. Please contact the Catalyst team if '
-           'the issue persists.').strip()
+        'the issue persists.'
+    ).strip()
 
 
 class ApiCandlesError(ZiplineError):
-    msg = ('Unable to fetch candles from the remote API: {error}.').strip()
+    msg = (
+        'Unable to fetch candles from the remote API: {error}.'
+    ).strip()
 
 
 class NoDataAvailableOnExchange(ZiplineError):
     msg = (
-        'Requested data for trading pair {symbol} is not available on exchange {exchange} '
+        'Requested data for trading pair {symbol} is not available on '
+        'exchange {exchange} '
         'in `{data_frequency}` frequency at this time. '
-        'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+        'Check `http://enigma.co/catalyst/status` for market coverage.'
+    ).strip()
 
 
 class NoValueForField(ZiplineError):
-    msg = ('Value not found for field: {field}.').strip()
+    msg = (
+        'Value not found for field: {field}.'
+    ).strip()
+
+
+class OrderTypeNotSupported(ZiplineError):
+    msg = (
+        'Order type `{order_type}` not currency supported by Catalyst. '
+        'Please use `limit` or `market` orders only.'
+    ).strip()
+
+
+class NotEnoughCapitalError(ZiplineError):
+    msg = (
+        'Not enough capital on exchange {exchange} for trading. Each '
+        'exchange should contain at least as much {base_currency} '
+        'as the specified `capital_base`. The current balance {balance} is '
+        'lower than the `capital_base`: {capital_base}'
+    ).strip()
+
+
+class NotEnoughCashError(ZiplineError):
+    msg = (
+        'Total {currency} amount on {exchange} is lower than the cash '
+        'reserved for this algo: {free} < {cash}. While trades can be made on '
+        'the exchange accounts outside of the algo, exchange must have enough '
+        'free {currency} to cover the algo cash.'
+    ).strip()
+
+
+class LastCandleTooEarlyError(ZiplineError):
+    msg = (
+        'The trade date of the last candle {last_traded} is before the '
+        'specified end date minus one candle {end_dt}. Please verify how '
+        '{exchange} calculates the start date of OHLCV candles.'
+    ).strip()
+
+
+class TickerNotFoundError(ZiplineError):
+    msg = (
+        'Unable to fetch ticker for {symbol} on {exchange}.'
+    ).strip()
+
+
+class BalanceNotFoundError(ZiplineError):
+    msg = (
+        '{currency} not found in account balance on {exchange}: {balances}.'
+    ).strip()
+
+
+class BalanceTooLowError(ZiplineError):
+    msg = (
+        'Balance for {currency} on {exchange} too low: {free} < {amount}. '
+        'Positions have likely been sold outside of this algorithm. Please '
+        'add positions to hold a free amount greater than {amount}, or clean '
+        'the state of this algo and restart.'
+    ).strip()

catalyst/exchange/exchange_execution.py

@@ -1,4 +1,4 @@
-from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder, MarketOrder
+from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
 
 
 class ExchangeLimitOrder(LimitOrder):

catalyst/exchange/exchange_portfolio.py

@@ -1,8 +1,7 @@
 import numpy as np
-from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from logbook import Logger
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
@@ -40,7 +39,13 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.debug('creating order {}'.format(order.id))
-        self.open_orders[order.id] = order
+
+        open_orders = self.open_orders[order.asset] \
+            if order.asset is self.open_orders else []
+
+        open_orders.append(order)
+
+        self.open_orders[order.asset] = open_orders
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
@@ -52,6 +57,17 @@ class ExchangePortfolio(Portfolio):
         order_position.amount += order.amount
         log.debug('open order added to portfolio')
 
+    def _remove_open_order(self, order):
+        try:
+            open_orders = self.open_orders[order.asset]
+            if order in open_orders:
+                open_orders.remove(order)
+
+        except Exception:
+            raise ValueError(
+                'unable to clear order not found in open order list.'
+            )
+
     def execute_order(self, order, transaction):
         """
         Update the open orders and positions to apply an executed order.
@@ -66,14 +82,15 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.debug('executing order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None
 
         if order_position is None:
             raise ValueError(
-                'Trying to execute order for a position not held: %s' % order.id
+                'Trying to execute order for a position not held:'
+                ' {}'.format(order.id)
             )
 
         self.capital_used += order.amount * transaction.price
@@ -99,7 +116,7 @@ class ExchangePortfolio(Portfolio):
 
         """
         log.info('removing cancelled order {}'.format(order.id))
-        del self.open_orders[order.id]
+        self._remove_open_order(order)
 
         order_position = self.positions[order.asset] \
             if order.asset in self.positions else None

catalyst/exchange/exchange_pricing_loader.py

@@ -0,0 +1,177 @@
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from catalyst.constants import LOG_LEVEL
+from catalyst.data.us_equity_pricing import BcolzDailyBarReader
+from catalyst.errors import NoFurtherDataError
+from catalyst.exchange.utils.factory import get_exchange
+from catalyst.lib.adjusted_array import AdjustedArray
+from catalyst.pipeline.data import DataSet, Column
+from catalyst.pipeline.loaders.base import PipelineLoader
+from catalyst.utils.calendars import get_calendar
+from catalyst.utils.numpy_utils import float64_dtype
+from logbook import Logger
+from numpy import (
+    iinfo,
+    uint32,
+)
+
+UINT32_MAX = iinfo(uint32).max
+
+log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
+
+
+class TradingPairPricing(DataSet):
+    """
+    Dataset representing daily trading prices and volumes.
+    """
+    open = Column(float64_dtype)
+    high = Column(float64_dtype)
+    low = Column(float64_dtype)
+    close = Column(float64_dtype)
+    volume = Column(float64_dtype)
+
+
+class ExchangePricingLoader(PipelineLoader):
+    """
+    PipelineLoader for Crypto Pricing data
+
+    Delegates loading of baselines and adjustments.
+    """
+
+    def __init__(self, data_frequency):
+
+        cal = get_calendar('OPEN')
+
+        if data_frequency == 'daily':
+            reader = None
+            all_sessions = cal.all_sessions
+
+        elif data_frequency == 'minute':
+            reader = None
+            all_sessions = cal.all_minutes
+
+        else:
+            raise ValueError(
+                'Invalid data frequency: {}'.format(data_frequency)
+            )
+
+        self.data_frequency = data_frequency
+        self.raw_price_loader = reader
+        self._columns = TradingPairPricing.columns
+        self._all_sessions = all_sessions
+
+    @classmethod
+    def from_files(cls, pricing_path):
+        """
+        Create a loader from a bcolz equity pricing dir and a SQLite
+        adjustments path.
+
+        Parameters
+        ----------
+        pricing_path : str
+            Path to a bcolz directory written by a BcolzDailyBarWriter.
+        """
+        return cls(
+            BcolzDailyBarReader(pricing_path),
+        )
+
+    def load_adjusted_array(self, columns, dates, assets, mask):
+        # load_adjusted_array is called with dates on which the user's algo
+        # will be shown data, which means we need to return the data that would
+        # be known at the start of each date.  We assume that the latest data
+        # known on day N is the data from day (N - 1), so we shift all query
+        # dates back by a day.
+        start_date, end_date = _shift_dates(
+            self._all_sessions, dates[0], dates[-1], shift=1,
+        )
+        colnames = [c.name for c in columns]
+
+        if len(assets) == 0:
+            raise ValueError(
+                'Pipeline cannot load data with eligible assets.'
+            )
+
+        exchange_names = []
+        for asset in assets:
+            if asset.exchange not in exchange_names:
+                exchange_names.append(asset.exchange)
+
+        exchange = get_exchange(exchange_names[0])
+        reader = exchange.bundle.get_reader(self.data_frequency)
+
+        raw_arrays = reader.load_raw_arrays(
+            colnames,
+            start_date,
+            end_date,
+            assets,
+        )
+
+        out = {}
+        for c, c_raw in zip(columns, raw_arrays):
+            out[c] = AdjustedArray(
+                c_raw.astype(c.dtype),
+                mask,
+                {},
+                c.missing_value,
+            )
+        return out
+
+    @property
+    def columns(self):
+        return self._columns
+
+
+def _shift_dates(dates, start_date, end_date, shift):
+    try:
+        start = dates.get_loc(start_date)
+    except KeyError:
+        if start_date < dates[0]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requested data starting on {query_start}, "
+                    "but first known date is {calendar_start}"
+                ).format(
+                    query_start=str(start_date),
+                    calendar_start=str(dates[0]),
+                )
+            )
+        else:
+            raise ValueError("Query start %s not in calendar" % start_date)
+
+    # Make sure that shifting doesn't push us out of the calendar.
+    if start < shift:
+        raise NoFurtherDataError(
+            msg=(
+                "Pipeline Query requested data from {shift}"
+                " days before {query_start}, but first known date is only "
+                "{start} days earlier."
+            ).format(shift=shift, query_start=start_date, start=start),
+        )
+
+    try:
+        end = dates.get_loc(end_date)
+    except KeyError:
+        if end_date > dates[-1]:
+            raise NoFurtherDataError(
+                msg=(
+                    "Pipeline Query requesting data up to {query_end}, "
+                    "but last known date is {calendar_end}"
+                ).format(
+                    query_end=end_date,
+                    calendar_end=dates[-1],
+                )
+            )
+        else:
+            raise ValueError("Query end %s not in calendar" % end_date)
+    return dates[start - shift], dates[end - shift]

catalyst/exchange/factory.py

@@ -1,36 +0,0 @@
-import os
-
-from catalyst.exchange.ccxt.ccxt_exchange import CCXT
-from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_exchange_folder
-
-
-def get_exchange(exchange_name, base_currency=None, portfolio=None,
-                 must_authenticate=False):
-    exchange_auth = get_exchange_auth(exchange_name)
-
-    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
-    if must_authenticate and not has_auth:
-        raise ExchangeAuthEmpty(
-            exchange=exchange_name.title(),
-            filename=os.path.join(
-                get_exchange_folder(exchange_name), 'auth.json'
-            )
-        )
-
-    return CCXT(
-        exchange_name=exchange_name,
-        key=exchange_auth['key'],
-        secret=exchange_auth['secret'],
-        base_currency=base_currency,
-        portfolio=portfolio
-    )
-
-
-def get_exchanges(exchange_names):
-    exchanges = dict()
-    for exchange_name in exchange_names:
-        exchanges[exchange_name] = get_exchange(exchange_name)
-
-    return exchanges

catalyst/exchange/live_graph_clock.py

@@ -1,14 +1,12 @@
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.utils.stats_utils import prepare_stats
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
@@ -38,177 +36,23 @@ class LiveGraphClock(object):
     the exchange and the live trading machine's clock. It's not used currently.
     """
 
-    def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
+    def __init__(self, sessions, context, callback=None,
-
+                 time_skew=pd.Timedelta('0s')):
-        global mdates, plt  # TODO: Could be cleaner
-        import matplotlib.dates as mdates
-        from matplotlib import pyplot as plt
-        from matplotlib import style
 
         self.sessions = sessions
         self.time_skew = time_skew
         self._last_emit = None
         self._before_trading_start_bar_yielded = True
         self.context = context
-        self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+        self.callback = callback
-
-        style.use('dark_background')
-
-        fig = plt.figure()
-        fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
-            self.context.algo_namespace))
-
-        self.ax_pnl = fig.add_subplot(311)
-
-        self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
-
-        self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
-
-        if len(context.minute_stats) > 0:
-            self.draw_pnl()
-            self.draw_custom_signals()
-            self.draw_exposure()
-
-        # rotates and right aligns the x labels, and moves the bottom of the
-        # axes up to make room for them
-        fig.autofmt_xdate()
-        fig.subplots_adjust(hspace=0.5)
-
-        plt.tight_layout()
-        plt.ion()
-        plt.show()
-
-    def format_ax(self, ax):
-        """
-        Trying to assign reasonable parameters to the time axis.
-
-        Parameters
-        ----------
-        ax:
-
-        """
-        # TODO: room for improvement
-        ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-        ax.xaxis.set_major_formatter(self.fmt)
-
-        locator = mdates.HourLocator(interval=4)
-        locator.MAXTICKS = 5000
-        ax.xaxis.set_minor_locator(locator)
-
-        datemin = pd.Timestamp.utcnow()
-        ax.set_xlim(datemin)
-
-        ax.grid(True)
-
-    def set_legend(self, ax):
-        """
-        Set legend on the chart.
-
-        Parameters
-        ----------
-        ax
-
-        """
-        ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-    def draw_pnl(self):
-        """
-        Draw p&l line on the chart.
-
-        """
-        ax = self.ax_pnl
-        df = self.context.pnl_stats
-
-        ax.clear()
-        ax.set_title('Performance')
-        ax.plot(df.index, df['performance'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Performance'
-                )
-
-        def perc(val):
-            return '{:2f}'.format(val)
-
-        ax.format_ydata = perc
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_custom_signals(self):
-        """
-        Draw custom signals on the chart.
-
-        """
-        ax = self.ax_custom_signals
-        df = self.context.custom_signals_stats
-
-        colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-        ax.clear()
-        ax.set_title('Custom Signals')
-        for index, column in enumerate(df.columns.values.tolist()):
-            ax.plot(df.index, df[column], '-',
-                    color=colors[index],
-                    linewidth=1.0,
-                    label=column
-                    )
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_exposure(self):
-        """
-        Draw exposure line on the chart.
-
-        """
-        ax = self.ax_exposure
-        context = self.context
-        df = context.exposure_stats
-
-        # TODO: list exchanges in graph
-        base_currency = None
-        positions = []
-        for exchange_name in context.exchanges:
-            exchange = context.exchanges[exchange_name]
-
-            if not base_currency:
-                base_currency = exchange.base_currency
-            elif base_currency != exchange.base_currency:
-                raise MismatchingBaseCurrenciesExchanges(
-                    base_currency=base_currency,
-                    exchange_name=exchange.name,
-                    exchange_currency=exchange.base_currency
-                )
-
-            positions += exchange.portfolio.positions
-
-        ax.clear()
-        ax.set_title('Exposure')
-        ax.plot(df.index, df['base_currency'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Base Currency: {}'.format(base_currency.upper())
-                )
-
-        symbols = []
-        for position in positions:
-            symbols.append(position.symbol)
-
-        ax.plot(df.index, df['long_exposure'], '-',
-                color='blue',
-                linewidth=1.0,
-                label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-        self.set_legend(ax)
-        self.format_ax(ax)
 
     def __iter__(self):
+        from matplotlib import pyplot as plt
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:
             current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
+            current_minute = current_time.floor('1T')
 
             if self._last_emit is None or current_minute > self._last_emit:
                 log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -216,14 +60,11 @@ class LiveGraphClock(object):
                 self._last_emit = current_minute
                 yield current_minute, BAR
 
-                try:
+                recorded_cols = list(self.context.recorded_vars.keys())
-                    self.draw_pnl()
+                df, _ = prepare_stats(
-                    self.draw_custom_signals()
+                    self.context.frame_stats, recorded_cols=recorded_cols
-                    self.draw_exposure()
+                )
-
+                self.callback(self.context, df)
-                    plt.draw()
-                except Exception as e:
-                    log.warn('Unable to update the graph: {}'.format(e))
 
             else:
                 # I can't use the "animate" reactive approach here because

catalyst/exchange/poloniex/poloniex.py

@@ -1,655 +0,0 @@
-import json
-import json
-import time
-from collections import defaultdict
-
-import numpy as np
-import pandas as pd
-import pytz
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-# import six
-from six import iteritems
-
-from catalyst.constants import LOG_LEVEL
-# from websocket import create_connection
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrphanOrderReverseError)
-from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols, get_symbols_string
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.protocol import Account
-
-log = Logger('Poloniex', level=LOG_LEVEL)
-
-
-class Poloniex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret)
-        self.name = 'poloniex'
-
-        self.assets = dict()
-        self.load_assets()
-
-        self.local_assets = dict()
-        self.load_assets(is_local=True)
-
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-        self.transactions = defaultdict(list)
-
-        self.num_candles_limit = 2000
-        self.max_requests_per_minute = 60
-        self.request_cpt = dict()
-
-        self.bundle = ExchangeBundle(self.name)
-
-    def sanitize_curency_symbol(self, exchange_symbol):
-        """
-        Helper method used to build the universal pair.
-        Include any symbol mapping here if appropriate.
-
-        :param exchange_symbol:
-        :return universal_symbol:
-        """
-        return exchange_symbol.lower()
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from the Exchange order dictionary
-        :param order_status:
-        :return: Order
-        """
-        # if order_status['is_cancelled']:
-        #    status = ORDER_STATUS.CANCELLED
-        # elif not order_status['is_live']:
-        #    log.info('found executed order {}'.format(order_status))
-        #    status = ORDER_STATUS.FILLED
-        # else:
-        status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['amount'])
-        # filled = float(order_status['executed_amount'])
-        filled = None
-
-        if order_status['type'] == 'sell':
-            amount = -amount
-            # filled = -filled
-
-        price = float(order_status['rate'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        # if order_type.endswith('limit'):
-        #    limit_price = price
-        # elif order_type.endswith('stop'):
-        #    stop_price = price
-
-        # executed_price = float(order_status['avg_execution_price'])
-        executed_price = price
-
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
-        commission = None
-
-        # date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        # date = pytz.utc.localize(date)
-        date = None
-
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            # No such field in Poloniex
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=str(order_status['orderNumber']),
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def get_balances(self):
-        balances = self.api.returnbalances()
-        try:
-            log.debug('retrieving wallets balances')
-        except Exception as e:
-            log.debug(e)
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['error'])
-            )
-
-        std_balances = dict()
-        for (key, value) in iteritems(balances):
-            currency = key.lower()
-            std_balances[currency] = float(value)
-
-        return std_balances
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def get_account(self):
-        # TODO: fetch account data and keep in cache
-        return None
-
-    def get_candles(self, freq, assets, bar_count=None,
-                    start_dt=None, end_dt=None):
-        """
-        Retrieve OHLVC candles from Poloniex
-
-        :param freq:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '5m', '15m', '30m', '2h', '4h', '1D'
-        """
-
-        if end_dt is None:
-            end_dt = pd.Timestamp.utcnow()
-
-        log.debug(
-            'retrieving {bars} {freq} candles on {exchange} from '
-            '{end_dt} for markets {symbols}, '.format(
-                bars=bar_count,
-                freq=freq,
-                exchange=self.name,
-                end_dt=end_dt,
-                symbols=get_symbols_string(assets)
-            )
-        )
-
-        if freq == '1T' and (bar_count == 1 or bar_count is None):
-            # TODO: use the order book instead
-            # We use the 5m to fetch the last bar
-            frequency = 300
-        elif freq == '5T':
-            frequency = 300
-        elif freq == '15T':
-            frequency = 900
-        elif freq == '30T':
-            frequency = 1800
-        elif freq == '120T':
-            frequency = 7200
-        elif freq == '240T':
-            frequency = 14400
-        elif freq == '1D':
-            frequency = 86400
-        else:
-            # Poloniex does not offer 1m data candles
-            # It is likely to error out there frequently
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
-
-        for asset in asset_list:
-            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
-            end = int(delta.total_seconds())
-
-            if bar_count is None:
-                start = end - 2 * frequency
-            else:
-                start = end - bar_count * frequency
-
-            try:
-                response = self.api.returnchartdata(
-                    self.get_symbol(asset), frequency, start, end
-                )
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            def ohlc_from_candle(candle):
-                last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
-                last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-                ohlc = dict(
-                    open=np.float64(candle['open']),
-                    high=np.float64(candle['high']),
-                    low=np.float64(candle['low']),
-                    close=np.float64(candle['close']),
-                    volume=np.float64(candle['volume']),
-                    price=np.float64(candle['close']),
-                    last_traded=last_traded
-                )
-
-                return ohlc
-
-            if bar_count is None:
-                ohlc_map[asset] = ohlc_from_candle(response[0])
-            else:
-                ohlc_bars = []
-                for candle in response:
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-                ohlc_map[asset] = ohlc_bars
-
-        return ohlc_map[assets] \
-            if isinstance(assets, TradingPair) else ohlc_map
-
-    def create_order(self, asset, amount, is_buy, style):
-        """
-        Creating order on the exchange.
-
-        :param asset:
-        :param amount:
-        :param is_buy:
-        :param style:
-        :return:
-        """
-        exchange_symbol = self.get_symbol(asset)
-
-        if isinstance(style, ExchangeLimitOrder) or isinstance(style,
-                                                               ExchangeStopLimitOrder):
-            if isinstance(style, ExchangeStopLimitOrder):
-                log.warn('{} will ignore the stop price'.format(self.name))
-
-            price = style.get_limit_price(is_buy)
-
-            try:
-                if (is_buy):
-                    response = self.api.buy(exchange_symbol, amount, price)
-                else:
-                    response = self.api.sell(exchange_symbol, -amount, price)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            date = pd.Timestamp.utcnow()
-
-            if ('orderNumber' in response):
-                order_id = str(response['orderNumber'])
-                order = Order(
-                    dt=date,
-                    asset=asset,
-                    amount=amount,
-                    stop=style.get_stop_price(is_buy),
-                    limit=style.get_limit_price(is_buy),
-                    id=order_id
-                )
-                return order
-            else:
-                log.warn(
-                    '{} order failed: {}'.format('buy' if is_buy else 'sell',
-                                                 response['error']))
-                return None
-        else:
-            raise InvalidOrderStyle(exchange=self.name,
-                                    style=style.__class__.__name__)
-
-    def get_open_orders(self, asset='all'):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not 'all', return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If 'all' is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-
-        return self.portfolio.open_orders
-
-        """
-            TODO: Why going to the exchange if we already have this info locally?
-                  And why creating all these Orders if we later discard them?
-        """
-
-        try:
-            if (asset == 'all'):
-                response = self.api.returnopenorders('all')
-            else:
-                response = self.api.returnopenorders(self.get_symbol(asset))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        print(self.portfolio.open_orders)
-
-        # TODO: Need to handle openOrders for 'all'
-        orders = list()
-        for order_status in response:
-            order, executed_price = self._create_order(
-                order_status)  # will Throw error b/c Polo doesn't track order['symbol']
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-
-        try:
-            order = self._portfolio.open_orders[order_id]
-        except Exception as e:
-            raise OrphanOrderError(order_id=order_id, exchange=self.name)
-
-        return order
-
-        # TODO: Need to decide whether we fetch orders locally or from exchnage
-        # The code below is ignored
-
-        try:
-            response = self.api.returnopenorders(self.get_symbol(order.sid))
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        for o in response:
-            if (int(o['orderNumber']) == int(order_id)):
-                return order
-
-        return None
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-
-        if (isinstance(order_param, Order)):
-            order = order_param
-        else:
-            order = self._portfolio.open_orders[order_param]
-
-        try:
-            response = self.api.cancelorder(order.id)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            log.info(
-                'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
-                    order_id=order.id,
-                    exchange=self.name,
-                    error=response['error']
-                ))
-
-            # raise OrderCancelError(
-            #    order_id=order.id,
-            #    exchange=self.name,
-            #    error=response['error']
-            # )
-
-        self.portfolio.remove_order(order)
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self.get_symbols(assets)
-
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            response = self.api.returnticker()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response['error'])
-            )
-
-        ticks = dict()
-
-        for index, symbol in enumerate(symbols):
-            ticks[assets[index]] = dict(
-                timestamp=pd.Timestamp.utcnow(),
-                bid=float(response[symbol]['highestBid']),
-                ask=float(response[symbol]['lowestAsk']),
-                last_price=float(response[symbol]['last']),
-                low=float(response[symbol]['lowestAsk']),
-                # TODO: Polo does not provide low
-                high=float(response[symbol]['highestBid']),
-                # TODO: Polo does not provide high
-                volume=float(response[symbol]['baseVolume']),
-            )
-
-        log.debug('got tickers {}'.format(ticks))
-        return ticks
-
-    def generate_symbols_json(self, filename=None, source_dates=False):
-        symbol_map = {}
-
-        if not source_dates:
-            fn, r = download_exchange_symbols(self.name)
-            with open(fn) as data_file:
-                cached_symbols = json.load(data_file)
-
-        response = self.api.returnticker()
-
-        for exchange_symbol in response:
-            base, market = self.sanitize_curency_symbol(exchange_symbol).split(
-                '_')
-            symbol = '{market}_{base}'.format(market=market, base=base)
-
-            if (source_dates):
-                start_date = self.get_symbol_start_date(exchange_symbol)
-            else:
-                try:
-                    start_date = cached_symbols[exchange_symbol]['start_date']
-                except KeyError as e:
-                    start_date = time.strftime('%Y-%m-%d')
-
-            try:
-                end_daily = cached_symbols[exchange_symbol]['end_daily']
-            except KeyError as e:
-                end_daily = 'N/A'
-
-            try:
-                end_minute = cached_symbols[exchange_symbol]['end_minute']
-            except KeyError as e:
-                end_minute = 'N/A'
-
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=start_date,
-                end_daily=end_daily,
-                end_minute=end_minute,
-            )
-
-        if (filename is None):
-            filename = get_exchange_symbols_filename(self.name)
-
-        with open(filename, 'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2,
-                      separators=(',', ':'))
-
-    def get_symbol_start_date(self, symbol):
-        try:
-            r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
-                '2010-1-1').value // 10 ** 9)
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
-
-    def check_open_orders(self):
-        """
-        Need to override this function for Poloniex:
-
-        Loop through the list of open orders in the Portfolio object.
-        Check if any transactions have been executed:
-            If so, create a transaction and apply to the Portfolio.
-        Check if the order is still open:
-            If not, remove it from open orders
-
-        :return:
-        transactions: Transaction[]
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-
-                order = self._portfolio.open_orders[order_id]
-                log.debug('found open order: {}'.format(order_id))
-
-                try:
-                    order_open = self.get_order(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if (order_open):
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta)
-                    )
-
-                try:
-                    response = self.api.returnordertrades(order_id)
-                except Exception as e:
-                    raise ExchangeRequestError(error=e)
-
-                if ('error' in response):
-                    if (not order_open):
-                        raise OrphanOrderReverseError(order_id=order_id,
-                                                      exchange=self.name)
-                else:
-                    for tx in response:
-                        """
-                            We maintain a list of dictionaries of transactions that correspond to
-                            partially filled orders, indexed by order_id. Every time we query 
-                            executed transactions from the exchange, we check if we had that 
-                            transaction for that order already. If not, we process it.
-
-                            When an order if fully filled, we flush the dict of transactions  
-                            associated with that order.
-                        """
-                        if (not filter(
-                                lambda item: item['order_id'] == tx['tradeID'],
-                                self.transactions[order_id])):
-                            log.debug(
-                                'Got new transaction for order {}: amount {}, price {}'.format(
-                                    order_id, tx['amount'], tx['rate']))
-                            tx['amount'] = float(tx['amount'])
-                            if (tx['type'] == 'sell'):
-                                tx['amount'] = -tx['amount']
-                            transaction = Transaction(
-                                asset=order.asset,
-                                amount=tx['amount'],
-                                dt=pd.to_datetime(tx['date'], utc=True),
-                                price=float(tx['rate']),
-                                order_id=tx['tradeID'],
-                                # it's a misnomer, but keeping it for compatibility
-                                commission=float(tx['fee'])
-                            )
-                            self.transactions[order_id].append(transaction)
-                            self.portfolio.execute_transaction(transaction)
-                            transactions.append(transaction)
-
-                    if (not order_open):
-                        """
-                            Since transactions have been executed individually
-                            the only thing left to do is remove them from list of open_orders
-                        """
-                        del self.portfolio.open_orders[order_id]
-                        del self.transactions[order_id]
-
-        return transactions
-
-    def get_orderbook(self, asset, order_type='all'):
-        exchange_symbol = asset.exchange_symbol
-        data = self.api.returnOrderBook(market=exchange_symbol)
-
-        result = dict()
-        for order_type in data:
-            # TODO: filter by type
-            if order_type != 'asks' and order_type != 'bids':
-                continue
-
-            result[order_type] = []
-            for entry in data[order_type]:
-                if len(entry) == 2:
-                    result[order_type].append(
-                        dict(
-                            rate=float(entry[0]),
-                            quantity=float(entry[1])
-                        )
-                    )
-        return result

catalyst/exchange/poloniex/poloniex_api.py

@@ -1,215 +0,0 @@
-#!/usr/bin/env python
-import json
-import time
-import hmac
-import hashlib
-import ssl
-
-from six.moves import urllib
-
-# Workaround for backwards compatibility
-# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
-urlopen = urllib.request.urlopen
-
-
-class Poloniex_api(object):
-    def __init__(self, key, secret):
-        self.key = key
-        self.secret = secret
-
-        self.max_requests_per_second = 6
-        self.request_cpt = dict()
-
-        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                       'returnTradeHistory', 'returnChartData',
-                       'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances', 'returnCompleteBalances',
-                        'returnDepositAddresses',
-                        'generateNewAddress', 'returnDepositsWithdrawals',
-                        'returnOpenOrders',
-                        'returnTradeHistory', 'returnOrderTrades',
-                        'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo',
-                        'returnAvailableAccountBalances',
-                        'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary', 'marginBuy',
-                        'marginSell',
-                        'getMarginPosition', 'closeMarginPosition',
-                        'createLoanOffer',
-                        'cancelLoanOffer', 'returnOpenLoanOffers',
-                        'returnActiveLoans',
-                        'returnLendingHistory', 'toggleAutoRenew']
-
-    def ask_request(self):
-        """
-        Asks permission to issue a request to the exchange.
-        The primary purpose is to avoid hitting rate limits.
-
-        The application will pause if the maximum requests per minute
-        permitted by the exchange is exceeded.
-
-        :return boolean:
-
-        """
-        now = time.time()
-        if not self.request_cpt:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        cpt_date = list(self.request_cpt.keys())[0]
-        cpt = self.request_cpt[cpt_date]
-
-        if now > cpt_date + 1:
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-
-        if cpt >= self.max_requests_per_second:
-
-            time.sleep(1)
-
-            now = time.time()
-            self.request_cpt = dict()
-            self.request_cpt[now] = 0
-            return True
-        else:
-            self.request_cpt[cpt_date] += 1
-
-    def query(self, method, req={}):
-
-        if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + \
-                  urllib.parse.urlencode(req)
-            headers = {}
-            post_data = None
-        elif method in self.trading:
-            url = 'https://poloniex.com/tradingApi'
-            req['command'] = method
-            req['nonce'] = int(time.time() * 1000)
-            post_data = urllib.parse.urlencode(req)
-
-            signature = hmac.new(self.secret.encode('utf-8'),
-                                 post_data.encode('utf-8'),
-                                 hashlib.sha512).hexdigest()
-            headers = {'Sign': signature, 'Key': self.key}
-
-            post_data = post_data.encode('utf-8')
-        else:
-            raise ValueError(
-                'Method "' + method + '" not found in neither the Public API '
-                                      'or Trading API endpoints'
-            )
-
-        self.ask_request()
-        req = urllib.request.Request(
-            url,
-            data=post_data,
-            headers=headers,
-        )
-        return json.loads(
-            urlopen(req, context=ssl._create_unverified_context()).read())
-
-    def returnticker(self):
-        return self.query('returnTicker', {})
-
-    def return24volume(self):
-        return self.query('return24Volume', {})
-
-    def returnOrderBook(self, market='all'):
-        return self.query('returnOrderBook', {'currencyPair': market})
-
-    def returntradehistory(self, market, start=None, end=None):
-        if (start is not None and end is not None):
-            return self.query('returntradehistory',
-                              {'currencyPair': market, 'start': start,
-                               'end': end})
-        else:
-            return self.query('returntradehistory', {'currencyPair': market})
-
-    def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData',
-                          {'currencyPair': market, 'period': period,
-                           'start': start, 'end': end})
-
-    def returncurrencies(self):
-        return self.query('returnCurrencies', {})
-
-    def returnloadorders(self, market):
-        return self.query('returnLoanOrders', {'currency': market})
-
-    def returnbalances(self):
-        return self.query('returnBalances')
-
-    def returncompletebalances(self, account):
-        if (account):
-            return self.query('returnCompleteBalances', {'account': account})
-        else:
-            return self.query('returnCompleteBalances')
-
-    def returndepositaddresses(self):
-        return self.query('returnDepositAddresses')
-
-    def generatenewaddress(self, currency):
-        return self.query('generateNewAddress', {'currency': currency})
-
-    def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals',
-                          {'start': start, 'end': end})
-
-    def returnopenorders(self, market):
-        return self.query('returnOpenOrders', {'currencyPair': market})
-
-    def returntradehistory(self, market):
-        # TODO: optional start and/or end and limit
-        return self.query('returnTradeHistory', {'currencyPair': market})
-
-    def returnordertrades(self, ordernumber):
-        return self.query('returnOrderTrades', {'orderNumber': ordernumber})
-
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-            postonly=0):
-        if (fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'immediateOrCancel': immediateorcancel, })
-        elif (postonly):
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount,
-                                      'postOnly': postonly, })
-        else:
-            return self.query('buy', {'currencyPair': market, 'rate': rate,
-                                      'amount': amount, })
-
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-             postonly=0):
-        if (fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'fillOrKill': fillorkill, })
-        elif (immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'immediateOrCancel': immediateorcancel, })
-        elif (postonly):
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount,
-                                       'postOnly': postonly, })
-        else:
-            return self.query('sell', {'currencyPair': market, 'rate': rate,
-                                       'amount': amount, })
-
-    def cancelorder(self, ordernumber):
-        return self.query('cancelOrder', {'orderNumber': ordernumber})
-
-    def withdraw(self, currency, quantity, address):
-        return self.query('withdraw',
-                          {'currency': currency, 'amount': quantity,
-                           'address': address})
-
-    def returnfeeinfo(self):
-        return self.query('returnFeeInfo')

catalyst/exchange/simple_clock.py

@@ -14,14 +14,13 @@
 from time import sleep
 
 import pandas as pd
+from catalyst.constants import LOG_LEVEL
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START
 )
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
@@ -31,7 +30,8 @@ class SimpleClock(object):
     This class is a drop-in replacement for
     :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
 
-    This is a stripped down version because crypto exchanges run around the clock.
+    This is a stripped down version because crypto exchanges run
+    around the clock.
 
     The :param:`time_skew` parameter represents the time difference between
     the Broker and the live trading machine's clock.

catalyst/exchange/stats_utils.py

@@ -1,221 +0,0 @@
-import numbers
-
-import numpy as np
-import pandas as pd
-
-
-def trend_direction(series):
-    if series[-1] is np.nan or series[-1] is np.nan:
-        return None
-
-    if series[-1] > series[-2]:
-        return 'up'
-    else:
-        return 'down'
-
-
-def crossover(source, target):
-    """
-    The `x`-series is defined as having crossed over `y`-series if the value
-    of `x` is greater than the value of `y` and the value of `x` was less than
-    the value of `y` on the bar immediately preceding the current bar.
-
-    Parameters
-    ----------
-    source: Series
-    target: Series
-
-    Returns
-    -------
-    bool
-
-    """
-    if isinstance(target, numbers.Number):
-        if source[-1] is np.nan or source[-2] is np.nan \
-                or target is np.nan:
-            return False
-
-        if source[-1] >= target > source[-2]:
-            return True
-        else:
-            return False
-
-    else:
-        if source[-1] is np.nan or source[-2] is np.nan \
-                or target[-1] is np.nan or target[-2] is np.nan:
-            return False
-
-        if source[-1] > target[-1] and source[-2] < target[-2]:
-            return True
-        else:
-            return False
-
-
-def crossunder(source, target):
-    """
-    The `x`-series is defined as having crossed under `y`-series if the value
-    of `x` is less than the value of `y` and the value of `x` was greater than
-    the value of `y` on the bar immediately preceding the current bar.
-
-    Parameters
-    ----------
-    source: Series
-    target: Series
-
-    Returns
-    -------
-    bool
-
-    """
-    if isinstance(target, numbers.Number):
-        if source[-1] is np.nan or source[-2] is np.nan \
-                or target is np.nan:
-            return False
-
-        if source[-1] < target <= source[-2]:
-            return True
-        else:
-            return False
-    else:
-        if source[-1] is np.nan or source[-2] is np.nan \
-                or target[-1] is np.nan or target[-2] is np.nan:
-            return False
-
-        if source[-1] < target[-1] and source[-2] >= target[-2]:
-            return True
-        else:
-            return False
-
-
-def vwap(df):
-    """
-    Volume-weighted average price (VWAP) is a ratio generally used by
-    institutional investors and mutual funds to make buys and sells so as not
-    to disturb the market prices with large orders. It is the average share
-    price of a stock weighted against its trading volume within a particular
-    time frame, generally one day.
-
-    Read more: Volume Weighted Average Price - VWAP
-    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-
-    Returns
-    -------
-
-    """
-    if 'close' not in df.columns or 'volume' not in df.columns:
-        raise ValueError('price data must include `volume` and `close`')
-
-    vol_sum = np.nansum(df['volume'].values)
-
-    try:
-        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
-    except ZeroDivisionError:
-        ret = np.nan
-
-    return ret
-
-
-def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
-    """
-    Format and print the last few rows of a statistics DataFrame.
-    See the pyfolio project for the data structure.
-
-    Parameters
-    ----------
-    stats_df: DataFrame
-    num_rows: int
-
-    Returns
-    -------
-    str
-
-    """
-    stats_df.set_index('period_close', drop=True, inplace=True)
-    stats_df.dropna(axis=1, how='all', inplace=True)
-
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 3)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-    columns = ['starting_cash', 'ending_cash', 'portfolio_value',
-               'pnl', 'long_exposure', 'short_exposure', 'orders',
-               'transactions', 'positions']
-
-    if recorded_cols is not None:
-        for column in recorded_cols:
-            columns.append(column)
-
-    def format_positions(positions):
-        parts = []
-        for position in positions:
-            msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format(
-                amount=position['amount'],
-                base=position['sid'].base_currency,
-                cost_basis=position['cost_basis'],
-                quote=position['sid'].quote_currency
-            )
-            parts.append(msg)
-        return ', '.join(parts)
-
-    formatters = {
-        'orders': lambda orders: len(orders),
-        'transactions': lambda transactions: len(transactions),
-        'returns': lambda returns: "{0:.4f}".format(returns),
-        'positions': format_positions
-    }
-
-    return stats_df.tail(num_rows).to_string(
-        columns=columns,
-        formatters=formatters
-    )
-
-
-def df_to_string(df):
-    """
-    Create a formatted str representation of the DataFrame.
-
-    Parameters
-    ----------
-    df: DataFrame
-
-    Returns
-    -------
-    str
-
-    """
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
-    return df.to_string()
-
-
-def extract_transactions(perf):
-    """
-    Compute indexes for buy and sell transactions
-
-    Parameters
-    ----------
-    perf: DataFrame
-        The algo performance DataFrame.
-
-    Returns
-    -------
-    DataFrame
-        A DataFrame of transactions.
-
-    """
-    trans_list = perf.transactions.values
-    all_trans = [t for sublist in trans_list for t in sublist]
-    all_trans.sort(key=lambda t: t['dt'])
-
-    transactions = pd.DataFrame(all_trans)
-    if not transactions.empty:
-        transactions.set_index('dt', inplace=True, drop=True)
-    return transactions

catalyst/exchange/utils/bundle_utils.py

@@ -0,0 +1,158 @@
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+import os
+import tarfile
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
+
+EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
+API_URL = 'http://data.enigma.co/api/v1'
+
+
+def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
+    """
+    Download and extract a bcolz bundle.
+
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
+
+    Returns
+    -------
+    str
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+
+    """
+    root = get_exchange_bundles_folder(exchange_name)
+    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
+        exchange=exchange_name,
+        frequency=data_frequency,
+        symbol=symbol,
+        period=period
+    )
+    path = os.path.join(root, name)
+
+    if not os.path.isdir(path):
+        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
+              'exchange-{exchange}/{name}.tar.gz'.format(
+            exchange=exchange_name,
+            name=name)
+
+        bytes = download_without_progress(url)
+        with tarfile.open('r', fileobj=bytes) as tar:
+            tar.extractall(path)
+
+    return path
+
+
+def get_df_from_arrays(arrays, periods):
+    """
+    A DataFrame from the specified OHCLV arrays.
+
+    Parameters
+    ----------
+    arrays: Object
+    periods: DateTimeIndex
+
+    Returns
+    -------
+    DataFrame
+
+    """
+    ohlcv = dict()
+    for index, field in enumerate(
+            ['open', 'high', 'low', 'close', 'volume']):
+        ohlcv[field] = arrays[index].flatten()
+
+    df = pd.DataFrame(
+        data=ohlcv,
+        index=periods
+    )
+    return df
+
+
+def range_in_bundle(asset, start_dt, end_dt, reader):
+    """
+    Evaluate whether price data of an asset is included has been ingested in
+    the exchange bundle for the given date range.
+
+    Parameters
+    ----------
+    asset: TradingPair
+    start_dt: datetime
+    end_dt: datetime
+    reader: BcolzBarMinuteReader
+
+    Returns
+    -------
+    bool
+
+    """
+    has_data = True
+    dates = [start_dt, end_dt]
+
+    while dates and has_data:
+        try:
+            dt = dates.pop(0)
+            close = reader.get_value(asset.sid, dt, 'close')
+
+            if np.isnan(close):
+                has_data = False
+
+        except Exception:
+            has_data = False
+
+    return has_data
+
+
+def get_assets(exchange, include_symbols, exclude_symbols):
+    """
+    Get assets from an exchange, including or excluding the specified
+    symbols.
+
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
+
+        return exchange.get_assets(include_symbols_list)
+
+    else:
+        all_assets = exchange.get_assets()
+
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/utils/datetime_utils.py

@@ -1,19 +1,12 @@
 import calendar
-import os
+import re
-import tarfile
+from datetime import datetime, timedelta, date
-from datetime import timedelta, datetime, date
 
-import numpy as np
 import pandas as pd
 import pytz
-from catalyst.assets._assets import TradingPair
 
-from catalyst.data.bundles.core import download_without_progress
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
-from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \
+    InvalidHistoryFrequencyAlias
-    get_exchange_symbols
-
-EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
-API_URL = 'http://data.enigma.co/api/v1'
 
 
 def get_date_from_ms(ms):
@@ -51,46 +44,6 @@ def get_seconds_from_date(date):
     return int((date - epoch).total_seconds())
 
 
-def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
-    """
-    Download and extract a bcolz bundle.
-
-    Parameters
-    ----------
-    exchange_name: str
-    symbol: str
-    data_frequency: str
-    period: str
-
-    Returns
-    -------
-    str
-        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-
-    """
-    root = get_exchange_bundles_folder(exchange_name)
-    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
-        exchange=exchange_name,
-        frequency=data_frequency,
-        symbol=symbol,
-        period=period
-    )
-    path = os.path.join(root, name)
-
-    if not os.path.isdir(path):
-        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
-              'exchange-{exchange}/{name}.tar.gz'.format(
-            exchange=exchange_name,
-            name=name
-        )
-
-        bytes = download_without_progress(url)
-        with tarfile.open('r', fileobj=bytes) as tar:
-            tar.extractall(path)
-
-    return path
-
-
 def get_delta(periods, data_frequency):
     """
     Get a time delta based on the specified data frequency.
@@ -109,7 +62,7 @@ def get_delta(periods, data_frequency):
         if data_frequency == 'minute' else timedelta(days=periods)
 
 
-def get_periods_range(start_dt, end_dt, freq):
+def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
     """
     Get a date range for the specified parameters.
 
@@ -130,8 +83,39 @@ def get_periods_range(start_dt, end_dt, freq):
     elif freq == 'daily':
         freq = 'D'
 
+    if start_dt is not None and end_dt is not None and periods is None:
+
         return pd.date_range(start_dt, end_dt, freq=freq)
 
+    elif periods is not None and (start_dt is not None or end_dt is not None):
+        _, unit_periods, unit, _ = get_frequency(freq)
+        adj_periods = periods * unit_periods
+
+        # TODO: standardize time aliases to avoid any mapping
+        unit = 'd' if unit == 'D' else 'm'
+        delta = pd.Timedelta(adj_periods, unit)
+
+        if start_dt is not None:
+            return pd.date_range(
+                start=start_dt,
+                end=start_dt + delta,
+                freq=freq,
+                closed='left',
+            )
+
+        else:
+            return pd.date_range(
+                start=end_dt - delta,
+                end=end_dt,
+                freq=freq,
+            )
+
+    else:
+        raise ValueError(
+            'Choose only two parameters between start_dt, end_dt '
+            'and periods.'
+        )
+
 
 def get_periods(start_dt, end_dt, freq):
     """
@@ -148,7 +132,7 @@ def get_periods(start_dt, end_dt, freq):
     int
 
     """
-    return len(get_periods_range(start_dt, end_dt, freq))
+    return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
 
 
 def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
@@ -160,6 +144,7 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     end_dt: datetime
     bar_count: int
     data_frequency: str
+    include_first
 
     Returns
     -------
@@ -193,8 +178,10 @@ def get_period_label(dt, data_frequency):
     str
 
     """
-    return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
+    if data_frequency == 'minute':
-        else '{}'.format(dt.year)
+        return '{}-{:02d}'.format(dt.year, dt.month)
+    else:
+        return '{}'.format(dt.year)
 
 
 def get_month_start_end(dt, first_day=None, last_day=None):
@@ -261,99 +248,80 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_df_from_arrays(arrays, periods):
+def get_frequency(freq, data_frequency=None):
     """
-    A DataFrame from the specified OHCLV arrays.
+    Get the frequency parameters.
+
+    Notes
+    -----
+    We're trying to use Pandas convention for frequency aliases.
 
     Parameters
     ----------
-    arrays: Object
+    freq: str
-    periods: DateTimeIndex
+    data_frequency: str
 
     Returns
     -------
-    DataFrame
+    str, int, str, str
 
     """
-    ohlcv = dict()
+    if data_frequency is None:
-    for index, field in enumerate(
+        data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
-            ['open', 'high', 'low', 'close', 'volume']):
-        ohlcv[field] = arrays[index].flatten()
 
-    df = pd.DataFrame(
+    if freq == 'minute':
-        data=ohlcv,
+        unit = 'T'
-        index=periods
+        candle_size = 1
-    )
-    return df
 
-
+    elif freq == 'daily':
-def range_in_bundle(asset, start_dt, end_dt, reader):
+        unit = 'D'
-    """
+        candle_size = 1
-    Evaluate whether price data of an asset is included has been ingested in
-    the exchange bundle for the given date range.
-
-    Parameters
-    ----------
-    asset: TradingPair
-    start_dt: datetime
-    end_dt: datetime
-    reader: BcolzBarMinuteReader
-
-    Returns
-    -------
-    bool
-
-    """
-    has_data = True
-    dates = [start_dt, end_dt]
-
-    while dates and has_data:
-        try:
-            dt = dates.pop(0)
-            close = reader.get_value(asset.sid, dt, 'close')
-
-            if np.isnan(close):
-                has_data = False
-
-        except Exception as e:
-            has_data = False
-
-    return has_data
-
-
-def get_assets(exchange, include_symbols, exclude_symbols):
-    """
-    Get assets from an exchange, including or excluding the specified
-    symbols.
-
-    Parameters
-    ----------
-    exchange: Exchange
-    include_symbols: str
-    exclude_symbols: str
-
-    Returns
-    -------
-    list[TradingPair]
-
-    """
-    if include_symbols is not None:
-        include_symbols_list = include_symbols.split(',')
-
-        return exchange.get_assets(include_symbols_list)
 
     else:
-        all_assets = exchange.get_assets()
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-
+        if freq_match:
-        if exclude_symbols is not None:
+            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
-            exclude_symbols_list = exclude_symbols.split(',')
+                else 1
-
+            unit = freq_match.group(2)
-            assets = []
-            for asset in all_assets:
-                if asset.symbol not in exclude_symbols_list:
-                    assets.append(asset)
-
-            return assets
 
         else:
-            return all_assets
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+    # TODO: some exchanges support H and W frequencies but not bundles
+    # Find a way to pass-through these parameters to exchanges
+    # but resample from minute or daily in backtest mode
+    # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
+    # Pandas offet aliases (used by Catalyst) and the CCXT timeframes
+    if unit.lower() == 'd':
+        unit = 'D'
+        alias = '{}D'.format(candle_size)
+
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm' or unit == 'T':
+        unit = 'T'
+        alias = '{}T'.format(candle_size)
+
+        if data_frequency == 'daily':
+            data_frequency = 'minute'
+
+    # elif unit.lower() == 'h':
+    #     candle_size = candle_size * 60
+    #
+    #     alias = '{}T'.format(candle_size)
+    #     if data_frequency == 'daily':
+    #         data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyAlias(freq=freq)
+
+    return alias, candle_size, unit, data_frequency
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)
+
+
+def get_epoch():
+    return pd.to_datetime('1970-1-1', utc=True)

catalyst/exchange/utils/exchange_utils.py

@@ -2,7 +2,6 @@ import hashlib
 import json
 import os
 import pickle
-import re
 import shutil
 from datetime import date, datetime
 
@@ -12,8 +11,9 @@ from six import string_types
 from six.moves.urllib import request
 
 from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
-    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
+from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
+    ExchangeJSONDecoder
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
@@ -62,6 +62,13 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
+def is_blacklist(exchange_name, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+    filename = os.path.join(exchange_folder, 'blacklist.txt')
+
+    return os.path.exists(filename)
+
+
 def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
@@ -101,20 +108,6 @@ def download_exchange_symbols(exchange_name, environ=None):
     return response
 
 
-def symbols_parser(asset_def):
-    for key, value in asset_def.items():
-        match = isinstance(value, string_types) \
-                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
-
-        if match:
-            try:
-                asset_def[key] = pd.to_datetime(value, utc=True)
-            except ValueError:
-                pass
-
-    return asset_def
-
-
 def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     """
     The de-serialized content of the exchange's symbols.json.
@@ -135,15 +128,18 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
                 pd.Timestamp('now', tz='UTC') - last_modified_time(
                 filename)).days > 1):
+        try:
             download_exchange_symbols(exchange_name, environ)
+        except Exception as e:
+            pass
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
             try:
-                data = json.load(data_file, object_hook=symbols_parser)
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
                 return data
 
-            except ValueError as e:
+            except ValueError:
                 return dict()
     else:
         raise ExchangeSymbolsNotFound(
@@ -195,7 +191,7 @@ def get_symbols_string(assets):
     return ', '.join([asset.symbol for asset in array])
 
 
-def get_exchange_auth(exchange_name, environ=None):
+def get_exchange_auth(exchange_name, alias=None, environ=None):
     """
     The de-serialized contend of the exchange's auth.json file.
 
@@ -210,7 +206,8 @@ def get_exchange_auth(exchange_name, environ=None):
 
     """
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'auth.json')
+    name = 'auth' if alias is None else alias
+    filename = os.path.join(exchange_folder, '{}.json'.format(name))
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
@@ -266,7 +263,7 @@ def get_algo_folder(algo_name, environ=None):
     return algo_folder
 
 
-def get_algo_object(algo_name, key, environ=None, rel_path=None):
+def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
     """
     The de-serialized object of the algo name and key.
 
@@ -290,19 +287,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
 
-    filename = os.path.join(folder, key + '.p')
+    name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
+    filename = os.path.join(folder, name)
 
     if os.path.isfile(filename):
-        try:
+        if how == 'pickle':
             with open(filename, 'rb') as handle:
                 return pickle.load(handle)
-        except Exception as e:
+
-            return None
+        else:
+            with open(filename) as data_file:
+                data = json.load(data_file, cls=ExchangeJSONDecoder)
+                return data
+
     else:
         return None
 
 
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
+                     how='pickle'):
     """
     Serialize and save an object by algo name and key.
 
@@ -321,8 +324,13 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
-    filename = os.path.join(folder, key + '.p')
+    if how == 'json':
+        filename = os.path.join(folder, '{}.json'.format(key))
+        with open(filename, 'wt') as handle:
+            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
 
+    else:
+        filename = os.path.join(folder, '{}.p'.format(key))
         with open(filename, 'wb') as handle:
             pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
@@ -428,6 +436,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
+def has_bundle(exchange_name, data_frequency, environ=None):
+    exchange_folder = get_exchange_folder(exchange_name, environ)
+
+    folder_name = '{}_bundle'.format(data_frequency.lower())
+    folder = os.path.join(exchange_folder, folder_name)
+
+    return os.path.isdir(folder)
+
+
 def symbols_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -495,67 +512,6 @@ def get_common_assets(exchanges):
     return assets
 
 
-def get_frequency(freq, data_frequency):
-    """
-    Get the frequency parameters.
-
-    Notes
-    -----
-    We're trying to use Pandas convention for frequency aliases.
-
-    Parameters
-    ----------
-    freq: str
-    data_frequency: str
-
-    Returns
-    -------
-    str, int, str, str
-
-    """
-    if freq == 'minute':
-        unit = 'T'
-        candle_size = 1
-
-    elif freq == 'daily':
-        unit = 'D'
-        candle_size = 1
-
-    else:
-        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
-                else 1
-            unit = freq_match.group(2)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=freq)
-
-    if unit.lower() == 'd':
-        alias = '{}D'.format(candle_size)
-
-        if data_frequency == 'minute':
-            data_frequency = 'daily'
-
-    elif unit.lower() == 'm' or unit == 'T':
-        alias = '{}T'.format(candle_size)
-
-        if data_frequency == 'daily':
-            data_frequency = 'minute'
-
-    # elif unit.lower() == 'h':
-    #     candle_size = candle_size * 60
-    #
-    #     alias = '{}T'.format(candle_size)
-    #     if data_frequency == 'daily':
-    #         data_frequency = 'minute'
-
-    else:
-        raise InvalidHistoryFrequencyAlias(freq=freq)
-
-    return alias, candle_size, unit, data_frequency
-
-
 def resample_history_df(df, freq, field):
     """
     Resample the OHCLV DataFrame using the specified frequency.
@@ -604,6 +560,7 @@ def mixin_market_params(exchange_name, params, market):
     # TODO: make this more externalized / configurable
     if 'lot' in market:
         params['min_trade_size'] = market['lot']
+        params['lot'] = market['lot']
 
     if exchange_name == 'bitfinex':
         params['maker'] = 0.001
@@ -624,6 +581,83 @@ def mixin_market_params(exchange_name, params, market):
         if 'minimum_order_size' in info:
             params['min_trade_size'] = float(info['minimum_order_size'])
 
+            if 'lot' not in params:
+                params['lot'] = params['min_trade_size']
 
-def from_ms_timestamp(ms):
+
-    return pd.to_datetime(ms, unit='ms', utc=True)
+def group_assets_by_exchange(assets):
+    exchange_assets = dict()
+    for asset in assets:
+        if asset.exchange not in exchange_assets:
+            exchange_assets[asset.exchange] = list()
+
+        exchange_assets[asset.exchange].append(asset)
+
+    return exchange_assets
+
+
+def get_catalyst_symbol(market_or_symbol):
+    """
+    The Catalyst symbol.
+
+    Parameters
+    ----------
+    market_or_symbol
+
+    Returns
+    -------
+
+    """
+    if isinstance(market_or_symbol, string_types):
+        parts = market_or_symbol.split('/')
+        return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+    else:
+        return '{}_{}'.format(
+            market_or_symbol['base'].lower(),
+            market_or_symbol['quote'].lower(),
+        )
+
+
+def save_asset_data(folder, df, decimals=8):
+    symbols = df.index.get_level_values('symbol')
+    for symbol in symbols:
+        symbol_df = df.loc[(symbols == symbol)]  # Type: pd.DataFrame
+
+        filename = os.path.join(folder, '{}.csv'.format(symbol))
+        if os.path.exists(filename):
+            print_headers = False
+
+        else:
+            print_headers = True
+
+        with open(filename, 'a') as f:
+            symbol_df.to_csv(
+                path_or_buf=f,
+                header=print_headers,
+                float_format='%.{}f'.format(decimals),
+            )
+
+
+def get_candles_df(candles, field, freq, bar_count, end_dt,
+                   previous_value=None):
+    all_series = dict()
+    for asset in candles:
+        periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
+
+        dates = [candle['last_traded'] for candle in candles[asset]]
+        values = [candle[field] for candle in candles[asset]]
+        series = pd.Series(values, index=dates)
+
+        series = series.reindex(
+            periods,
+            method='ffill',
+            fill_value=previous_value,
+        )
+        series.sort_index(inplace=True)
+        all_series[asset] = series
+
+    df = pd.DataFrame(all_series)
+    df.dropna(inplace=True)
+
+    return df

catalyst/exchange/utils/factory.py

@@ -0,0 +1,97 @@
+import os
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
+    get_exchange_folder, is_blacklist
+from logbook import Logger
+
+log = Logger('factory', level=LOG_LEVEL)
+exchange_cache = dict()
+
+
+def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
+                 skip_init=False, auth_alias=None):
+    key = (exchange_name, base_currency)
+    if key in exchange_cache:
+        return exchange_cache[key]
+
+    exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
+        )
+
+    exchange = CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+    )
+    exchange_cache[key] = exchange
+
+    if not skip_init:
+        exchange.init()
+
+    return exchange
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges
+
+
+def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
+                   base_currency=None):
+    """
+    Find exchanges filtered by a list of feature.
+
+    Parameters
+    ----------
+    features: str
+        The list of features.
+
+    skip_blacklist: bool
+    is_authenticated: bool
+    base_currency: bool
+
+    Returns
+    -------
+    list[Exchange]
+
+    """
+    exchange_names = CCXT.find_exchanges(features, is_authenticated)
+
+    exchanges = []
+    for exchange_name in exchange_names:
+        if skip_blacklist and is_blacklist(exchange_name):
+            continue
+
+        exchange = get_exchange(
+            exchange_name=exchange_name,
+            skip_init=True,
+            base_currency=base_currency,
+        )
+
+        if features is not None:
+            if 'dailyBundle' in features \
+                    and not exchange.has_bundle('daily'):
+                continue
+
+            elif 'minuteBundle' in features \
+                    and not exchange.has_bundle('minute'):
+                continue
+
+        exchanges.append(exchange)
+
+    return exchanges

catalyst/exchange/utils/live_chart_utils.py

@@ -0,0 +1,131 @@
+import matplotlib.dates as mdates
+import pandas as pd
+
+from catalyst.exchange.exchange_errors import \
+    MismatchingBaseCurrenciesExchanges
+
+fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+
+
+def format_ax(ax):
+    """
+    Trying to assign reasonable parameters to the time axis.
+
+    Parameters
+    ----------
+    ax:
+
+    """
+    # TODO: room for improvement
+    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
+    ax.xaxis.set_major_formatter(fmt)
+
+    locator = mdates.HourLocator(interval=4)
+    locator.MAXTICKS = 5000
+    ax.xaxis.set_minor_locator(locator)
+
+    datemin = pd.Timestamp.utcnow()
+    ax.set_xlim(datemin)
+
+    ax.grid(True)
+
+
+def set_legend(ax):
+    """
+    Set legend on the chart.
+
+    Parameters
+    ----------
+    ax
+
+    """
+    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
+
+
+def draw_pnl(ax, df):
+    """
+    Draw p&l line on the chart.
+
+    """
+    ax.clear()
+    ax.set_title('Performance')
+    index = df.index.unique()
+    dt = index.get_level_values(level=0)
+    pnl = index.get_level_values(level=4)
+    ax.plot(
+        dt, pnl, '-',
+        color='green',
+        linewidth=1.0,
+        label='Performance'
+    )
+
+    def perc(val):
+        return '{:2f}'.format(val)
+
+    ax.format_ydata = perc
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_custom_signals(ax, df):
+    """
+    Draw custom signals on the chart.
+
+    """
+    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
+
+    ax.clear()
+    ax.set_title('Custom Signals')
+    for index, column in enumerate(df.columns.values.tolist()):
+        ax.plot(df.index, df[column], '-',
+                color=colors[index],
+                linewidth=1.0,
+                label=column
+                )
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_exposure(ax, df, context):
+    """
+    Draw exposure line on the chart.
+
+    """
+    # TODO: list exchanges in graph
+    base_currency = None
+    positions = []
+    for exchange_name in context.exchanges:
+        exchange = context.exchanges[exchange_name]
+
+        if not base_currency:
+            base_currency = exchange.base_currency
+        elif base_currency != exchange.base_currency:
+            raise MismatchingBaseCurrenciesExchanges(
+                base_currency=base_currency,
+                exchange_name=exchange.name,
+                exchange_currency=exchange.base_currency
+            )
+
+        positions += exchange.portfolio.positions
+
+    ax.clear()
+    ax.set_title('Exposure')
+    ax.plot(df.index, df['base_currency'], '-',
+            color='green',
+            linewidth=1.0,
+            label='Base Currency: {}'.format(base_currency.upper())
+            )
+
+    symbols = []
+    for position in positions:
+        symbols.append(position.symbol)
+
+    ax.plot(df.index, df['long_exposure'], '-',
+            color='blue',
+            linewidth=1.0,
+            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
+
+    set_legend(ax)
+    format_ax(ax)

catalyst/exchange/utils/serialization_utils.py

@@ -0,0 +1,69 @@
+import json
+import re
+from json import JSONEncoder
+
+import pandas as pd
+from catalyst.constants import DATE_TIME_FORMAT
+from six import string_types
+
+
+class ExchangeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, pd.Timestamp):
+            return obj.strftime(DATE_TIME_FORMAT)
+
+        # Let the base class default method raise the TypeError
+        return JSONEncoder.default(self, obj)
+
+
+class ExchangeJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        json.JSONDecoder.__init__(
+            self, object_hook=self.object_hook, *args, **kwargs
+        )
+
+    def recursive_iter(self, obj):
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                match = isinstance(value, string_types) and re.search(
+                    r'(\d{4}-\d{2}-\d{2}).*', value
+                )
+                if match:
+                    try:
+                        obj[key] = pd.to_datetime(value, utc=True)
+                    except ValueError:
+                        pass
+
+        elif any(isinstance(obj, t) for t in (list, tuple)):
+            for item in obj:
+                self.recursive_iter(item)
+
+    def object_hook(self, obj):
+        self.recursive_iter(obj)
+        return obj
+
+
+def portfolio_to_dict(portfolio):
+    positions = []
+    for asset in portfolio.positions:
+        p = portfolio.positions[asset]  # Type: Position
+
+        position = dict(
+            symbol=asset.symbol,
+            exchange=asset.exchange,
+            amount=p.amount,
+            cost_basis=p.cost_basis,
+            last_sale_price=p.last_sale_price,
+            last_sale_date=p.last_sale_date,
+        )
+        positions.append(position)
+
+    portfolio_dict = vars(portfolio)
+    portfolio_dict['positions'] = positions
+
+    return portfolio_dict
+
+
+def portfolio_from_dict(self, portfolio_data):
+    from catalyst.protocol import Portfolio
+    return Portfolio()

catalyst/exchange/utils/stats_utils.py

@@ -0,0 +1,486 @@
+import copy
+import csv
+import json
+import numbers
+import os
+import time
+
+import numpy as np
+import pandas as pd
+from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_algo_folder
+from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
+
+s3_conn = []
+mailgun = []
+
+
+def trend_direction(series):
+    if series[-1] is np.nan or series[-1] is np.nan:
+        return None
+
+    if series[-1] > series[-2]:
+        return 'up'
+    else:
+        return 'down'
+
+
+def crossover(source, target):
+    """
+    The `x`-series is defined as having crossed over `y`-series if the value
+    of `x` is greater than the value of `y` and the value of `x` was less than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] >= target > source[-2]:
+            return True
+        else:
+            return False
+
+    else:
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] > target[-1] and source[-2] < target[-2]:
+            return True
+        else:
+            return False
+
+
+def crossunder(source, target):
+    """
+    The `x`-series is defined as having crossed under `y`-series if the value
+    of `x` is less than the value of `y` and the value of `x` was greater than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] < target <= source[-2]:
+            return True
+        else:
+            return False
+    else:
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] < target[-1] and source[-2] >= target[-2]:
+            return True
+        else:
+            return False
+
+
+def vwap(df):
+    """
+    Volume-weighted average price (VWAP) is a ratio generally used by
+    institutional investors and mutual funds to make buys and sells so as not
+    to disturb the market prices with large orders. It is the average share
+    price of a stock weighted against its trading volume within a particular
+    time frame, generally one day.
+
+    Read more: Volume Weighted Average Price - VWAP
+    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+
+    Returns
+    -------
+
+    """
+    if 'close' not in df.columns or 'volume' not in df.columns:
+        raise ValueError('price data must include `volume` and `close`')
+
+    vol_sum = np.nansum(df['volume'].values)
+
+    try:
+        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
+    except ZeroDivisionError:
+        ret = np.nan
+
+    return ret
+
+
+def set_position_row(row, asset, asset_values=list()):
+    """
+    Apply the position data as individual columns.
+
+    Parameters
+    ----------
+    row: dict[str, Object]
+    asset: TradingPair
+    asset_values: list[str]
+        If a recorded_col contains a tuple which first value is an asset
+        matching a position, its value will be displayed with the
+        position and not in the index.
+
+    Returns
+    -------
+
+    """
+    asset_cols = ['symbol']
+    row['symbol'] = asset.symbol
+
+    position = next((p for p in row['positions'] if p['sid'] == asset), None)
+
+    columns = ['amount', 'cost_basis', 'last_sale_price']
+    for column in columns:
+        if position is not None:
+            row[column] = position[column]
+
+        else:
+            row[column] = 0
+
+        asset_cols.append(column)
+
+    values = asset_values[asset] if asset in asset_values else list()
+    for column in values:
+        row[column] = values[column]
+
+        asset_cols.append(column)
+
+    return asset_cols
+
+
+def prepare_stats(stats, recorded_cols=list()):
+    """
+    Prepare the stats DataFrame for user-friendly output.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    asset_cols = list()
+
+    stats = copy.deepcopy(stats)
+    # Using a copy since we are adding rows inside the loop.
+    for row_index, row_data in enumerate(list(stats)):
+        assets = [p['sid'] for p in row_data['positions']]
+
+        asset_values = dict()
+        if recorded_cols is not None:
+            for column in recorded_cols[:]:
+                value = row_data[column]
+                if isinstance(value, pd.Series):
+                    value = value.to_dict()
+
+                if type(value) is dict:
+                    for asset in value:
+                        if not isinstance(asset, TradingPair):
+                            break
+
+                        if asset not in assets:
+                            assets.append(asset)
+
+                        if asset not in asset_values:
+                            asset_values[asset] = dict()
+
+                        asset_values[asset][column] = value[asset]
+
+        if len(assets) == 1:
+            row = stats[row_index]
+            asset_cols = set_position_row(row, assets[0], asset_values)
+
+        elif len(assets) > 1:
+            for asset_index, asset in enumerate(assets):
+                if asset_index > 0:
+                    row = copy.deepcopy(row_data)
+                    stats.append(row)
+
+                else:
+                    row = stats[row_index]
+
+                asset_cols = set_position_row(row, assets[asset_index],
+                                              asset_values)
+
+    df = pd.DataFrame(stats)
+
+    index_cols = [
+        'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
+        'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
+    ]
+
+    # Removing the asset specific entries
+    if recorded_cols is not None:
+        recorded_cols = [x for x in recorded_cols if x not in asset_cols]
+        for column in recorded_cols:
+            index_cols.append(column)
+
+    df['orders'] = df['orders'].apply(lambda orders: len(orders))
+    df['transactions'] = df['transactions'].apply(
+        lambda transactions: len(transactions)
+    )
+
+    if asset_cols:
+        columns = asset_cols
+        df.set_index(index_cols, drop=True, inplace=True)
+
+    else:
+        columns = index_cols
+        columns.remove('period_close')
+        df.set_index('period_close', drop=False, inplace=True)
+
+    df.dropna(axis=1, how='all', inplace=True)
+    df.sort_index(axis=0, level=0, inplace=True)
+
+    return df, columns
+
+
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
+    """
+    Format and print the last few rows of a statistics DataFrame.
+    See the pyfolio project for the data structure.
+
+    Parameters
+    ----------
+    stats: list[Object]
+        An array of statistics for the period.
+
+    num_rows: int
+        The number of rows to display on the screen.
+
+    Returns
+    -------
+    str
+
+    """
+    if isinstance(stats, pd.DataFrame):
+        stats = list(stats.T.to_dict().values())
+        stats.sort(key=itemgetter('period_close'))
+
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
+
+    df, columns = prepare_stats(
+        display_stats, recorded_cols=recorded_cols
+    )
+    set_print_settings()
+    return df.to_string(columns=columns)
+
+
+def get_csv_stats(stats, recorded_cols=None):
+    """
+    Create a CSV buffer from the stats DataFrame.
+
+    Parameters
+    ----------
+    path: str
+    stats: list[Object]
+    recorded_cols: list[str]
+
+    Returns
+    -------
+
+    """
+    df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
+
+    return df.to_csv(
+        None,
+        columns=columns,
+        # encoding='utf-8',
+        quoting=csv.QUOTE_NONNUMERIC
+    ).encode()
+
+
+def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
+                folder='catalyst/stats', bytes_to_write=None):
+    """
+    Uploads the performance stats to a S3 bucket.
+
+    Parameters
+    ----------
+    uri: str
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+    folder: str
+    bytes_to_write: str
+        Option to reuse bytes instead of re-computing the csv
+
+    Returns
+    -------
+
+    """
+    if not s3_conn:
+        import boto3
+        s3_conn.append(boto3.resource('s3'))
+
+    s3 = s3_conn[0]
+
+    if bytes_to_write is None:
+        bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    now = pd.Timestamp.utcnow()
+    timestr = now.strftime('%Y%m%d')
+    pid = os.getpid()
+
+    parts = uri.split('//')
+    path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
+        folder=folder,
+        algo=algo_namespace,
+        time=timestr,
+        pid=pid,
+    )
+    obj = s3.Object(parts[1], path)
+    obj.put(Body=bytes_to_write)
+
+
+def email_error(algo_name, dt, e, environ=None):
+    import requests
+    import traceback
+
+    if not mailgun:
+        root = data_root(environ)
+        filename = os.path.join(root, 'mailgun.json')
+        if not os.path.exists(filename):
+            raise ValueError(
+                'mailgun.json not found in the catalyst data folder'
+            )
+
+        with open(filename) as data_file:
+            mailgun.append(json.load(data_file))
+
+    mg = mailgun[0]
+
+    return requests.post(
+        mg['url'],
+        auth=("api", mg['api']),
+        data={
+            "from": mg['from'],
+            "to": mg['to'],
+            "subject": 'Error: {}'.format(algo_name),
+            "text": '{}\n\n{}\n{}'.format(
+                dt, e, traceback.format_exc()
+            )})
+
+
+def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
+    """
+    Saves the performance stats to the algo local folder.
+
+    Parameters
+    ----------
+    stats: list[Object]
+    algo_namespace: str
+    recorded_cols: list[str]
+
+    Returns
+    -------
+    str
+
+    """
+    bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
+
+    timestr = time.strftime('%Y%m%d')
+    folder = get_algo_folder(algo_namespace)
+
+    stats_folder = os.path.join(folder, 'stats')
+    ensure_directory(stats_folder)
+
+    filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
+
+    with open(filename, 'wb') as handle:
+        handle.write(bytes_to_write)
+
+    return bytes_to_write
+
+
+def df_to_string(df):
+    """
+    Create a formatted str representation of the DataFrame.
+
+    Parameters
+    ----------
+    df: DataFrame
+
+    Returns
+    -------
+    str
+
+    """
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+    return df.to_string()
+
+
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
+def extract_transactions(perf):
+    """
+    Compute indexes for buy and sell transactions
+
+    Parameters
+    ----------
+    perf: DataFrame
+        The algo performance DataFrame.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of transactions.
+
+    """
+    trans_list = perf.transactions.values
+    all_trans = [t for sublist in trans_list for t in sublist]
+    all_trans.sort(key=lambda t: t['dt'])
+
+    transactions = pd.DataFrame(all_trans)
+    if not transactions.empty:
+        transactions.set_index('dt', inplace=True, drop=True)
+    return transactions

catalyst/exchange/utils/test_utils.py

@@ -0,0 +1,82 @@
+import os
+import random
+import tempfile
+
+from catalyst.assets._assets import TradingPair
+from catalyst.exchange.utils.exchange_utils import get_exchange_folder
+from catalyst.exchange.utils.factory import find_exchanges
+from catalyst.utils.paths import ensure_directory
+
+
+def handle_exchange_error(exchange, e):
+    try:
+        message = '{}: {}'.format(
+            e.__class__, e.message.decode('ascii', 'ignore')
+        )
+    except Exception:
+        message = 'unexpected error'
+
+    folder = get_exchange_folder(exchange.name)
+    filename = os.path.join(folder, 'blacklist.txt')
+    with open(filename, 'wt') as handle:
+        handle.write(message)
+
+
+def select_random_exchanges(population=3, features=None,
+                            is_authenticated=False, base_currency=None):
+    all_exchanges = find_exchanges(
+        features=features,
+        is_authenticated=is_authenticated,
+        base_currency=base_currency,
+    )
+
+    if population is not None:
+        if len(all_exchanges) < population:
+            population = len(all_exchanges)
+
+        exchanges = random.sample(all_exchanges, population)
+
+    else:
+        exchanges = all_exchanges
+
+    return exchanges
+
+
+def select_random_assets(all_assets, population=3):
+    assets = random.sample(all_assets, population)
+    return assets
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
+    else:
+        asset_folder = ','.join(
+            ['{}_{}'.format(a.exchange, a.symbol) for a in assets]
+        )
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path, folder

catalyst/exchange/validator.py

@@ -1,142 +0,0 @@
-import os
-import tempfile
-
-import pandas as pd
-import six
-from catalyst.assets._assets import TradingPair, get_calendar
-from logbook import Logger
-from pandas.util.testing import assert_frame_equal
-
-from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
-from catalyst.exchange.factory import get_exchanges
-from catalyst.utils.paths import ensure_directory
-
-log = Logger('Validator', level=LOG_LEVEL)
-
-
-def output_df(df, assets, name=None):
-    """
-    Outputs a price DataFrame to a temp folder.
-
-    Parameters
-    ----------
-    df: pd.DataFrame
-    assets
-    name
-
-    Returns
-    -------
-
-    """
-    if isinstance(assets, TradingPair):
-        exchange_folder = assets.exchange
-        asset_folder = assets.symbol
-    else:
-        exchange_folder = ','.join([asset.exchange for asset in assets])
-        asset_folder = ','.join([asset.symbol for asset in assets])
-
-    folder = os.path.join(
-        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
-    )
-    ensure_directory(folder)
-
-    if name is None:
-        name = 'output'
-
-    path = os.path.join(folder, '{}.csv'.format(name))
-    df.to_csv(path)
-
-    return path
-
-
-class Validator(object):
-    def __init__(self, data_portal):
-        self.data_portal = data_portal
-
-    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
-                                     sample_minutes):
-        """
-        Creates DataFrames from the bundle and exchange for the specified
-        data set.
-
-        Parameters
-        ----------
-        exchange: Exchange
-        assets
-        end_dt
-        bar_count
-        sample_minutes
-
-        Returns
-        -------
-
-        """
-        freq = '{}T'.format(sample_minutes)
-
-        log.info('creating data sample from bundle')
-        df1 = self.data_portal.get_history_window(
-            assets=assets,
-            end_dt=end_dt,
-            bar_count=bar_count,
-            frequency=freq,
-            field='close',
-            data_frequency='minute'
-        )
-        path = output_df(df1, assets, '{}_resampled'.format(freq))
-        log.info('saved resampled bundle candles: {}\n{}'.format(
-            path, df1.tail(10))
-        )
-
-        log.info('creating data sample from exchange api')
-        candles = exchange.get_candles(
-            end_dt=end_dt,
-            freq='{}T'.format(sample_minutes),
-            assets=assets,
-            bar_count=bar_count
-        )
-
-        series = dict()
-        for asset in assets:
-            series[asset] = pd.Series(
-                data=[candle['close'] for candle in candles[asset]],
-                index=[candle['last_traded'] for candle in candles[asset]]
-            )
-
-        df2 = pd.DataFrame(series)
-        path = output_df(df2, assets, '{}_api'.format(freq))
-        log.info('saved exchange api candles: {}\n{}'.format(
-            path, df2.tail(10))
-        )
-
-        try:
-            assert_frame_equal(df1, df2)
-            return True
-        except:
-            log.warn('differences found in dataframes')
-            return False
-
-
-if __name__ == '__main__':
-    exchanges = get_exchanges(['poloniex'])
-    exchange = six.next(six.itervalues(exchanges))
-    assets = exchange.get_assets(symbols=['eth_btc'])
-
-    open_calendar = get_calendar('OPEN')
-    asset_finder = AssetFinderExchange()
-    data_portal = DataPortalExchangeBacktest(
-        exchanges=exchanges,
-        asset_finder=asset_finder,
-        trading_calendar=open_calendar,
-        first_trading_day=None  # will set dynamically based on assets
-    )
-    validator = Validator(data_portal=data_portal)
-
-    validator.compare_bundle_with_exchange(
-        exchange=exchange,
-        assets=assets,
-        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
-        bar_count=200,
-        sample_minutes=30
-    )

catalyst/finance/execution.py

@@ -15,13 +15,8 @@
 
 import abc
 
-from sys import float_info
-
-from six import with_metaclass
-
-import catalyst.utils.math_utils as zp_math
-
 from numpy import isfinite
+from six import with_metaclass
 
 from catalyst.errors import BadOrderParameters
 

catalyst/finance/risk/cumulative.py

@@ -27,15 +27,15 @@ from .risk import (
     choose_treasury
 )
 
-from empyrical import (
+from catalyst.patches.stats import (
     alpha_beta_aligned,
     annual_volatility,
-    cum_returns,
     downside_risk,
     information_ratio,
     max_drawdown,
     sharpe_ratio,
     sortino_ratio,
+    cum_returns,
 )
 import warnings
 from catalyst.constants import LOG_LEVEL
@@ -161,9 +161,13 @@ class RiskMetricsCumulative(object):
             if len(self.algorithm_returns) == 1:
                 self.algorithm_returns = np.append(0.0, self.algorithm_returns)
 
+        try:
             self.algorithm_cumulative_returns[dt_loc] = cum_returns(
                 self.algorithm_returns
             )[-1]
+        except Exception as e:
+            log.debug('unable to calculate cum returns: {}'.format(e))
+            self.algorithm_cumulative_returns[dt_loc] = np.nan
 
         algo_cumulative_returns_to_date = \
             self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -196,8 +200,11 @@ class RiskMetricsCumulative(object):
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
-        except Exception:
+        except Exception as e:
-            self.benchmark_cumulative_returns[dt_loc] = 0
+            log.debug(
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.benchmark_cumulative_returns[dt_loc] = np.nan
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -269,9 +276,16 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.sharpe[dt_loc] = sharpe_ratio(
             self.algorithm_returns,
         )
+
+        try:
             self.downside_risk[dt_loc] = downside_risk(
                 self.algorithm_returns
             )
+        except Exception as e:
+            log.debug(
+                'unable to calculate downside risk returns: {}'.format(e)
+            )
+            self.downside_risk[dt_loc] = np.nan
 
         try:
             risk = self.downside_risk[dt_loc]
@@ -279,17 +293,26 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
                 self.algorithm_returns,
                 _downside_risk=risk
             )
-        except Exception:
+        except Exception as e:
-            # TODO: what causes it to error out?
+            log.debug(
-            self.sortino[dt_loc] = 0
+                'unable to calculate benchmark cum returns: {}'.format(e)
+            )
+            self.sortino[dt_loc] = np.nan
 
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
         )
+        try:
             self.max_drawdown = max_drawdown(
                 self.algorithm_returns
             )
+        except Exception as e:
+            log.debug(
+                'unable to calculate max drawdown: {}'.format(e)
+            )
+            self.max_drawdown = np.nan
+
         self.max_drawdowns[dt_loc] = self.max_drawdown
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage

catalyst/finance/risk/period.py

@@ -154,8 +154,8 @@ class RiskMetricsPeriod(object):
             self.algorithm_returns.values,
             self.benchmark_returns.values,
         )
-        self.excess_return = self.algorithm_period_returns - \
+        self.excess_return = self.algorithm_period_returns \
-                             self.treasury_period_return
+            - self.treasury_period_return
         self.max_drawdown = max_drawdown(self.algorithm_returns.values)
         self.max_leverage = self.calculate_max_leverage()
 

catalyst/finance/risk/risk.py

@@ -160,7 +160,8 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
                 )
                 break
 
-        if search_day and trading_calendar.name != 'OPEN':          # Supress warning for 'OPEN' calendar
+        # Supress warning for 'OPEN' calendar
+        if search_day and trading_calendar.name != 'OPEN':
             if (search_dist is None or search_dist > 1) and \
                     search_days[0] <= end_session <= search_days[-1]:
                 message = "No rate within 1 trading day of end date = \

catalyst/finance/slippage.py

@@ -41,7 +41,6 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
 
 
-
 class LiquidityExceeded(Exception):
     pass
 

catalyst/pipeline/engine.py

@@ -7,6 +7,7 @@ from abc import (
 )
 from uuid import uuid4
 
+import six
 from six import (
     iteritems,
     with_metaclass,
@@ -33,7 +34,6 @@ from catalyst.utils.sharedoc import copydoc
 
 
 class PipelineEngine(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def run_pipeline(self, pipeline, start_date, end_date):
         """
@@ -118,6 +118,7 @@ class ExplodingPipelineEngine(PipelineEngine):
     """
     A PipelineEngine that doesn't do anything.
     """
+
     def run_pipeline(self, pipeline, start_date, end_date):
         raise NoEngineRegistered(
             "Attempted to run a pipeline but no pipeline "
@@ -484,8 +485,10 @@ class SimplePipelineEngine(PipelineEngine):
             )
 
             if isinstance(term, LoadableTerm):
+                term_key = loader_group_key(term)
+                # TODO: temp workaround
                 to_load = sorted(
-                    loader_groups[loader_group_key(term)],
+                    six.next(six.itervalues(loader_groups)),
                     key=lambda t: t.dataset
                 )
                 loader = get_loader(term)
@@ -565,9 +568,10 @@ class SimplePipelineEngine(PipelineEngine):
                 index=MultiIndex.from_arrays([empty_dates, empty_assets]),
             )
 
-        resolved_assets = array(self._finder.retrieve_all(assets))
+        # TODO: not sure what's wrong with the resolved_assets
+        # resolved_assets = array(self._finder.retrieve_all(assets))
         dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
-        assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
+        assets_kept = repeat_first_axis(assets, len(dates))[mask]
 
         final_columns = {}
         for name in data:

catalyst/pipeline/factors/equity/__init__.py

@@ -1,9 +1,6 @@
 from .statistical import (
-    RollingPearson,
-    RollingLinearRegression,
     RollingLinearRegressionOfReturns,
     RollingPearsonOfReturns,
-    RollingSpearman,
     RollingSpearmanOfReturns,
 )
 from .technical import (

catalyst/pipeline/graph.py

@@ -142,7 +142,7 @@ class TermGraph(object):
         at the end of execution.
         """
         refcounts = self.graph.out_degree()
-        for t in self.outputs.values():
+        for t in list(self.outputs.values()):
             refcounts[t] += 1
 
         for t in initial_terms:
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
                  min_extra_rows=0):
         super(ExecutionPlan, self).__init__(terms)
 
-        for term in terms.values():
+        for term in list(terms.values()):
             self.set_extra_rows(
                 term,
                 all_dates,

catalyst/pipeline/loaders/equity_pricing_loader.py

@@ -38,9 +38,11 @@ class USEquityPricingLoader(PipelineLoader):
 
     def __init__(self, bundle, data_frequency, dataset):
 
-        if data_frequency == 'daily':
+        # TODO: This is currently broken, No Pipeline support for Catalyst
-            reader = bundle.daily_bar_reader
+        # if data_frequency == 'daily':
-        elif daily_bar_reader == 'minute':
+        #    reader = bundle.daily_bar_reader
+        # elif daily_bar_reader == 'minute':
+        if data_frequency == 'minute':
             reader = bundle.minute_bar_reader
         else:
             raise ValueError(
@@ -51,7 +53,9 @@ class USEquityPricingLoader(PipelineLoader):
 
         if data_frequency == 'daily':
             all_sessions = cal.all_sessions
-        elif daily_bar_reader == 'minute':
+        # TODO: this cannot be right, but no pipeline support at the moment
+        # elif daily_bar_reader == 'minute':
+        elif data_frequency == 'minute':
             reader = bundle.minute_bar_reader
             all_sessions = cal.all_minutes
 

catalyst/pipeline/loaders/frame.py

@@ -180,4 +180,3 @@ class DataFrameLoader(PipelineLoader):
     @property
     def columns(self):
         return self._columns
-

catalyst/sources/test_source.py

@@ -144,7 +144,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
             for event in self.event_list:
                 event.sid = assets_by_identifier[event.sid].sid
 
@@ -167,7 +167,7 @@ class SpecificEquityTrades(object):
             for identifier in self.identifiers:
                 assets_by_identifier[identifier] = env.asset_finder.\
                     lookup_generic(identifier, datetime.now())[0]
-            self.sids = [asset.sid for asset in assets_by_identifier.values()]
+            self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
 
         # Hash_value for downstream sorting.
         self.arg_string = hash_args(*args, **kwargs)

catalyst/support/binance_history.py

@@ -0,0 +1,57 @@
+import pandas as pd
+from catalyst import run_algorithm
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = 'cryptopia'
+    context.base_currency = 'btc'
+    context.coins = context.exchanges[context.exchange].assets
+    context.coins = [c for c in context.coins if
+                     c.quote_currency == context.base_currency]
+
+
+def handle_data(context, data):
+    # current date formatted into a string
+    today = data.current_dt
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.coins = context.exchanges[context.exchange].assets
+        context.coins = [c for c in context.coins if
+                         c.quote_currency == context.base_currency]
+
+    # get data every 30 minutes
+    minutes = 1
+    if not context.i % minutes:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            price = data.current(coin, 'price')
+            print(today, pair, price)
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2018-01-17', utc=True)
+    end_date = pd.to_datetime('2018-01-18', utc=True)
+
+    performance = run_algorithm(
+        capital_base=1.0,
+        # amount of base_currency, not always in dollars unless usd
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='cryptopia',
+        data_frequency='minute',
+        base_currency='btc',
+        live=True,
+        live_graph=False,
+        simulate_orders=True,
+        algo_namespace='simple_universe'
+    )

catalyst/support/ccxt_issue_1358.py

@@ -0,0 +1,8 @@
+import ccxt
+
+bitfinex = ccxt.bitfinex()
+bitfinex.verbose = True
+ohlcvs = bitfinex.fetch_ohlcv('ETH/BTC', '30m', 1504224000000)
+
+dt = bitfinex.iso8601(ohlcvs[0][0])
+print(dt)  # should print '2017-09-01T00:00:00.000Z'

catalyst/support/history_multiple_assets.py

@@ -0,0 +1,50 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset1 = symbol('fct_btc')
+    context.asset2 = symbol('btc_usdt')
+    context.coins = [context.asset1, context.asset2]
+
+
+def handle_data(context, data):
+    df = data.history(context.coins,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+    print(df)
+    print(data.current(context.asset1, 'close'))
+    print(data.current(context.asset2, 'close'))
+    exit(0)
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_multi_assets',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_111.py

@@ -0,0 +1,44 @@
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import order_target_percent
+
+NAMESPACE = 'goose7'
+log = Logger(NAMESPACE)
+
+from catalyst.api import record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('trx_btc')
+
+
+def handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    record(btc=price)
+
+    # Only ordering if it does not have any position to avoid trying some
+    # tiny orders with the leftover btc
+    pos_amount = context.portfolio.positions[context.asset].amount
+    if pos_amount > 0:
+        return
+
+    # Adding a limit price to workaround an issue with performance
+    # calculations of market orders
+    order_target_percent(
+        context.asset, 1, limit_price=price * 1.01
+    )
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.003,
+        initialize=initialize,
+        handle_data=handle_data,
+        exchange_name='binance',
+        live=True,
+        algo_namespace=NAMESPACE,
+        base_currency='btc',
+        live_graph=False,
+        simulate_orders=False,
+    )

catalyst/support/issue_112.py

@@ -0,0 +1,44 @@
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    df = data.history(context.asset,
+                      'close',
+                      bar_count=10,
+                      frequency='5T',
+                      )
+
+
+if __name__ == '__main__':
+    LIVE = True
+    if LIVE:
+        run_algorithm(
+            capital_base=1,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='test_algo',
+            base_currency='usdt',
+            live=False,
+            start=pd.to_datetime('2017-12-1', utc=True),
+            end=pd.to_datetime('2017-12-1', utc=True),
+        )

catalyst/support/issue_126.py

@@ -0,0 +1,52 @@
+from catalyst import run_algorithm
+from catalyst.api import order, record, symbol
+import pandas as pd
+
+from catalyst.exchange.utils.stats_utils import get_pretty_stats
+
+
+def initialize(context):
+    context.assets = [symbol('eth_btc'), symbol('eth_usdt')]
+
+
+def handle_data(context, data):
+    order(context.assets[0], 1)
+
+    prices = data.current(context.assets, 'price')
+    record(price=prices)
+    pass
+
+
+def analyze(context, perf):
+    stats = get_pretty_stats(perf)
+    print(stats)
+    pass
+
+
+if __name__ == '__main__':
+    live = True
+    if live:
+        run_algorithm(
+            capital_base=0.01,
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='btc',
+            analyze=analyze,
+            live=True,
+            simulate_orders=True,
+        )
+    else:
+        run_algorithm(
+            capital_base=1000,
+            data_frequency='daily',
+            initialize=initialize,
+            handle_data=handle_data,
+            exchange_name='poloniex',
+            algo_namespace='buy_btc_polo_jh',
+            base_currency='usd',
+            analyze=analyze,
+            start=pd.to_datetime('2017-01-01', utc=True),
+            end=pd.to_datetime('2017-12-25', utc=True),
+        )

catalyst/support/issue_169.py

@@ -0,0 +1,44 @@
+import pandas as pd
+from catalyst.utils.run_algo import run_algorithm
+from catalyst.api import symbol
+from exchange.utils.stats_utils import set_print_settings
+
+
+def initialize(context):
+    context.i = 0
+    context.data = []
+
+
+def handle_data(context, data):
+    prices = data.history(
+        symbol('xlm_eth'),
+        fields=['open', 'high', 'low', 'close'],
+        bar_count=50,
+        frequency='1T'
+    )
+    set_print_settings()
+    print(prices.tail(10))
+    context.data.append(prices)
+
+    context.i = context.i + 1
+    if context.i == 3:
+        context.interrupt_algorithm()
+
+
+def analyze(context, prefs):
+    for dataset in context.data:
+        print(dataset[-2:])
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=0.1,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='binance',
+        algo_namespace='Test candles',
+        base_currency='eth',
+        data_frequency='minute',
+        live=True,
+        simulate_orders=True)

catalyst/support/issue_30.py

@@ -0,0 +1,28 @@
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+
+def initialize(context):
+    context.asset = symbol('bcc_usdt')
+
+
+def handle_data(context, data):
+    data.history(context.asset, ['close'], bar_count=100, frequency='5T')
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=100,
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bittrex',
+        algo_namespace="bittrex_is_broken",
+        base_currency='usdt',
+        data_frequency='minute',
+        simulate_orders=True,
+        live=True)

catalyst/support/issue_44.py

@@ -1,109 +0,0 @@
-import pandas as pd
-from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-
-from catalyst.api import (
-    symbols,
-)
-
-
-def initialize(context):
-    context.i = -1
-    context.base_currency = 'btc'
-
-
-def handle_data(context, data):
-    lookback = 60 * 24 * 7  # (minutes, hours, days)
-    context.i += 1
-    if context.i < lookback:
-        return
-
-    today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
-
-    try:
-        # update universe everyday
-        new_day = 60 * 24
-        if not context.i % new_day:
-            context.universe = universe(context, today)
-
-        # get data every 30 minutes
-        minutes = 30
-        if not context.i % minutes and context.universe:
-            for coin in context.coins:
-                pair = str(coin.symbol)
-
-                # ohlcv data
-                open = data.history(coin, 'open', lookback,
-                                    '1m').ffill().bfill().resample(
-                    '30T').first()
-                high = data.history(coin, 'high', lookback,
-                                    '1m').ffill().bfill().resample('30T').max()
-                low = data.history(coin, 'low', lookback,
-                                   '1m').ffill().bfill().resample('30T').min()
-                close = data.history(coin, 'price', lookback,
-                                     '1m').ffill().bfill().resample(
-                    '30T').last()
-                volume = data.history(coin, 'volume', lookback,
-                                      '1m').ffill().bfill().resample(
-                    '30T').sum()
-
-                print(today, pair, close[-1])
-
-    except Exception as e:
-        print(e)
-
-
-def analyze(context=None, results=None):
-    pass
-
-
-def universe(context, today):
-    json_symbols = get_exchange_symbols('poloniex')
-    poloniex_universe_df = pd.DataFrame.from_dict(
-        json_symbols).transpose().astype(str)
-    poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
-        lambda row: row.symbol.split('_')[1],
-        axis=1)
-    poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
-        lambda row: row.symbol.split('_')[0],
-        axis=1)
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df['base_currency'] == context.base_currency]
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df.symbol != 'gas_btc']
-
-    # Markets currently not working on Catalyst 0.3.1
-    # 2017-01-01
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
-    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
-    print(poloniex_universe_df.head())
-
-    date = str(today).split(' ')[0]
-
-    poloniex_universe_df = poloniex_universe_df[
-        poloniex_universe_df.start_date < date]
-    context.coins = symbols(*poloniex_universe_df.symbol)
-    print(len(poloniex_universe_df))
-    return poloniex_universe_df.symbol.tolist()
-
-
-if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-01-01', utc=True)
-    end_date = pd.to_datetime('2017-10-15', utc=True)
-
-    performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=10000.0,
-                                initialize=initialize,
-                                handle_data=handle_data,
-                                analyze=analyze,
-                                exchange_name='poloniex',
-                                data_frequency='minute',
-                                base_currency='btc',
-                                live=False,
-                                live_graph=False,
-                                algo_namespace='test')

catalyst/support/issue_47.py

@@ -1,139 +0,0 @@
-"""
-Requires Catalyst version 0.3.0 or above
-Tested on Catalyst version 0.3.3
-
-These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
-You simply need to specify the exchange and the market that you want to focus on.
-You will all see how to create a universe and filter it base on the exchange and the market you desire.
-
-The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
-The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
-Use this as the backbone to create your own trading strategies.
-
-Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
-"""
-
-import numpy as np
-import pandas as pd
-from datetime import timedelta
-from catalyst import run_algorithm
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-
-from catalyst.api import (
-    symbols,
-)
-
-
-def initialize(context):
-    context.i = -1  # counts the minutes
-    context.exchange = 'poloniex'  # must match the exchange specified in run_algorithm
-    context.base_currency = 'btc'  # must match the base currency specified in run_algorithm
-
-
-def handle_data(context, data):
-    lookback = 60 * 24 * 7  # (minutes, hours, days) of how far to lookback in the data history
-    context.i += 1
-
-    # current date formatted into a string
-    today = context.blotter.current_dt
-    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
-    lookback_date = today - timedelta(days=(
-    lookback / (60 * 24)))  # subtract the amount of days specified in lookback
-    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[
-        0]  # get only the date as a string
-
-    # update universe everyday
-    new_day = 60 * 24
-    if not context.i % new_day:
-        context.universe = universe(context, lookback_date, date)
-
-    # get data every 30 minutes
-    minutes = 30
-    if not context.i % minutes and context.universe:
-        # we iterate for every pair in the current universe
-        for coin in context.coins:
-            pair = str(coin.symbol)
-
-            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
-            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
-            opened = fill(data.history(coin, 'open', bar_count=lookback,
-                                       frequency='30T')).values
-            high = fill(data.history(coin, 'high', bar_count=lookback,
-                                     frequency='30T')).values
-            low = fill(data.history(coin, 'low', bar_count=lookback,
-                                    frequency='30T')).values
-            close = fill(data.history(coin, 'price', bar_count=lookback,
-                                      frequency='30T')).values
-            volume = fill(data.history(coin, 'volume', bar_count=lookback,
-                                       frequency='30T')).values
-
-            # close[-1] is the equivalent to current price
-            # displays the minute price for each pair every 30 minutes
-            print(
-            today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
-
-            # ----------------------------------------------------------------------------------------------------------
-            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
-            # ----------------------------------------------------------------------------------------------------------
-
-
-def analyze(context=None, results=None):
-    pass
-
-
-# Get the universe for a given exchange and a given base_currency market
-# Example: Poloniex btc Market
-def universe(context, lookback_date, current_date):
-    json_symbols = get_exchange_symbols(
-        context.exchange)  # get all the pairs for the exchange
-    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(
-        str)  # convert into a dataframe
-    universe_df['base_currency'] = universe_df.apply(
-        lambda row: row.symbol.split('_')[1],
-        axis=1)
-    universe_df['market_currency'] = universe_df.apply(
-        lambda row: row.symbol.split('_')[0],
-        axis=1)
-    # Filter all the exchange pairs to only the ones for a give base currency
-    universe_df = universe_df[
-        universe_df['base_currency'] == context.base_currency]
-
-    # Filter all the pairs to ensure that pair existed in the current date range
-    universe_df = universe_df[universe_df.start_date < lookback_date]
-    universe_df = universe_df[universe_df.end_daily >= current_date]
-    context.coins = symbols(
-        *universe_df.symbol)  # convert all the pairs to symbols
-    return universe_df.symbol.tolist()
-
-
-# Replace all NA, NAN or infinite values with its nearest value
-def fill(series):
-    if isinstance(series, pd.Series):
-        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
-    elif isinstance(series, np.ndarray):
-        return pd.Series(series).replace([np.inf, -np.inf],
-                                         np.nan).ffill().bfill().values
-    else:
-        return series
-
-
-if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-01-08', utc=True)
-    end_date = pd.to_datetime('2017-11-13', utc=True)
-
-    performance = run_algorithm(start=start_date, end=end_date,
-                                capital_base=10000.0,
-                                initialize=initialize,
-                                handle_data=handle_data,
-                                analyze=analyze,
-                                exchange_name='poloniex',
-                                data_frequency='minute',
-                                base_currency='btc',
-                                live=False,
-                                live_graph=False,
-                                algo_namespace='simple_universe')
-
-"""
-Run in Terminal (inside catalyst environment):
-python simple_universe.py
-"""

catalyst/support/issue_55.py

@@ -1,4 +1,3 @@
-import talib
 import pandas as pd
 
 from catalyst import run_algorithm

catalyst/support/issue_57.py

@@ -1,46 +0,0 @@
-import talib
-import pandas as pd
-
-from catalyst import run_algorithm
-from catalyst.api import symbol
-
-
-def initialize(context):
-    print('initializing')
-    context.asset = symbol('btc_usdt')
-
-
-def handle_data(context, data):
-    print('handling bar: {}'.format(data.current_dt))
-
-    price = data.current(context.asset, 'close')
-    print('got price {price}'.format(price=price))
-
-    try:
-        prices = data.history(
-            context.asset,
-            fields='close',
-            bar_count=60,
-            frequency='1D'
-        )
-        print('got {} price entries\n'.format(len(prices), prices))
-    except Exception as e:
-        print(e)
-
-
-run_algorithm(
-    capital_base=1,
-    start=pd.to_datetime('2016-2-11', utc=True),
-    end=pd.to_datetime('2017-8-31', utc=True),
-    data_frequency='daily',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=None,
-    exchange_name='bittrex',
-    algo_namespace='issue_57',
-    base_currency='btc'
-<<<<<<< HEAD
-)
-=======
-)
->>>>>>> develop

catalyst/support/issue_74.py

@@ -1,127 +0,0 @@
-from __future__ import division
-import os
-import pytz
-import numpy as np
-import pandas as pd
-from scipy.optimize import minimize
-import matplotlib.pyplot as plt
-from datetime import datetime
-
-from catalyst.api import record, symbol, symbols, order_target_percent
-from catalyst.utils.run_algo import run_algorithm
-
-np.set_printoptions(threshold='nan', suppress=True)
-
-
-def initialize(context):
-    # Portfolio assets list
-    context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
-                             'xmr_usdt')
-    context.nassets = len(context.assets)
-    # Set the time window that will be used to compute expected return
-    # and asset correlations
-    context.window = 180
-    # Set the number of days between each portfolio rebalancing
-    context.rebalance_period = 30
-    context.i = 0
-
-
-def handle_data(context, data):
-    # Only rebalance at the beggining of the algorithm execution and
-    # every multiple of the rebalance period
-    if context.i == 0 or context.i % context.rebalance_period == 0:
-        n = context.window
-        prices = data.history(context.assets, fields='price',
-                           bar_count=n + 1, frequency='daily')
-        pr = np.asmatrix(prices)
-        t_prices = prices.iloc[1:n + 1]
-        t_val = t_prices.values
-        tminus_prices = prices.iloc[0:n]
-        tminus_val = tminus_prices.values
-        # Compute daily returns (r)
-        r = np.asmatrix(t_val / tminus_val - 1)
-        # Compute the expected returns of each asset with the average
-        # daily return for the selected time window
-        m = np.asmatrix(np.mean(r, axis=0))
-        # ###
-        stds = np.std(r, axis=0)
-        # Compute excess returns matrix (xr)
-        xr = r - m
-        # Matrix algebra to get variance-covariance matrix
-        cov_m = np.dot(np.transpose(xr), xr) / n
-        # Compute asset correlation matrix (informative only)
-        corr_m = cov_m / np.dot(np.transpose(stds), stds)
-
-        # Define portfolio optimization parameters
-        n_portfolios = 50000
-        results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in xrange(n_portfolios):
-            weights = np.random.random(context.nassets)
-            weights /= np.sum(weights)
-            w = np.asmatrix(weights)
-            p_r = np.sum(np.dot(w, np.transpose(m))) * 365
-            p_std = np.sqrt(
-                np.dot(np.dot(w, cov_m), np.transpose(w))) * np.sqrt(365)
-
-            # store results in results array
-            results_array[0, p] = p_r
-            results_array[1, p] = p_std
-            # store Sharpe Ratio (return / volatility) - risk free rate element
-            # excluded for simplicity
-            results_array[2, p] = results_array[0, p] / results_array[1, p]
-            i = 0
-            for iw in weights:
-                results_array[3 + i, p] = weights[i]
-                i += 1
-
-        # convert results array to Pandas DataFrame
-        results_frame = pd.DataFrame(np.transpose(results_array),
-                                     columns=['r', 'stdev',
-                                              'sharpe'] + context.assets)
-        # locate position of portfolio with highest Sharpe Ratio
-        max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
-        # locate positon of portfolio with minimum standard deviation
-        min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
-
-        # order optimal weights for each asset
-        for asset in context.assets:
-            if data.can_trade(asset):
-                order_target_percent(asset, max_sharpe_port[asset])
-
-        # create scatter plot coloured by Sharpe Ratio
-        plt.scatter(results_frame.stdev, results_frame.r,
-                    c=results_frame.sharpe, cmap='RdYlGn')
-        plt.xlabel('Volatility')
-        plt.ylabel('Returns')
-        plt.colorbar()
-        # plot red star to highlight position of portfolio with highest Sharpe Ratio
-        plt.scatter(max_sharpe_port[1], max_sharpe_port[0], marker='o',
-                    color='b', s=200)
-        # plot green star to highlight position of minimum variance portfolio
-        plt.show()
-        print(max_sharpe_port)
-        record(pr=pr, r=r, m=m, stds=stds, max_sharpe_port=max_sharpe_port,
-               corr_m=corr_m)
-    context.i += 1
-
-
-def analyze(context=None, results=None):
-    # Form DataFrame with selected data
-    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
-                    'portfolio_value']]
-
-    # Save results in CSV file
-    filename = os.path.splitext(os.path.basename(__file__))[0]
-    data.to_csv(filename + '.csv')
-
-
-# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
-start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
-end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
-results = run_algorithm(initialize=initialize,
-                        handle_data=handle_data,
-                        analyze=analyze,
-                        start=start,
-                        end=end,
-                        exchange_name='poloniex',
-                        capital_base=100000, )

catalyst/support/rodrigo_1.py

@@ -1,153 +0,0 @@
-import pandas as pd
-from logbook import Logger, DEBUG
-
-from catalyst import run_algorithm
-from catalyst.api import (schedule_function, order_target_percent, symbol,
-                          date_rules, get_open_orders, cancel_order, record,
-                          set_commission, set_slippage)
-
-log = Logger('rodrigo_1', level=DEBUG)
-"""
-The initialize function sets any data or variables that 
-you'll use in your algorithm. 
-It's only called once at the beginning of your algorithm.
-"""
-
-
-def initialize(context):
-    # Select asset of interest
-    context.asset = symbol('BTC_USD')
-
-    # set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5))
-    # set_slippage(TradingPairFixedSlippage(spread=0.5))
-    # Set up a rebalance method to run every day
-    schedule_function(rebalance, date_rule=date_rules.every_day())
-
-
-"""
-Rebalance function scheduled to run once per day.
-"""
-
-
-def rebalance(context, data):
-    # To make market decisions, we're calculating the token's
-    # moving average for the last 5 days.
-
-    # We get the price history for the last 5 days.
-    price_history = data.history(context.asset, fields='price', bar_count=5,
-                                 frequency='1d')
-
-    # Then we take an average of those 5 days.
-    average_price = price_history.mean()
-
-    # We also get the coin's current price.
-    price = data.current(context.asset, 'price')
-
-    # Cancel any outstanding orders
-    orders = get_open_orders(context.asset) or []
-    for order in orders:
-        cancel_order(order)
-
-    # If our coin is currently listed on a major exchange
-    if data.can_trade(context.asset):
-        # If the current price is 1% above the 5-day average price,
-        # we open a long position. If the current price is below the
-        # average price, then we want to close our position to 0 shares.
-        if price > (1.01 * average_price):
-            # Place the buy order (positive means buy, negative means sell)
-            order_target_percent(context.asset, .99)
-            log.info("Buying %s" % (context.asset.symbol))
-        elif price < average_price:
-            # Sell all of our shares by setting the target position to zero
-            order_target_percent(context.asset, 0)
-            log.info("Selling %s" % (context.asset.symbol))
-
-    # Use the record() method to track up to five custom signals.
-    # Record Apple's current price and the average price over the last
-    # five days.
-    cash = context.portfolio.cash
-    leverage = context.account.leverage
-
-    record(price=price, average_price=average_price, cash=cash,
-           leverage=leverage)
-
-
-def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
-
-    # Plot the portfolio and asset data.
-    ax1 = plt.subplot(511)
-    results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value (USD)')
-
-    ax2 = plt.subplot(512, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset))
-    (results[[
-        'price',
-    ]]).plot(ax=ax2)
-
-    trans = results.ix[[t != [] for t in results.transactions]]
-    buys = trans.ix[
-        [t[0]['amount'] > 0 for t in trans.transactions]
-    ]
-    sells = trans.ix[
-        [t[0]['amount'] < 0 for t in trans.transactions]
-    ]
-
-    ax2.plot(
-        buys.index,
-        results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
-    )
-    ax2.plot(
-        sells.index,
-        results.price[sells.index],
-        'v',
-        markersize=10,
-        color='r',
-    )
-
-    ax3 = plt.subplot(513, sharex=ax1)
-    results[['leverage']].plot(ax=ax3)
-    ax3.set_ylabel('Leverage ')
-
-    ax4 = plt.subplot(514, sharex=ax1)
-    results[['cash']].plot(ax=ax4)
-    ax4.set_ylabel('Cash (USD)')
-
-    results[[
-        'algorithm',
-        'benchmark',
-    ]] = results[[
-        'algorithm_period_return',
-        'benchmark_period_return',
-    ]]
-
-    ax5 = plt.subplot(515, sharex=ax1)
-    results[[
-        'algorithm',
-        'benchmark',
-    ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
-
-    plt.legend(loc=3)
-
-    # Show the plot.
-    plt.gcf().set_size_inches(18, 8)
-    plt.show()
-
-
-run_algorithm(
-    capital_base=100000,
-    start=pd.to_datetime('2017-1-1', utc=True),
-    end=pd.to_datetime('2017-10-22', utc=True),
-    data_frequency='minute',
-    initialize=initialize,
-    handle_data=None,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    algo_namespace='rodrigo_1',
-    base_currency='usd'
-)

catalyst/testing/__init__.py

@@ -55,4 +55,4 @@ from .core import (  # noqa
     write_bcolz_minute_data,
     write_compressed,
 )
-from .fixtures import CatalystTestCase  # noqa
+from .fixtures import ZiplineTestCase  # noqa

catalyst/testing/core.py

@@ -643,7 +643,7 @@ def create_data_portal_from_trade_history(asset_finder, trading_calendar,
         return DataPortal(
             asset_finder, trading_calendar,
             first_trading_day=equity_daily_reader.first_trading_day,
-            daily_reader=equity_daily_reader,
+            equity_daily_reader=equity_daily_reader,
         )
     else:
         minutes = trading_calendar.minutes_in_range(

catalyst/testing/fixtures.py

@@ -62,7 +62,7 @@ from catalyst.utils.paths import ensure_directory
 catalyst_dir = os.path.dirname(catalyst.__file__)
 
 
-class CatalystTestCase(with_metaclass(FinalMeta, TestCase)):
+class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)):
     """
     Shared extensions to core unittest.TestCase.
 
@@ -92,7 +92,7 @@ class CatalystTestCase(with_metaclass(FinalMeta, TestCase)):
             cls._base_init_fixtures_was_called = False
             cls.init_class_fixtures()
             assert cls._base_init_fixtures_was_called, (
-                "CatalystTestCase.init_class_fixtures() was not called.\n"
+                "ZiplineTestCase.init_class_fixtures() was not called.\n"
                 "This probably means that you overrode init_class_fixtures"
                 " without calling super()."
             )
@@ -170,7 +170,7 @@ class CatalystTestCase(with_metaclass(FinalMeta, TestCase)):
             self._init_instance_fixtures_was_called = False
             self.init_instance_fixtures()
             assert self._init_instance_fixtures_was_called, (
-                "CatalystTestCase.init_instance_fixtures() was not"
+                "ZiplineTestCase.init_instance_fixtures() was not"
                 " called.\n"
                 "This probably means that you overrode"
                 " init_instance_fixtures without calling super()."
@@ -251,7 +251,7 @@ def alias(attr_name):
 
 class WithDefaultDateBounds(object):
     """
-    CatalystTestCase mixin which makes it possible to synchronize date bounds
+    ZiplineTestCase mixin which makes it possible to synchronize date bounds
     across fixtures.
 
     This fixture should always be the last fixture in bases of any fixture or
@@ -264,13 +264,13 @@ class WithDefaultDateBounds(object):
         The date bounds to be used for fixtures that want to have consistent
         dates.
     """
-    START_DATE = pd.Timestamp('2016-01-03', tz='utc')
+    START_DATE = pd.Timestamp('2006-01-03', tz='utc')
-    END_DATE = pd.Timestamp('2016-12-29', tz='utc')
+    END_DATE = pd.Timestamp('2006-12-29', tz='utc')
 
 
 class WithLogger(object):
     """
-    CatalystTestCase mixin providing cls.log_handler as an instance-level
+    ZiplineTestCase mixin providing cls.log_handler as an instance-level
     fixture.
 
     After init_instance_fixtures has been called `self.log_handler` will be a
@@ -295,7 +295,7 @@ class WithLogger(object):
 
 class WithAssetFinder(WithDefaultDateBounds):
     """
-    CatalystTestCase mixin providing cls.asset_finder as a class-level fixture.
+    ZiplineTestCase mixin providing cls.asset_finder as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.asset_finder` is populated
     with an AssetFinder.
@@ -402,7 +402,7 @@ class WithAssetFinder(WithDefaultDateBounds):
 
 class WithTradingCalendars(object):
     """
-    CatalystTestCase mixin providing cls.trading_calendar,
+    ZiplineTestCase mixin providing cls.trading_calendar,
     cls.all_trading_calendars, cls.trading_calendar_for_asset_type as a
     class-level fixture.
 
@@ -423,7 +423,7 @@ class WithTradingCalendars(object):
         with that asset type.
     """
     TRADING_CALENDAR_STRS = ('NYSE',)
-    TRADING_CALENDAR_FOR_ASSET_TYPE = {Equity: 'NYSE', Future: 'us_futures', }
+    TRADING_CALENDAR_FOR_ASSET_TYPE = {Equity: 'NYSE', Future: 'us_futures'}
     TRADING_CALENDAR_FOR_EXCHANGE = {}
     # For backwards compatibility, exisitng tests and fixtures refer to
     # `trading_calendar` with the assumption that the value is the NYSE
@@ -460,7 +460,7 @@ class WithTradingEnvironment(WithAssetFinder,
                              WithTradingCalendars,
                              WithDefaultDateBounds):
     """
-    CatalystTestCase mixin providing cls.env as a class-level fixture.
+    ZiplineTestCase mixin providing cls.env as a class-level fixture.
 
     After ``init_class_fixtures`` has been called, `cls.env` is populated
     with a trading environment whose `asset_finder` is the result of
@@ -560,7 +560,7 @@ class WithTradingEnvironment(WithAssetFinder,
 
 class WithSimParams(WithTradingEnvironment):
     """
-    CatalystTestCase mixin providing cls.sim_params as a class level fixture.
+    ZiplineTestCase mixin providing cls.sim_params as a class level fixture.
 
     The arguments used to construct the trading environment may be overridded
     by putting ``SIM_PARAMS_{argname}`` in the class dict except for the
@@ -615,7 +615,7 @@ class WithSimParams(WithTradingEnvironment):
 
 class WithTradingSessions(WithTradingCalendars, WithDefaultDateBounds):
     """
-    CatalystTestCase mixin providing cls.trading_days, cls.all_trading_sessions
+    ZiplineTestCase mixin providing cls.trading_days, cls.all_trading_sessions
     as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.all_trading_sessions`
@@ -668,7 +668,7 @@ class WithTradingSessions(WithTradingCalendars, WithDefaultDateBounds):
 
 class WithTmpDir(object):
     """
-    CatalystTestCase mixing providing cls.tmpdir as a class-level fixture.
+    ZiplineTestCase mixing providing cls.tmpdir as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.tmpdir` is populated with
     a `testfixtures.TempDirectory` object whose path is `cls.TMP_DIR_PATH`.
@@ -691,7 +691,7 @@ class WithTmpDir(object):
 
 class WithInstanceTmpDir(object):
     """
-    CatalystTestCase mixing providing self.tmpdir as an instance-level fixture.
+    ZiplineTestCase mixing providing self.tmpdir as an instance-level fixture.
 
     After init_instance_fixtures has been called, `self.tmpdir` is populated
     with a `testfixtures.TempDirectory` object whose path is
@@ -714,7 +714,7 @@ class WithInstanceTmpDir(object):
 
 class WithEquityDailyBarData(WithTradingEnvironment):
     """
-    CatalystTestCase mixin providing cls.make_equity_daily_bar_data.
+    ZiplineTestCase mixin providing cls.make_equity_daily_bar_data.
 
     Attributes
     ----------
@@ -810,7 +810,7 @@ class WithEquityDailyBarData(WithTradingEnvironment):
 
 class WithBcolzEquityDailyBarReader(WithEquityDailyBarData, WithTmpDir):
     """
-    CatalystTestCase mixin providing cls.bcolz_daily_bar_path,
+    ZiplineTestCase mixin providing cls.bcolz_daily_bar_path,
     cls.bcolz_daily_bar_ctable, and cls.bcolz_equity_daily_bar_reader
     class level fixtures.
 
@@ -895,7 +895,7 @@ class WithBcolzEquityDailyBarReader(WithEquityDailyBarData, WithTmpDir):
 
 class WithBcolzEquityDailyBarReaderFromCSVs(WithBcolzEquityDailyBarReader):
     """
-    CatalystTestCase mixin that provides
+    ZiplineTestCase mixin that provides
     cls.bcolz_equity_daily_bar_reader from a mapping of sids to CSV
     file paths.
     """
@@ -925,7 +925,7 @@ class _WithMinuteBarDataBase(WithTradingEnvironment):
 
 class WithEquityMinuteBarData(_WithMinuteBarDataBase):
     """
-    CatalystTestCase mixin providing cls.equity_minute_bar_days.
+    ZiplineTestCase mixin providing cls.equity_minute_bar_days.
 
     After init_class_fixtures has been called:
     - `cls.equity_minute_bar_days` has the range over which data has been
@@ -984,7 +984,7 @@ class WithEquityMinuteBarData(_WithMinuteBarDataBase):
 
 class WithFutureMinuteBarData(_WithMinuteBarDataBase):
     """
-    CatalystTestCase mixin providing cls.future_minute_bar_days.
+    ZiplineTestCase mixin providing cls.future_minute_bar_days.
 
     After init_class_fixtures has been called:
     - `cls.future_minute_bar_days` has the range over which data has been
@@ -1044,7 +1044,7 @@ class WithFutureMinuteBarData(_WithMinuteBarDataBase):
 
 class WithBcolzEquityMinuteBarReader(WithEquityMinuteBarData, WithTmpDir):
     """
-    CatalystTestCase mixin providing cls.bcolz_minute_bar_path,
+    ZiplineTestCase mixin providing cls.bcolz_minute_bar_path,
     cls.bcolz_minute_bar_ctable, and cls.bcolz_equity_minute_bar_reader
     class level fixtures.
 
@@ -1103,7 +1103,7 @@ class WithBcolzEquityMinuteBarReader(WithEquityMinuteBarData, WithTmpDir):
 
 class WithBcolzFutureMinuteBarReader(WithFutureMinuteBarData, WithTmpDir):
     """
-    CatalystTestCase mixin providing cls.bcolz_minute_bar_path,
+    ZiplineTestCase mixin providing cls.bcolz_minute_bar_path,
     cls.bcolz_minute_bar_ctable, and cls.bcolz_equity_minute_bar_reader
     class level fixtures.
 
@@ -1227,7 +1227,7 @@ class WithConstantFutureMinuteBarData(WithFutureMinuteBarData):
 
 class WithAdjustmentReader(WithBcolzEquityDailyBarReader):
     """
-    CatalystTestCase mixin providing cls.adjustment_reader as a class level
+    ZiplineTestCase mixin providing cls.adjustment_reader as a class level
     fixture.
 
     After init_class_fixtures has been called, `cls.adjustment_reader` will be
@@ -1359,7 +1359,7 @@ class WithEquityPricingPipelineEngine(WithAdjustmentReader,
 
 class WithSeededRandomPipelineEngine(WithTradingSessions, WithAssetFinder):
     """
-    CatalystTestCase mixin providing class-level fixtures for running pipelines
+    ZiplineTestCase mixin providing class-level fixtures for running pipelines
     against deterministically-generated random data.
 
     Attributes
@@ -1434,7 +1434,7 @@ class WithDataPortal(WithAdjustmentReader,
                      WithBcolzEquityMinuteBarReader,
                      WithBcolzFutureMinuteBarReader):
     """
-    CatalystTestCase mixin providing self.data_portal as an instance level
+    ZiplineTestCase mixin providing self.data_portal as an instance level
     fixture.
 
     After init_instance_fixtures has been called, `self.data_portal` will be
@@ -1485,12 +1485,12 @@ class WithDataPortal(WithAdjustmentReader,
             self.env.asset_finder,
             self.trading_calendar,
             first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY,
-            daily_reader=(
+            equity_daily_reader=(
                 self.bcolz_equity_daily_bar_reader
                 if self.DATA_PORTAL_USE_DAILY_DATA else
                 None
             ),
-            minute_reader=(
+            equity_minute_reader=(
                 self.bcolz_equity_minute_bar_reader
                 if self.DATA_PORTAL_USE_MINUTE_DATA else
                 None
@@ -1526,7 +1526,7 @@ class WithDataPortal(WithAdjustmentReader,
 
 class WithResponses(object):
     """
-    CatalystTestCase mixin that provides self.responses as an instance
+    ZiplineTestCase mixin that provides self.responses as an instance
     fixture.
 
     After init_instance_fixtures has been called, `self.responses` will be

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,5 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(
+            start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/cli.py

@@ -9,6 +9,7 @@ DEFAULT_BAR_TEMPLATE = '    [%(bar)s]  %(label)s:  %(info)s'
 DEFAULT_EMPTY_CHAR = ' '
 DEFAULT_FILL_CHAR = '='
 
+
 def item_show_count(total=None):
     def maybe_show_total(index):
         if total is not None:
@@ -17,12 +18,13 @@ def item_show_count(total=None):
 
     def item_show_func(item, _it=iter(count())):
         if item is not None:
-            starting = False
+            # starting = False
             return maybe_show_total(next(_it))
         return 'DONE'
 
     return item_show_func
 
+
 def maybe_show_progress(it,
                         show_progress,
                         empty_char=DEFAULT_EMPTY_CHAR,