TST: fixing zipline tests (WIP)

# Conflicts:
#	catalyst/examples/mean_reversion_simple.py

catalyst/__main__.py

@@ -9,6 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_utils import delete_algo_folder
 from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
@@ -29,16 +30,17 @@ except NameError:
 @click.option(
     '--strict-extensions/--non-strict-extensions',
     is_flag=True,
-    help='If --strict-extensions is passed then catalyst will not run if it'
-         ' cannot load all of the specified extensions. If this is not passed or'
-         ' --non-strict-extensions is passed then the failure will be logged but'
-         ' execution will continue.',
+    help='If --strict-extensions is passed then catalyst will not run '
+         'if it cannot load all of the specified extensions. If this is '
+         'not passed or --non-strict-extensions is passed then the '
+         'failure will be logged but execution will continue.',
 )
 @click.option(
     '--default-extension/--no-default-extension',
     is_flag=True,
     default=True,
-    help="Don't load the default catalyst extension.py file in $CATALYST_HOME.",
+    help="Don't load the default catalyst extension.py file "
+         "in $CATALYST_HOME.",
 )
 @click.version_option()
 def main(extension, strict_extensions, default_extension):
@@ -123,9 +125,9 @@ def ipython_only(option):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '--data-frequency',
@@ -137,7 +139,6 @@ def ipython_only(option):
 @click.option(
     '--capital-base',
     type=float,
-    default=10e6,
     show_default=True,
     help='The starting capital for the simulation.',
 )
@@ -175,8 +176,8 @@ def ipython_only(option):
     default='-',
     metavar='FILENAME',
     show_default=True,
-    help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
+    help="The location to write the perf data. If this is '-' the perf"
+         " will be written to stdout.",
 )
 @click.option(
     '--print-algo/--no-print-algo',
@@ -194,7 +195,8 @@ def ipython_only(option):
     '-x',
     '--exchange-name',
     type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange (supported: bitfinex,'
+         ' bittrex, poloniex).',
 )
 @click.option(
     '-n',
@@ -239,16 +241,26 @@ def run(ctx,
         # does not pass either of these and then passes the first only
         # to be told they need to pass the second argument also
         ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'",
+            "must specify dates with '-s' / '--start' and '-e' / '--end'"
+            " in backtest mode",
         )
     if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'")
+        ctx.fail("must specify a start date with '-s' / '--start'"
+                 " in backtest mode")
     if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'")
+        ctx.fail("must specify an end date with '-e' / '--end'"
+                 " in backtest mode")
 
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
+    if base_currency is None:
+        ctx.fail("must specify a base currency with '-c' in backtest mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'"
+                 " in backtest mode")
+
     perf = _run(
         initialize=None,
         handle_data=None,
@@ -334,9 +346,9 @@ def catalyst_magic(line, cell=None):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
 )
 @click.option(
     '-o',
@@ -363,7 +375,8 @@ def catalyst_magic(line, cell=None):
     '-x',
     '--exchange-name',
     type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
-    help='The name of the targeted exchange (supported: bitfinex, bittrex, poloniex).',
+    help='The name of the targeted exchange (supported: bitfinex,'
+         ' bittrex, poloniex).',
 )
 @click.option(
     '-n',
@@ -485,13 +498,31 @@ def live(ctx,
     help='A list of symbols to exclude from the ingestion '
          '(optional comma separated list)',
 )
+@click.option(
+    '--csv',
+    default=None,
+    help='The path of a CSV file containing the data. If specified, start, '
+         'end, include-symbols and exclude-symbols will be ignored. Instead,'
+         'all data in the file will be ingested.',
+)
 @click.option(
     '--show-progress/--no-show-progress',
     default=True,
     help='Print progress information to the terminal.'
 )
+@click.option(
+    '--verbose/--no-verbose`',
+    default=False,
+    help='Show a progress indicator for every currency pair.'
+)
+@click.option(
+    '--validate/--no-validate`',
+    default=False,
+    help='Report potential anomalies found in data bundles.'
+)
 def ingest_exchange(exchange_name, data_frequency, start, end,
-                    include_symbols, exclude_symbols, show_progress):
+                    include_symbols, exclude_symbols, csv, show_progress,
+                    verbose, validate):
     """
     Ingest data for the given exchange.
     """
@@ -499,8 +530,7 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
-    exchange = get_exchange(exchange_name)
-    exchange_bundle = ExchangeBundle(exchange)
+    exchange_bundle = ExchangeBundle(exchange_name)
 
     click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
@@ -509,10 +539,28 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
         exclude_symbols=exclude_symbols,
         start=start,
         end=end,
-        show_progress=show_progress
+        show_progress=show_progress,
+        show_breakdown=verbose,
+        show_report=validate,
+        csv=csv
     )
 
 
+@main.command(name='clean-algo')
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='The label of the algorithm to for which to clean the state.'
+)
+@click.pass_context
+def clean_algo(ctx, algo_namespace):
+    click.echo(
+        'Cleaning algo state: {}'.format(algo_namespace)
+    )
+    delete_algo_folder(algo_namespace)
+    click.echo('Done')
+
+
 @main.command(name='clean-exchange')
 @click.option(
     '-x',
@@ -537,8 +585,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
     if exchange_name is None:
         ctx.fail("must specify an exchange name '-x'")
 
-    exchange = get_exchange(exchange_name)
-    exchange_bundle = ExchangeBundle(exchange)
+    exchange_bundle = ExchangeBundle(exchange_name)
 
     click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
     exchange_bundle.clean(

catalyst/assets/_assets.pyx

@@ -38,6 +38,7 @@ from numpy cimport int64_t
 import warnings
 cimport numpy as np
 
+from catalyst.exchange.exchange_utils import get_sid
 from catalyst.utils.calendars import get_calendar
 from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
@@ -395,11 +396,15 @@ cdef class Future(Asset):
 
 cdef class TradingPair(Asset):
     cdef readonly float leverage
-    cdef readonly object market_currency
+    cdef readonly object quote_currency
     cdef readonly object base_currency
     cdef readonly object end_daily
     cdef readonly object end_minute
     cdef readonly object exchange_symbol
+    cdef readonly float maker
+    cdef readonly float taker
+    cdef readonly int trading_state
+    cdef readonly object data_source
 
     _kwargnames = frozenset({
         'sid',
@@ -412,12 +417,16 @@ cdef class TradingPair(Asset):
         'exchange',
         'exchange_full',
         'leverage',
-        'market_currency',
+        'quote_currency',
         'base_currency',
         'end_daily',
         'end_minute',
         'exchange_symbol',
-        'min_trade_size'
+        'min_trade_size',
+        'maker',
+        'taker',
+        'trading_state',
+        'data_source'
     })
     def __init__(self,
                  object symbol,
@@ -433,10 +442,14 @@ cdef class TradingPair(Asset):
                  object first_traded=None,
                  object auto_close_date=None,
                  object exchange_full=None,
-                 object min_trade_size=None):
+                 float min_trade_size=0.0001,
+                 float maker=0.0015,
+                 float taker=0.0025,
+                 int trading_state=0,
+                 object data_source='catalyst'):
         """
         Replicates the Asset constructor with some built-in conventions
-        and a new 'leverage' attribute.
+        and adds properties for leverage and fees.
 
         Symbol
         ------
@@ -468,8 +481,6 @@ cdef class TradingPair(Asset):
         highest volume and market cap generally benefit from high leverage.
         New currencies from ICO generally cannot be leveraged.
 
-        The leverage value is either None or and integer.
-
         Leverage allows you to open a larger position with a smaller amount
         of funds. For example, if you open a $5,000 position in BTC/USD
         with 5:1 leverage, only one-fifth of this amount, or $1000, will be
@@ -479,6 +490,11 @@ cdef class TradingPair(Asset):
         the position. If you open with 1:1 leverage, $5,000 of your balance
         will be tied to the position.
 
+        Fees
+        ----
+        Exchanges generally charge a taker (taking from the order book) or
+        maker (adding to the order book) fee.
+
         :param symbol:
         :param exchange:
         :param start_date:
@@ -493,21 +509,20 @@ cdef class TradingPair(Asset):
         :param auto_close_date:
         :param exchange_full:
         :param min_trade_size:
+        :param maker:
+        :param taker:
+        :param data_source
         """
 
         symbol = symbol.lower()
         try:
-            self.market_currency, self.base_currency = symbol.split('_')
+            self.base_currency, self.quote_currency = symbol.split('_')
         except Exception as e:
             raise InvalidSymbolError(symbol=symbol, error=e)
 
         if sid == 0 or sid is None:
             try:
-                # sid = abs(hash(symbol)) % (10 ** 4)
-                # TODO: try to encode the symbol in the main scope
-                sid = int(
-                    hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
-                ) % 10 ** 6
+                sid = get_sid(symbol)
             except Exception as e:
                 raise SidHashError(symbol=symbol)
 
@@ -515,7 +530,7 @@ cdef class TradingPair(Asset):
             asset_name = ' / '.join(symbol.split('_')).upper()
 
         if start_date is None:
-            start_date = pd.Timestamp.utcnow()
+            start_date = pd.to_datetime('2009-1-1', utc=True)
 
         if end_date is None:
             end_date = pd.Timestamp.utcnow() + timedelta(days=365)
@@ -530,19 +545,23 @@ cdef class TradingPair(Asset):
             first_traded=first_traded,
             auto_close_date=auto_close_date,
             exchange_full=exchange_full,
-            min_trade_size=min_trade_size
+            min_trade_size=min_trade_size,
         )
 
+        self.maker = maker
+        self.taker = taker
         self.leverage = leverage
         self.end_daily = end_daily
         self.end_minute = end_minute
         self.exchange_symbol = exchange_symbol
+        self.trading_state = trading_state
+        self.data_source = data_source
 
     def __repr__(self):
         return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
                'Introduced On: {start_date}, ' \
-               'Market Currency: {market_currency}, ' \
                'Base Currency: {base_currency}, ' \
+               'Quote Currency: {quote_currency}, ' \
                'Exchange Leverage: {leverage}, ' \
                'Minimum Trade Size: {min_trade_size} ' \
                'Last daily ingestion: {end_daily} ' \
@@ -551,7 +570,7 @@ cdef class TradingPair(Asset):
             sid=self.sid,
             exchange=self.exchange,
             start_date=self.start_date,
-            market_currency=self.market_currency,
+            quote_currency=self.quote_currency,
             base_currency=self.base_currency,
             leverage=self.leverage,
             min_trade_size=self.min_trade_size,
@@ -559,6 +578,17 @@ cdef class TradingPair(Asset):
             end_minute=self.end_minute
         )
 
+    cpdef to_dict(self):
+        """
+        Convert to a python dict.
+        """
+        super_dict = super(TradingPair, self).to_dict()
+        super_dict['end_daily'] = self.end_daily
+        super_dict['end_minute'] = self.end_minute
+        super_dict['leverage'] = self.leverage
+        super_dict['min_trade_size'] = self.min_trade_size
+        return super_dict
+
     def is_exchange_open(self, dt_minute):
         """
         Parameters
@@ -570,7 +600,7 @@ cdef class TradingPair(Asset):
         -------
         boolean: whether the asset's exchange is open at the given minute.
         """
-        #TODO: consider implementing to spot holds
+        #TODO: make more dymanic to catch holds
         return True
 
     cpdef __reduce__(self):

catalyst/constants.py

@@ -1,5 +1,19 @@
 # -*- coding: utf-8 -*-
 
+import os
 import logbook
 
-LOG_LEVEL = logbook.INFO
+''' You can override the LOG level from your environment.
+    For example, if you want to see the DEBUG messages, run:
+    $ export CATALYST_LOG_LEVEL=10
+'''
+# LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
+LOG_LEVEL = logbook.DEBUG
+
+SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
+              '{exchange}/symbols.json'
+
+DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+DATE_FORMAT = '%Y-%m-%d'
+
+AUTO_INGEST = False

catalyst/curate/poloniex.py

@@ -6,9 +6,8 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 
 DT_START        = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END          = int(time.time())
-CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
-CSV_OUT_FOLDER  = '/Volumes/enigma/data/poloniex/'
+DT_END          = pd.to_datetime('today').value // 10 ** 9
+CSV_OUT_FOLDER  = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
 CONN_RETRIES    = 2
 
 logbook.StderrHandler().push_application()
@@ -27,13 +26,15 @@ class PoloniexCurator(object):
             try:
                 os.makedirs(CSV_OUT_FOLDER)
             except Exception as e:
-                log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
+                log.error('Failed to create data folder: {}'.format(
+                            CSV_OUT_FOLDER))
                 log.exception(e)
 
-    '''
-        Retrieves and returns all currency pairs from the exchange 
-    '''
+
     def get_currency_pairs(self):
+        '''
+        Retrieves and returns all currency pairs from the exchange 
+        '''
         url = self._api_path + 'command=returnTicker'
 
         try:
@@ -49,89 +50,136 @@ class PoloniexCurator(object):
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
 
-        log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
+        log.debug('Currency pairs retrieved successfully: {}'.format(
+                    len(self.currency_pairs)
+                    ))
+
 
 
-    '''
-        Helper function that reads tradeID and date fields from CSV readline
-    '''
     def _retrieve_tradeID_date(self, row):
+        '''
+        Helper function that reads tradeID and date fields from CSV readline
+        '''
         tId = int(row.split(',')[0])
-        d   = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9
+        d   = pd.to_datetime(row.split(',')[1], 
+                             infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
-    '''
-        Retrieves TradeHistory from exchange for a given currencyPair between start and end dates.
-        If no start date is provided, uses a system-wide one (beginning of time for cryptotrading)
-        If no end date is provided, 'now' is used
+
+    def retrieve_trade_history(self, currencyPair, start=DT_START, 
+                               end=DT_END, temp=None):
+        '''
+        Retrieves TradeHistory from exchange for a given currencyPair 
+        between start and end dates. If no start date is provided, uses 
+        a system-wide one (beginning of time for cryptotrading).
+        If no end date is provided, 'now' is used.
+
         Stores results in CSV file on disk.
-        This function is called recursively to work around the limitations imposed by the provider API.
-    '''
-    def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None):
+        
+        This function is called recursively to work around the 
+        limitations imposed by the provider API.
+        '''
         csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
 
         '''
-            Check what data we already have on disk, reading first and last lines from file.
-            Data is stored on file from NEWEST to OLDEST.
+        Check what data we already have on disk, reading first and last 
+        lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
             with open(csv_fn, 'ab+') as f: 
                 f.seek(0, os.SEEK_END)
-                if(f.tell() > 2):                           # First check file is not zero size
-                    f.seek(0)                               # Go to the beginning to read first line
-                    last_tradeID, end_file    = self._retrieve_tradeID_date(f.readline())
-                    f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                    while f.read(1) != b"\n":               # Until EOL is found...
-                        f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
+                if(f.tell() > 2):                   # Check file size is not 0
+                    f.seek(0)                       # Go to start to read 
+                    last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
+                    f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
+                    while f.read(1) != b"\n":       # Until EOL is found...
+                        f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
                     first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
 
-                    if( first_tradeID == 1 and end_file + 3600 > DT_END ):
+                    if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
+                        or (currencyPair == 'BTC_HUC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_RIC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_XCP' and first_tradeID ==     2)
+                        or (currencyPair == 'BTC_NAV' and first_tradeID ==  4569)
+                        or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
                         return
 
         except Exception as e:
-            log.error('Error opening file: %s' % csv_fn)
+            log.error('Error opening file: {}'.format(csv_fn))
             log.exception(e)
 
         '''
-        Poloniex API limits querying TradeHistory to intervals smaller than 1 month,
-        so we make sure that start date is never more than 1 month apart from end date
+        Poloniex API limits querying TradeHistory to intervals smaller 
+        than 1 month, so we make sure that start date is never more than
+        1 month apart from end date
         '''
-        if( end - start > 2419200 ):       # 60 s/min * 60 min/hr * 24 hr/day * 28 days     
+        if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days     
             newstart = end - 2419200
         else:
             newstart = start
 
-        log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t '
-                    + time.ctime(newstart) + ' - '+ time.ctime(end))
+        log.debug('{}: Retrieving from {} to {}\t {} - {}'.format(
+                    currencyPair, str(newstart), str(end),
+                    time.ctime(newstart), time.ctime(end)))
 
-        url = self._api_path + 'command=returnTradeHistory&currencyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end)
+        url = '{path}command=returnTradeHistory&currencyPair={pair}' \
+              '&start={start}&end={end}'.format(
+                    path = self._api_path,
+                    pair = currencyPair,
+                    start = str(newstart),
+                    end = str(end)
+                )
+        print url
 
-        try:
-            response = requests.get(url)
-        except Exception as e:
-            log.error('Failed to retrieve trade history data for %s' % currencyPair)
-            log.exception(e)
+        attempts = 0
+        success = 0
+        while attempts < CONN_RETRIES:
+            try:
+                response = requests.get(url)
+            except Exception as e:
+                log.error('Failed to retrieve trade history data for {}'.format(
+                            currencyPair
+                            ))
+                log.exception(e)
+                attempts += 1
+            else:
+                try:
+                    if isinstance(response.json(), dict) and response.json()['error']:
+                        log.error('Failed to to retrieve trade history data '
+                                  'for {}: {}'.format(
+                                    currencyPair,
+                                    response.json()['error']
+                                    ))
+                        attempts += 1
+                except Exception as e:
+                    log.exception(e)
+                    attempts += 1
+                else:
+                    success = 1
+                    break
+
+        if not success:
             return None
-        else:
-            if isinstance(response.json(), dict) and response.json()['error']:
-                log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error']))
-                exit(1)
+
 
         '''
-            If we get to transactionId == 1, and we already have that on disk, 
-            we got to the end of TradeHistory for this coin.
+        If we get to transactionId == 1, and we already have that on 
+        disk, we got to the end of TradeHistory for this coin.
         '''
-        if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID): 
+        if('first_tradeID' in locals() 
+                and response.json()[-1]['tradeID'] == first_tradeID): 
             return
 
         '''
             There are primarily two scenarios:
-                a) There is newer data available that we need to add at the beginning
-                   of the file. We'll retrieve all what we need until we get to what 
-                   we already have, writing it to a temporary file; and we will write
-                   that at the beginning of our existing file.
-                b) We are going back in time, appending at the end of our existing
-                   TradeHistory until the first transaction for this currencyPair
+                a) There is newer data available that we need to add at 
+                   the beginning of the file. We'll retrieve all what we 
+                   need until we get to what we already have, writing it 
+                   to a temporary file; and we will write that at the 
+                   beginning of our existing file.
+                b) We are going back in time, appending at the end of 
+                   our existing TradeHistory until the first transaction 
+                   for this currencyPair
         '''
         try: 
             if( 'end_file' in locals() and end_file + 3600 < end):
@@ -151,8 +199,10 @@ class PoloniexCurator(object):
                         item['globalTradeID']        
                     ])
                 if( response.json()[-1]['tradeID'] > last_tradeID ):
-                    end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
-                    self.retrieve_trade_history(currencyPair, start, end, temp=temp) 
+                    end = pd.to_datetime( response.json()[-1]['date'], 
+                                infer_datetime_format=True).value // 10 ** 9
+                    self.retrieve_trade_history(currencyPair, start, 
+                                                end, temp=temp) 
                 else:
                     with open(csv_fn,'rb+') as f:
                         shutil.copyfileobj(f,temp)
@@ -165,7 +215,8 @@ class PoloniexCurator(object):
                 with open(csv_fn, 'ab') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in response.json():
-                        if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ):
+                        if( 'first_tradeID' in locals() 
+                                and item['tradeID'] >= first_tradeID ):
                             continue
                         csvwriter.writerow([
                             item['tradeID'],
@@ -176,84 +227,112 @@ class PoloniexCurator(object):
                             item['total'],
                             item['globalTradeID']
                         ])
-                end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
+                end = pd.to_datetime(response.json()[-1]['date'], 
+                            infer_datetime_format=True).value // 10 ** 9
 
         except Exception as e:
-            log.error('Error opening %s' % csv_fn)
+            log.error('Error opening {}'.format(csv_fn))
             log.exception(e)
 
         '''
-            If we got here, we aren't done yet. Call recursively with 'end' times
-            that go sequentially back in time.
+            If we got here, we aren't done yet. Call recursively with 
+            'end' times that go sequentially back in time.
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
 
-    '''
+
+    def generate_ohlcv(self, df):
+        '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
         by resampling with 1-minute period
-    '''
-    def generate_ohlcv(self, df):
-        df.set_index('date', inplace=True)                      # Index by date
-        vol = df['total'].to_frame('volume')                    # Will deal with vol separately, as ohlc() messes it up
-        df.drop('total', axis=1, inplace=True)                  # Drop volume data from dataframe
-        ohlc = df.resample('T').ohlc()                          # Resample OHLC in 1min bins
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])         # Raname columns by dropping 'rate'
-        closes = ohlc['close'].fillna(method='pad')             # Pad forward missing 'close'
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))           # Fill N/A with last close
-        vol = vol.resample('T').sum().fillna(0)                 # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)                   # Concatenate OHLC + Volume
+        '''
+        df.set_index('date', inplace=True)               # Index by date
+        vol = df['total'].to_frame('volume')             # set Vol aside
+        df.drop('total', axis=1, inplace=True)           # Drop volume data 
+        ohlc = df.resample('T').ohlc()                   # Resample OHLC 1min
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Raname columns by dropping 'rate'
+        closes = ohlc['close'].fillna(method='pad')      # Pad fwd missing 'close'
+        ohlc = ohlc.apply(lambda x: x.fillna(closes))    # Fill N/A with last close
+        vol = vol.resample('T').sum().fillna(0)          # Add volumes by bin
+        ohlcv = pd.concat([ohlc,vol], axis=1)            # Concatenate OHLC + Vol
         return ohlcv
 
 
-    '''
+
+    def write_ohlcv_file(self, currencyPair): 
+        '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
-    '''
-    def write_ohlcv_file(self, currencyPair):        
+        '''       
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
         csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        #if( os.path.isfile(csv_1min) ):
-        #    log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
-        #else:
-        df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
-                dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
-        df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
-        df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
-        ohlcv = self.generate_ohlcv(df)
-        try: 
-            with open(csv_1min, 'w') as csvfile:
-                csvwriter = csv.writer(csvfile)
-                for item in ohlcv.itertuples():
-                    if item.Index == 0:
-                        continue
-                    csvwriter.writerow([
-                        item.Index.value // 10 ** 9,
-                        item.open,
-                        item.high,
-                        item.low,
-                        item.close,
-                        item.volume,
-                    ])
-        except Exception as e:
-            log.error('Error opening %s' % csv_fn)
-            log.exception(e)
-        log.debug(currencyPair+': Generated 1min OHLCV data.')
+        if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
+            log.debug(currencyPair+': 1min data file already up to date. '
+                      'Delete the file if you want to rebuild it.')
+        else:
+            df = pd.read_csv(csv_trades, 
+                                names=['tradeID',
+                                       'date',
+                                       'type',
+                                       'rate',
+                                       'amount',
+                                       'total',
+                                       'globalTradeID'], 
+                                dtype = {'tradeID': int, 
+                                         'date': str, 
+                                         'type': str, 
+                                         'rate': float, 
+                                         'amount': float, 
+                                         'total': float, 
+                                         'globalTradeID': int } 
+                                )
+            df.drop(['tradeID','type','amount','globalTradeID'], 
+                    axis=1, inplace=True)
+            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
+            ohlcv = self.generate_ohlcv(df)
+            try: 
+                with open(csv_1min, 'w') as csvfile:
+                    csvwriter = csv.writer(csvfile)
+                    for item in ohlcv.itertuples():
+                        if item.Index == 0:
+                            continue
+                        csvwriter.writerow([
+                            item.Index.value // 10 ** 9,
+                            item.open,
+                            item.high,
+                            item.low,
+                            item.close,
+                            item.volume,
+                        ])
+            except Exception as e:
+                log.error('Error opening {}'.format(csv_fn))
+                log.exception(e)
+            log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
+
 
 
-    '''
-    Returns a data frame for a given currencyPair from data on disk
-    '''
     def onemin_to_dataframe(self, currencyPair, start, end):
+        '''
+        Returns a data frame for a given currencyPair from data on disk
+        '''
         csv_fn     = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df         = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
+        df         = pd.read_csv(csv_fn, names=['date', 
+                                                'open', 
+                                                'high', 
+                                                'low', 
+                                                'close', 
+                                                'volume']
+                                )
         df['date'] = pd.to_datetime(df['date'],unit='s')
         df.set_index('date', inplace=True)
         return df[start : end]
 
-    '''
-    Generates a symbols.json file with corresponding start_date for each currencyPair
-    '''
+
     def generate_symbols_json(self, filename=None):
+        '''
+        Generates a symbols.json file with corresponding start_date 
+        for each currencyPair
+        '''
         symbol_map = {}
 
         if(filename is None):
@@ -262,14 +341,16 @@ class PoloniexCurator(object):
         with open(filename, 'w') as symbols:
             for currencyPair in self.currency_pairs:
                 start = None
-                csv_fn     = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
+                csv_fn     = '{}crypto_trades-{}.csv'.format(
+                                CSV_OUT_FOLDER, currencyPair)
                 with open(csv_fn, 'r') as f: 
                     f.seek(0, os.SEEK_END)
-                    if(f.tell() > 2):                           # First check file is not zero size
-                        f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                        while f.read(1) != b"\n":               # Until EOL is found...
-                            f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
-                        start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True)
+                    if(f.tell() > 2):               # Check file size is not 0
+                        f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
+                        while f.read(1) != b"\n":   # Until EOL is found...
+                            f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
+                        start = pd.to_datetime( f.readline().split(',')[1], 
+                                                infer_datetime_format=True)
 
                 if(start is None):
                     start = time.gmtime()
@@ -279,7 +360,8 @@ class PoloniexCurator(object):
                     symbol = symbol,
                     start_date = start.strftime("%Y-%m-%d")
                 )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':'))    
+            json.dump(symbol_map, symbols, sort_keys=True, indent=2, 
+                        separators=(',',':'))    
 
 
 if __name__ == '__main__':
@@ -289,6 +371,6 @@ if __name__ == '__main__':
     
     for currencyPair in pc.currency_pairs:
         pc.retrieve_trade_history(currencyPair)
+        log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
 
-    

catalyst/data/loader.py

@@ -142,20 +142,23 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
         #  breaks things and it's only needed here
-        from catalyst.exchange.poloniex.poloniex import Poloniex
-        exchange = Poloniex('', '', '')
+        from catalyst.exchange.factory import get_exchange
+        exchange = get_exchange(
+            exchange_name='poloniex', base_currency='usdt'
+        )
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 
     # exchange.get_history_window() already ensures that we have the right data
     # for the right dates
-    br = exchange.get_history_window(
+    br = exchange.get_history_window_with_bundle(
         assets=[benchmark_asset],
         end_dt=last_date,
         bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
-        data_frequency='daily')
+        data_frequency='daily',
+        force_auto_ingest=True)
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
     br.loc[start_dt] = 0

catalyst/examples/README.rst

@@ -0,0 +1,3 @@
+An overview of most of the trading strategies in this folder can be found in the 
+`Examples Algorithms <https://enigmampc.github.io/catalyst/example-algos.html>`_
+section of our documentation website.

catalyst/examples/arbitrage_with_interface.py

@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
     else:
         raise ValueError('invalid order action')
 
-    base_currency = enter_exchange.base_currency
-    base_currency_amount = enter_exchange.portfolio.cash
+    quote_currency = enter_exchange.quote_currency
+    quote_currency_amount = enter_exchange.portfolio.cash
 
     exit_balances = exit_exchange.get_balances()
     exit_currency = context.trading_pairs[
-        context.selling_exchange].market_currency
+        context.selling_exchange].quote_currency
 
     if exit_currency in exit_balances:
-        market_currency_amount = exit_balances[exit_currency]
+        quote_currency_amount = exit_balances[exit_currency]
     else:
         log.warn(
             'the selling exchange {exchange_name} does not hold '
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
         )
         return
 
-    if base_currency_amount < (amount * entry_price):
-        adj_amount = base_currency_amount / entry_price
+    if quote_currency_amount < (amount * entry_price):
+        adj_amount = quote_currency_amount / entry_price
         log.warn(
-            'not enough {base_currency} ({base_currency_amount}) to buy '
+            'not enough {quote_currency} ({quote_currency_amount}) to buy '
             '{amount}, adjusting the amount to {adj_amount}'.format(
-                base_currency=base_currency,
-                base_currency_amount=base_currency_amount,
+                quote_currency=quote_currency,
+                quote_currency_amount=quote_currency_amount,
                 amount=amount,
                 adj_amount=adj_amount
             )
         )
         amount = adj_amount
 
-    elif market_currency_amount < amount:
+    elif quote_currency_amount < amount:
         log.warn(
             'not enough {currency} ({currency_amount}) to sell '
             '{amount}, aborting'.format(
                 currency=exit_currency,
-                currency_amount=market_currency_amount,
+                currency_amount=quote_currency_amount,
                 amount=amount
             )
         )
@@ -270,6 +270,6 @@ run_algorithm(
     exchange_name='poloniex,bitfinex',
     live=True,
     algo_namespace=algo_namespace,
-    base_currency='btc',
+    quote_currency='btc',
     live_graph=False
 )

catalyst/examples/buy_and_hodl.py

@@ -14,30 +14,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (order_target_value, symbol, record,
+    cancel_order, get_open_orders, )
 
-from catalyst.api import (
-    order_target_value,
-    symbol,
-    record,
-    cancel_order,
-    get_open_orders,
-)
 
 def initialize(context):
-    context.ASSET_NAME = 'BTC_USDT'
+    context.ASSET_NAME = 'btc_usd'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
     context.is_buying = True
     context.asset = symbol(context.ASSET_NAME)
 
     context.i = 0
 
+
 def handle_data(context, data):
     context.i += 1
 
@@ -60,12 +55,13 @@ def handle_data(context, data):
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
+        print('buying')
         # Place order to make position in asset equal to target_hodl_value
         order_target_value(
             context.asset,
             target_hodl_value,
-            limit_price=price*1.1,
-            stop_price=price*0.9,
+            limit_price=price * 1.1,
+            stop_price=price * 0.9,
         )
 
     record(
@@ -76,28 +72,29 @@ def handle_data(context, data):
         leverage=context.account.leverage,
     )
 
+
 def analyze(context=None, results=None):
-    import matplotlib.pyplot as plt
 
     # Plot the portfolio and asset data.
     ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
-    ax1.set_ylabel('Portfolio Value (USD)')
+    ax1.set_ylabel('Portfolio\nValue\n(USD)')
 
     ax2 = plt.subplot(612, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
-    (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
+    ax2.set_ylabel('{asset}\n(USD)'.format(asset=context.ASSET_NAME))
+    results[['price']].plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     buys = trans.ix[
         [t[0]['amount'] > 0 for t in trans.transactions]
     ]
-    ax2.plot(
-        buys.index,
-        context.TICK_SIZE * results.price[buys.index],
-        '^',
-        markersize=10,
-        color='g',
+    ax2.scatter(
+        buys.index.to_pydatetime(),
+        results.price[buys.index],
+        marker='^',
+        s=100,
+        c='g',
+        label=''
     )
 
     ax3 = plt.subplot(613, sharex=ax1)
@@ -124,14 +121,29 @@ def analyze(context=None, results=None):
         'algorithm',
         'benchmark',
     ]].plot(ax=ax5)
-    ax5.set_ylabel('Percent Change')
+    ax5.set_ylabel('Percent\nChange')
 
     ax6 = plt.subplot(616, sharex=ax1)
     results[['volume']].plot(ax=ax6)
-    ax6.set_ylabel('Volume (mCoins/5min)')
+    ax6.set_ylabel('Volume')
 
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
-    plt.show()
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='bitfinex',
+        algo_namespace='buy_and_hodl',
+        base_currency='usd',
+        start=pd.to_datetime('2015-03-01', utc=True),
+        end=pd.to_datetime('2017-10-31', utc=True),
+    )

catalyst/examples/buy_btc_simple.py

@@ -3,7 +3,8 @@
 	https://enigmampc.github.io/catalyst/beginner-tutorial.html
 
 	Run this example, by executing the following from your terminal:
-	catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+	  catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
+	  catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
 
 	If you want to run this code using another exchange, make sure that 
 	the asset is available on that exchange. For example, if you were to run 
@@ -12,7 +13,7 @@
 		context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
 
 	and specify exchange poloniex as follows:
-
+	catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
 	catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
 
 	To see which assets are available on each exchange, visit:

catalyst/examples/dual_moving_average.py

@@ -0,0 +1,153 @@
+import numpy as np
+import pandas as pd
+from logbook import Logger
+import matplotlib.pyplot as plt
+
+from catalyst import run_algorithm
+from catalyst.api import (order, record, symbol, order_target_percent,
+        get_open_orders)
+from catalyst.exchange.stats_utils import extract_transactions
+
+NAMESPACE = 'dual_moving_average'
+log = Logger(NAMESPACE)
+
+def initialize(context):
+    context.i = 0
+    context.asset = symbol('ltc_usd')
+    context.base_price = None
+
+
+def handle_data(context, data):
+    # define the windows for the moving averages
+    short_window = 50
+    long_window = 200
+
+    # Skip as many bars as long_window to properly compute the average
+    context.i += 1
+    if context.i < long_window:
+       return
+
+    # Compute moving averages calling data.history() for each
+    # moving average with the appropriate parameters. We choose to use
+    # minute bars for this simulation -> freq="1m"
+    # Returns a pandas dataframe.
+    short_mavg = data.history(context.asset, 'price',
+                        bar_count=short_window, frequency="1m").mean()
+    long_mavg = data.history(context.asset, 'price',
+                        bar_count=long_window, frequency="1m").mean()
+
+    # Let's keep the price of our asset in a more handy variable
+    price = data.current(context.asset, 'price')
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+    price_change = (price - context.base_price) / context.base_price
+
+    # Save values for later inspection
+    record(price=price,
+           cash=context.portfolio.cash,
+           price_change=price_change,
+           short_mavg=short_mavg,
+           long_mavg=long_mavg)
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.asset):
+        return
+
+    # We check what's our position on our portfolio and trade accordingly
+    pos_amount = context.portfolio.positions[context.asset].amount
+
+    # Trading logic
+    if short_mavg > long_mavg and pos_amount == 0:
+       # we buy 100% of our portfolio for this asset
+       order_target_percent(context.asset, 1)
+    elif short_mavg < long_mavg and pos_amount > 0:
+       # we sell all our positions for this asset
+       order_target_percent(context.asset, 0)
+
+
+def analyze(context, perf):
+
+    # Get the base_currency that was passed as a parameter to the simulation
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # First chart: Plot portfolio value using base_currency
+    ax1 = plt.subplot(411)
+    perf.loc[:, ['portfolio_value']].plot(ax=ax1)
+    ax1.legend_.remove()
+    ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
+    start, end = ax1.get_ylim()
+    ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Second chart: Plot asset price, moving averages and buys/sells
+    ax2 = plt.subplot(412, sharex=ax1)
+    perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
+    ax2.legend_.remove()
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset = context.asset.symbol,
+        base = base_currency
+        ))
+    start, end = ax2.get_ylim()
+    ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    # Third chart: Compare percentage change between our portfolio
+    # and the price of the asset
+    ax3 = plt.subplot(413, sharex=ax1)
+    perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
+    ax3.legend_.remove()
+    ax3.set_ylabel('Percent Change')
+    start, end = ax3.get_ylim()
+    ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+    # Fourth chart: Plot our cash
+    ax4 = plt.subplot(414, sharex=ax1)
+    perf.cash.plot(ax=ax4)
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+    start, end = ax4.get_ylim()
+    ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+
+    plt.show()
+
+
+if __name__ == '__main__':
+    run_algorithm(
+            capital_base=1000,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='usd',
+            start=pd.to_datetime('2017-9-22', utc=True),
+            end=pd.to_datetime('2017-9-23', utc=True),
+        )

catalyst/examples/mean_reversion_simple.py

@@ -0,0 +1,279 @@
+# For this example, we're going to write a simple momentum script.  When the 
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.neo_eth = symbol('neo_eth')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 55
+    context.RSI_OVERBOUGHT = 82
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.neo_eth variable.  For this example, we're 
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.neo_eth,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.neo_eth, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        volume=current['volume'],
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.neo_eth)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.neo_eth):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.   
+    pos_amount = context.portfolio.positions[context.neo_eth].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.neo_eth, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.neo_eth, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.neo_eth.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end/5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'live'
+
+    if MODE == 'backtest':
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))
+
+    elif MODE == 'live':
+        run_algorithm(
+            capital_base=0.1,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bittrex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            live_graph=False
+        )

catalyst/examples/portfolio_optimization.py

@@ -0,0 +1,133 @@
+'''Use this code to execute a portfolio optimization model. This code 
+   will select the portfolio with the maximum Sharpe Ratio. The parameters 
+   are set to use 180 days of historical data and rebalance every 30 days.
+   
+   This is the code used in the following article:
+   https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
+
+   You can run this code using the Python interpreter:
+
+   $ python portfolio_optimization.py
+'''
+
+from __future__ import division
+import os
+import pytz
+import numpy as np
+import pandas as pd
+from scipy.optimize import minimize
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+from catalyst.api import record, symbol, symbols, order_target_percent
+from catalyst.utils.run_algo import run_algorithm
+
+np.set_printoptions(threshold='nan', suppress=True)
+
+
+def initialize(context):
+   # Portfolio assets list
+   context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
+                            'xmr_usdt')
+   context.nassets = len(context.assets)
+   # Set the time window that will be used to compute expected return 
+   # and asset correlations
+   context.window = 180
+   # Set the number of days between each portfolio rebalancing
+   context.rebalance_period = 30                   
+   context.i = 0
+
+   
+def handle_data(context, data):
+   # Only rebalance at the beggining of the algorithm execution and 
+   # every multiple of the rebalance period
+   if context.i == 0 or context.i%context.rebalance_period == 0:
+       n = context.window
+       prices = data.history(context.assets, fields='price', 
+                             bar_count=n+1, frequency='1d') 
+       pr = np.asmatrix(prices)
+       t_prices = prices.iloc[1:n+1]
+       t_val = t_prices.values
+       tminus_prices = prices.iloc[0:n]
+       tminus_val = tminus_prices.values
+       # Compute daily returns (r)
+       r = np.asmatrix(t_val/tminus_val-1)
+       # Compute the expected returns of each asset with the average 
+       # daily return for the selected time window
+       m = np.asmatrix(np.mean(r, axis=0))
+       # ###
+       stds = np.std(r, axis=0)
+       # Compute excess returns matrix (xr)
+       xr = r - m
+       # Matrix algebra to get variance-covariance matrix
+       cov_m = np.dot(np.transpose(xr),xr)/n
+       # Compute asset correlation matrix (informative only)
+       corr_m = cov_m/np.dot(np.transpose(stds),stds)
+       
+       # Define portfolio optimization parameters
+       n_portfolios = 50000
+       results_array = np.zeros((3+context.nassets,n_portfolios))
+       for p in xrange(n_portfolios):
+           weights = np.random.random(context.nassets)
+           weights /= np.sum(weights)
+           w = np.asmatrix(weights)
+           p_r = np.sum(np.dot(w,np.transpose(m)))*365
+           p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
+           
+           #store results in results array
+           results_array[0,p] = p_r
+           results_array[1,p] = p_std
+           #store Sharpe Ratio (return / volatility) - risk free rate element 
+           #excluded for simplicity
+           results_array[2,p] = results_array[0,p] / results_array[1,p]
+           i = 0
+           for iw in weights:
+               results_array[3+i,p] = weights[i]
+               i += 1
+       
+       #convert results array to Pandas DataFrame
+       results_frame = pd.DataFrame(np.transpose(results_array),
+                          columns=['r','stdev','sharpe']+context.assets)
+       #locate position of portfolio with highest Sharpe Ratio
+       max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
+       #locate positon of portfolio with minimum standard deviation
+       min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+       
+       #order optimal weights for each asset
+       for asset in context.assets:
+           if data.can_trade(asset):
+               order_target_percent(asset, max_sharpe_port[asset])
+       
+       #create scatter plot coloured by Sharpe Ratio
+       plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
+       plt.xlabel('Volatility')
+       plt.ylabel('Returns')
+       plt.colorbar()
+       #plot red star to highlight position of portfolio with highest Sharpe Ratio
+       plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
+       #plot green star to highlight position of minimum variance portfolio
+       plt.show()
+       print(max_sharpe_port)
+       record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
+   context.i += 1
+   
+       
+def analyze(context=None, results=None):
+   # Form DataFrame with selected data
+   data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
+   
+   # Save results in CSV file
+   filename = os.path.splitext(os.path.basename(__file__))[0]
+   data.to_csv(filename + '.csv')
+
+
+# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
+start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) 
+results = run_algorithm(initialize=initialize,
+                        handle_data=handle_data,
+                        analyze=analyze,
+                        start=start,
+                        end=end,
+                        exchange_name='poloniex',
+                        capital_base=100000, )

catalyst/examples/rsi_profit_target.py

@@ -0,0 +1,276 @@
+from datetime import timedelta
+
+import pandas as pd
+import numpy as np
+import talib
+from logbook import Logger
+
+from catalyst.api import (
+    order,
+    symbol,
+    record,
+    get_open_orders,
+)
+from catalyst.exchange.stats_utils import crossover, crossunder
+from catalyst.utils.run_algo import run_algorithm
+
+algo_namespace = 'rsi'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('initializing algo')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
+
+    context.MAX_HOLDINGS = 0.2
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERSOLD_BBANDS = 45
+    context.RSI_OVERBOUGHT_BBANDS = 55
+    context.SLIPPAGE_ALLOWED = 0.03
+
+    context.TARGET = 0.15
+    context.STOP_LOSS = 0.1
+    context.STOP = 0.03
+    context.position = None
+
+    context.last_bar = None
+
+    context.errors = []
+    pass
+
+
+def _handle_buy_sell_decision(context, data, signal, price):
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        log.info('skipping bar until all open orders execute')
+        return
+
+    positions = context.portfolio.positions
+    if context.position is None and context.asset in positions:
+        position = positions[context.asset]
+        context.position = dict(
+            cost_basis=position['cost_basis'],
+            amount=position['amount'],
+            stop=None
+        )
+
+    action = None
+    if context.position is not None:
+        cost_basis = context.position['cost_basis']
+        amount = context.position['amount']
+        log.info(
+            'found {amount} positions with cost basis {cost_basis}'.format(
+                amount=amount,
+                cost_basis=cost_basis
+            )
+        )
+        stop = context.position['stop']
+
+        target = cost_basis * (1 + context.TARGET)
+        if price >= target:
+            context.position['cost_basis'] = price
+            context.position['stop'] = context.STOP
+
+        stop_target = context.STOP_LOSS if stop is None else context.STOP
+        if price < cost_basis * (1 - stop_target):
+            log.info('executing stop loss')
+            order(
+                asset=context.asset,
+                amount=-amount,
+                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            action = 0
+            context.position = None
+
+    else:
+        if signal == 'long':
+            log.info('opening position')
+            buy_amount = context.MAX_HOLDINGS / price
+            order(
+                asset=context.asset,
+                amount=buy_amount,
+                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
+            )
+            context.position = dict(
+                cost_basis=price,
+                amount=buy_amount,
+                stop=None
+            )
+            action = 0
+
+
+def _handle_data_rsi_only(context, data):
+    price = data.current(context.asset, 'close')
+    log.info('got price {price}'.format(price=price))
+
+    if price is np.nan:
+        log.warn('no pricing data')
+        return
+
+    if context.base_price is None:
+        context.base_price = price
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=17,
+            frequency='30T'
+        )
+    except Exception as e:
+        log.warn('historical data not available: '.format(e))
+        return
+
+    rsi = talib.RSI(prices.values, timeperiod=16)[-1]
+    log.info('got rsi {}'.format(rsi))
+
+    signal = None
+    if rsi < context.RSI_OVERSOLD:
+        signal = 'long'
+
+    # Making sure that the price is still current
+    price = data.current(context.asset, 'close')
+    cash = context.portfolio.cash
+    log.info(
+        'base currency available: {cash}, cap: {cap}'.format(
+            cash=cash,
+            cap=context.MAX_HOLDINGS
+        )
+    )
+    volume = data.current(context.asset, 'volume')
+    price_change = (price - context.base_price) / context.base_price
+    record(
+        price=price,
+        price_change=price_change,
+        rsi=rsi,
+        volume=volume,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+    )
+
+    _handle_buy_sell_decision(context, data, signal, price)
+
+
+def handle_data(context, data):
+    dt = data.current_dt
+
+    if context.last_bar is None or (
+                context.last_bar + timedelta(minutes=15)) <= dt:
+        context.last_bar = dt
+    else:
+        return
+
+    log.info('BAR {}'.format(dt))
+    try:
+        _handle_data_rsi_only(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    results.loc[:, 'price'].plot(ax=ax2)
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    trans = results.loc[[t != [] for t in results.transactions], :]
+    buys = trans.loc[[t[0]['amount'] > 0 for t in trans.transactions], :]
+    sells = trans.loc[[t[0]['amount'] < 0 for t in trans.transactions], :]
+    # buys = results.loc[results['action'] == 1, :]
+    # sells = results.loc[results['action'] == 0, :]
+
+    ax2.plot(
+        buys.index,
+        results.loc[buys.index, 'price'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        results.loc[sells.index, 'price'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results.loc[:, ['alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Alpha / Beta ')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results.loc[:, ['starting_cash', 'cash']].plot(ax=ax4)
+    ax4.set_ylabel('Base Currency ({})'.format(base_currency))
+
+    results['algorithm'] = results.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results.loc[:, 'rsi'].plot(ax=ax6)
+    ax6.set_ylabel('RSI')
+
+    ax6.plot(
+        buys.index,
+        results.loc[buys.index, 'rsi'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax6.plot(
+        sells.index,
+        results.loc[sells.index, 'rsi'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='bittrex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc',
+#     live_graph=False
+# )
+
+# Backtest
+run_algorithm(
+    capital_base=0.5,
+    data_frequency='minute',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=analyze,
+    exchange_name='poloniex',
+    algo_namespace=algo_namespace,
+    base_currency='btc',
+    start=pd.to_datetime('2017-9-1', utc=True),
+    end=pd.to_datetime('2017-10-1', utc=True),
+)

catalyst/examples/simple_loop.py

@@ -2,12 +2,15 @@ import talib
 import pandas as pd
 
 from catalyst import run_algorithm
-from catalyst.api import symbol
+from catalyst.api import symbol, record
+from catalyst.exchange.stats_utils import get_pretty_stats, \
+    extract_transactions
 
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('eth_btc')
+    context.asset = symbol('neo_eth')
+    context.base_price = None
 
 
 def handle_data(context, data):
@@ -16,37 +19,113 @@ def handle_data(context, data):
     price = data.current(context.asset, 'close')
     print('got price {price}'.format(price=price))
 
-    try:
-        prices = data.history(
-            context.asset,
-            fields='price',
-            bar_count=16,
-            frequency='5T'
+    prices = data.history(
+        context.asset,
+        fields='price',
+        bar_count=20,
+        frequency='15T'
+    )
+    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
+    print('got rsi: {}'.format(rsi))
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        price_change=price_change,
+        cash=cash
+    )
+
+
+def analyze(context, perf):
+    import matplotlib.pyplot as plt
+    print('the stats: {}'.format(get_pretty_stats(perf)))
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
         )
-        rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-        print('got rsi: {}'.format(rsi))
-    except Exception as e:
-        print(e)
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
 
 
 run_algorithm(
     capital_base=250,
-    start=pd.to_datetime('2016-6-1', utc=True),
-    end=pd.to_datetime('2016-12-31', utc=True),
+    start=pd.to_datetime('2017-11-1 0:00', utc=True),
+    end=pd.to_datetime('2017-11-10 23:59', utc=True),
     data_frequency='daily',
     initialize=initialize,
     handle_data=handle_data,
-    analyze=None,
+    analyze=analyze,
     exchange_name='bitfinex',
     algo_namespace='simple_loop',
-    base_currency='btc'
+    base_currency='usd'
 )
 # run_algorithm(
 #     initialize=initialize,
 #     handle_data=handle_data,
 #     analyze=None,
-#     exchange_name='poloniex',
+#     exchange_name='binance',
 #     live=True,
 #     algo_namespace='simple_loop',
 #     base_currency='eth',
-#     live_graph=False
+#     live_graph=False,
+# )

catalyst/examples/simple_universe.py

@@ -0,0 +1,129 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.3
+
+These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
+You simply need to specify the exchange and the market that you want to focus on.
+You will all see how to create a universe and filter it base on the exchange and the market you desire.
+
+The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
+The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
+Use this as the backbone to create your own trading strategies.
+
+Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
+"""
+
+import numpy as np
+import pandas as pd
+from datetime import timedelta
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = context.exchanges.values()[0].name.lower()  # exchange name
+    context.base_currency = context.exchanges.values()[0].base_currency.lower()  # market base currency
+
+
+def handle_data(context, data):
+    context.i += 1
+    lookback_days = 7  # 7 days
+
+    # current date formatted into a string
+    today = data.current_dt
+    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = today - timedelta(days=lookback_days)  # subtract the amount of days specified in lookback
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  # get only the date as a string
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+    one_day_in_minutes = 1440  # 1440 assumes data_frequency='minute'
+    lookback = one_day_in_minutes / minutes * lookback_days  # get N lookback_days of history data
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
+            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
+            opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
+            high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
+            low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
+            close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
+            volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
+
+            # close[-1] is the equivalent to current price
+            # displays the minute price for each pair every 30 minutes
+            print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
+
+            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
+            # ----------------------------------------------------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    json_symbols = get_exchange_symbols(context.exchange)  # get all the pairs for the exchange
+    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)  # convert into a dataframe
+    universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
+                                                                       axis=1)
+    universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
+                                                                         axis=1)
+
+    # Filter all the exchange pairs to only the ones for a give base currency
+    universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
+
+    # Filter all the pairs to ensure that pair existed in the current date range
+    universe_df = universe_df[universe_df.start_date < lookback_date]
+    universe_df = universe_df[universe_df.end_daily >= current_date]
+    context.coins = symbols(*universe_df.symbol)  # convert all the pairs to symbols
+
+    # print(universe_df.symbol.tolist())
+    return universe_df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-11-10', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=100.0,  # amount of base_currency, not always in dollars unless usd
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='bitfinex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')
+
+"""
+Run in Terminal (inside catalyst environment):
+python simple_universe.py
+"""

catalyst/examples/talib_simple.py

@@ -0,0 +1,364 @@
+# Run Command
+# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex
+# 
+# Description
+# Simple TALib Example showing how to use various indicators in you strategy
+# Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
+
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import talib as ta
+from logbook import Logger
+from matplotlib.dates import date2num
+from matplotlib.finance import candlestick_ohlc
+
+from catalyst import run_algorithm
+from catalyst.api import (
+    order,
+    order_target_percent,
+    symbol,
+)
+from catalyst.exchange.stats_utils import get_pretty_stats
+
+algo_namespace = 'talib_sample'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('Starting TALib Simple Example')
+
+    context.ASSET_NAME = 'BTC_USDT'
+    context.asset = symbol(context.ASSET_NAME)
+
+    context.ORDER_SIZE = 10
+    context.SLIPPAGE_ALLOWED = 0.05
+
+    context.swallow_errors = True
+    context.errors = []
+
+    # Bars to look at per iteration should be bigger than SMA_SLOW
+    context.BARS = 365
+    context.COUNT = 0
+
+    # Technical Analysis Settings
+    context.SMA_FAST = 50
+    context.SMA_SLOW = 100
+    context.RSI_PERIOD = 14
+    context.RSI_OVER_BOUGHT = 80
+    context.RSI_OVER_SOLD = 20
+    context.RSI_AVG_PERIOD = 15
+    context.MACD_FAST = 12
+    context.MACD_SLOW = 26
+    context.MACD_SIGNAL = 9
+    context.STOCH_K = 14
+    context.STOCH_D = 3
+    context.STOCH_OVER_BOUGHT = 80
+    context.STOCH_OVER_SOLD = 20
+
+    pass
+
+
+def _handle_data(context, data):
+    # Get price, open, high, low, close
+    prices = data.history(
+        context.asset,
+        bar_count=context.BARS,
+        fields=['price', 'open', 'high', 'low', 'close'],
+        frequency='1d')
+
+    # Create a analysis data frame
+    analysis = pd.DataFrame(index=prices.index)
+
+    # SMA FAST
+    analysis['sma_f'] = ta.SMA(prices.close.as_matrix(), context.SMA_FAST)
+    # SMA SLOW
+    analysis['sma_s'] = ta.SMA(prices.close.as_matrix(), context.SMA_SLOW)
+
+    # Relative Strength Index
+    analysis['rsi'] = ta.RSI(prices.close.as_matrix(), context.RSI_PERIOD)
+    # RSI SMA
+    analysis['sma_r'] = ta.SMA(analysis.rsi.as_matrix(),
+                               context.RSI_AVG_PERIOD)
+
+    # MACD, MACD Signal, MACD Histogram
+    analysis['macd'], analysis['macdSignal'], analysis['macdHist'] = ta.MACD(
+        prices.close.as_matrix(), fastperiod=context.MACD_FAST,
+        slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
+
+    # Stochastics %K %D 
+    # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
+    # %D = 3-day SMA of %K
+    analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
+        prices.high.as_matrix(), prices.low.as_matrix(),
+        prices.close.as_matrix(), slowk_period=context.STOCH_K,
+        slowd_period=context.STOCH_D)
+
+    # SMA FAST over SLOW Crossover
+    analysis['sma_test'] = np.where(analysis.sma_f > analysis.sma_s, 1, 0)
+
+    # MACD over Signal Crossover
+    analysis['macd_test'] = np.where((analysis.macd > analysis.macdSignal), 1,
+                                     0)
+
+    # Stochastics OVER BOUGHT & Decreasing
+    analysis['stoch_over_bought'] = np.where(
+        (analysis.stoch_k > context.STOCH_OVER_BOUGHT) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # Stochastics OVER SOLD & Increasing
+    analysis['stoch_over_sold'] = np.where(
+        (analysis.stoch_k < context.STOCH_OVER_SOLD) & (
+            analysis.stoch_k > analysis.stoch_k.shift(1)), 1, 0)
+
+    # RSI OVER BOUGHT & Decreasing
+    analysis['rsi_over_bought'] = np.where(
+        (analysis.rsi > context.RSI_OVER_BOUGHT) & (
+            analysis.rsi < analysis.rsi.shift(1)), 1, 0)
+
+    # RSI OVER SOLD & Increasing
+    analysis['rsi_over_sold'] = np.where(
+        (analysis.rsi < context.RSI_OVER_SOLD) & (
+            analysis.rsi > analysis.rsi.shift(1)), 1, 0)
+
+    # Save the prices and analysis to send to analyze
+    context.prices = prices
+    context.analysis = analysis
+    context.price = data.current(context.asset, 'price')
+
+    makeOrders(context, analysis)
+
+    # Log the values of this bar
+    logAnalysis(analysis)
+
+
+def handle_data(context, data):
+    log.info('handling bar {}'.format(data.current_dt))
+    try:
+        _handle_data(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    log.info('completed bar {}, total execution errors {}'.format(
+        data.current_dt,
+        len(context.errors)
+    ))
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context, results):
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename('talib_simple'))[0]
+    results.to_csv(filename + '.csv')
+
+    log.info('the daily stats:\n{}'.format(get_pretty_stats(results)))
+    chart(context, context.prices, context.analysis, results)
+    pass
+
+
+def makeOrders(context, analysis):
+    if context.asset in context.portfolio.positions:
+
+        # Current position
+        position = context.portfolio.positions[context.asset]
+
+        if (position == 0):
+            log.info('Position Zero')
+            return
+
+        # Cost Basis
+        cost_basis = position.cost_basis
+
+        log.info(
+            'Holdings: {amount} @ {cost_basis}'.format(
+                amount=position.amount,
+                cost_basis=cost_basis
+            )
+        )
+
+        # Sell when holding and got sell singnal
+        if isSell(context, analysis):
+            profit = (context.price * position.amount) - (
+                cost_basis * position.amount)
+            order_target_percent(
+                asset=context.asset,
+                target=0,
+                limit_price=context.price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            log.info(
+                'Sold {amount} @ {price} Profit: {profit}'.format(
+                    amount=position.amount,
+                    price=context.price,
+                    profit=profit
+                )
+            )
+        else:
+            log.info('no buy or sell opportunity found')
+    else:
+        # Buy when not holding and got buy signal
+        if isBuy(context, analysis):
+            order(
+                asset=context.asset,
+                amount=context.ORDER_SIZE,
+                limit_price=context.price * (1 + context.SLIPPAGE_ALLOWED)
+            )
+            log.info(
+                'Bought {amount} @ {price}'.format(
+                    amount=context.ORDER_SIZE,
+                    price=context.price
+                )
+            )
+
+
+def isBuy(context, analysis):
+    # Bullish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 1):
+        # Bullish MACD
+        if (getLast(analysis, 'macd_test') == 1):
+            return True
+
+    # # Bullish Stochastics
+    # if(getLast(analysis, 'stoch_over_sold') == 1):
+    #     return True
+
+    # # Bullish RSI
+    # if(getLast(analysis, 'rsi_over_sold') == 1):
+    #     return True
+
+    return False
+
+
+def isSell(context, analysis):
+    # Bearish SMA Crossover
+    if (getLast(analysis, 'sma_test') == 0):
+        # Bearish MACD
+        if (getLast(analysis, 'macd_test') == 0):
+            return True
+
+    # # Bearish Stochastics
+    # if(getLast(analysis, 'stoch_over_bought') == 0):
+    #     return True
+
+    # # Bearish RSI
+    # if(getLast(analysis, 'rsi_over_bought') == 0):
+    #     return True
+
+    return False
+
+
+def chart(context, prices, analysis, results):
+    results.portfolio_value.plot()
+
+    # Data for matplotlib finance plot
+    dates = date2num(prices.index.to_pydatetime())
+
+    # Create the Open High Low Close Tuple
+    prices_ohlc = [tuple([dates[i],
+                          prices.open[i],
+                          prices.high[i],
+                          prices.low[i],
+                          prices.close[i]]) for i in range(len(dates))]
+
+    fig = plt.figure(figsize=(14, 18))
+
+    # Draw the candle sticks
+    ax1 = fig.add_subplot(411)
+    ax1.set_ylabel(context.ASSET_NAME, size=20)
+    candlestick_ohlc(ax1, prices_ohlc, width=0.4, colorup='g', colordown='r')
+
+    # Draw Moving Averages
+    analysis.sma_f.plot(ax=ax1, c='r')
+    analysis.sma_s.plot(ax=ax1, c='g')
+
+    # RSI
+    ax2 = fig.add_subplot(412)
+    ax2.set_ylabel('RSI', size=12)
+    analysis.rsi.plot(ax=ax2, c='g',
+                      label='Period: ' + str(context.RSI_PERIOD))
+    analysis.sma_r.plot(ax=ax2, c='r',
+                        label='MA: ' + str(context.RSI_AVG_PERIOD))
+    ax2.axhline(y=30, c='b')
+    ax2.axhline(y=50, c='black')
+    ax2.axhline(y=70, c='b')
+    ax2.set_ylim([0, 100])
+    handles, labels = ax2.get_legend_handles_labels()
+    ax2.legend(handles, labels)
+
+    # Draw MACD computed with Talib
+    ax3 = fig.add_subplot(413)
+    ax3.set_ylabel('MACD: ' + str(context.MACD_FAST) + ', ' + str(
+        context.MACD_SLOW) + ', ' + str(context.MACD_SIGNAL), size=12)
+    analysis.macd.plot(ax=ax3, color='b', label='Macd')
+    analysis.macdSignal.plot(ax=ax3, color='g', label='Signal')
+    analysis.macdHist.plot(ax=ax3, color='r', label='Hist')
+    ax3.axhline(0, lw=2, color='0')
+    handles, labels = ax3.get_legend_handles_labels()
+    ax3.legend(handles, labels)
+
+    # Stochastic plot
+    ax4 = fig.add_subplot(414)
+    ax4.set_ylabel('Stoch (k,d)', size=12)
+    analysis.stoch_k.plot(ax=ax4, label='stoch_k:' + str(context.STOCH_K),
+                          color='r')
+    analysis.stoch_d.plot(ax=ax4, label='stoch_d:' + str(context.STOCH_D),
+                          color='g')
+    handles, labels = ax4.get_legend_handles_labels()
+    ax4.legend(handles, labels)
+    ax4.axhline(y=20, c='b')
+    ax4.axhline(y=50, c='black')
+    ax4.axhline(y=80, c='b')
+
+    plt.show()
+
+
+def logAnalysis(analysis):
+    # Log only the last value in the array
+    log.info('- sma_f:          {:.2f}'.format(getLast(analysis, 'sma_f')))
+    log.info('- sma_s:          {:.2f}'.format(getLast(analysis, 'sma_s')))
+
+    log.info('- rsi:            {:.2f}'.format(getLast(analysis, 'rsi')))
+    log.info('- sma_r:          {:.2f}'.format(getLast(analysis, 'sma_r')))
+
+    log.info('- macd:           {:.2f}'.format(getLast(analysis, 'macd')))
+    log.info(
+        '- macdSignal:     {:.2f}'.format(getLast(analysis, 'macdSignal')))
+    log.info('- macdHist:       {:.2f}'.format(getLast(analysis, 'macdHist')))
+
+    log.info('- stoch_k:        {:.2f}'.format(getLast(analysis, 'stoch_k')))
+    log.info('- stoch_d:        {:.2f}'.format(getLast(analysis, 'stoch_d')))
+
+    log.info('- sma_test:       {}'.format(getLast(analysis, 'sma_test')))
+    log.info('- macd_test:      {}'.format(getLast(analysis, 'macd_test')))
+
+    log.info('- stoch_over_bought:   {}'.format(
+        getLast(analysis, 'stoch_over_bought')))
+    log.info(
+        '- stoch_over_sold:   {}'.format(getLast(analysis, 'stoch_over_sold')))
+
+    log.info('- rsi_over_bought:       {}'.format(
+        getLast(analysis, 'rsi_over_bought')))
+    log.info(
+        '- rsi_over_sold:       {}'.format(getLast(analysis, 'rsi_over_sold')))
+
+
+def getLast(arr, name):
+    return arr[name][arr[name].index[-1]]
+
+
+if __name__ == '__main__':
+    run_algorithm(
+        capital_base=10000,
+        data_frequency='daily',
+        initialize=initialize,
+        handle_data=handle_data,
+        analyze=analyze,
+        exchange_name='poloniex',
+        base_currency='usdt',
+        start=pd.to_datetime('2016-11-1', utc=True),
+        end=pd.to_datetime('2017-11-10', utc=True),
+    )

catalyst/exchange/asset_finder_exchange.py

@@ -41,14 +41,15 @@ class AssetFinderExchange(object):
         SidsNotFound
             When a requested sid is not found and default_none=False.
         """
-        for sid in sids:
-            if sid in self._asset_cache:
-                log.debug('got asset from cache: {}'.format(sid))
-            else:
-                log.debug('fetching asset: {}'.format(sid))
+        # for sid in sids:
+        #     if sid in self._asset_cache:
+        #         log.debug('got asset from cache: {}'.format(sid))
+        #     else:
+        #         log.debug('fetching asset: {}'.format(sid))
         return list()
 
-    def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):
+    def lookup_symbol(self, symbol, exchange, data_frequency=None,
+                      as_of_date=None, fuzzy=False):
         """Lookup an asset by symbol.
 
         Parameters
@@ -84,10 +85,15 @@ class AssetFinderExchange(object):
         """
         log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
 
-        key = ','.join([exchange.name, symbol])
+        if data_frequency is not None:
+            key = ','.join([exchange.name, symbol, data_frequency])
+
+        else:
+            key = ','.join([exchange.name, symbol])
+
         if key in self._asset_cache:
             return self._asset_cache[key]
         else:
-            asset = exchange.get_asset(symbol)
+            asset = exchange.get_asset(symbol, data_frequency)
             self._asset_cache[key] = asset
             return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -46,8 +46,13 @@ class Bitfinex(Exchange):
         self.secret = secret.encode('UTF-8')
         self.name = 'bitfinex'
         self.color = 'green'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -61,7 +66,7 @@ class Bitfinex(Exchange):
         self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def _request(self, operation, data, version='v1'):
         payload_object = {

catalyst/exchange/bittrex/bittrex.py

@@ -46,7 +46,10 @@ class Bittrex(Exchange):
         self.assets = dict()
         self.load_assets()
 
-        self.bundle = ExchangeBundle(self)
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
+        self.bundle = ExchangeBundle(self.name)
 
     @property
     def account(self):

catalyst/exchange/bittrex/bittrex_api.py

@@ -3,11 +3,12 @@ import json
 import time
 import hmac
 import hashlib
-
+import ssl
 
 # Workaround for backwards compatibility
 # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
 from six.moves import urllib
+
 urlopen = urllib.request.urlopen
 
 
@@ -48,7 +49,8 @@ class Bittrex_api(object):
             headers = {}
 
         req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(req).read())
+        response = json.loads(urlopen(
+            req, context=ssl._create_unverified_context()).read())
 
         if response["result"]:
             return response["result"]

catalyst/exchange/bundle_utils.py

@@ -6,9 +6,11 @@ from datetime import timedelta, datetime, date
 import numpy as np
 import pandas as pd
 import pytz
+from catalyst.assets._assets import TradingPair
 
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
+from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \
+    get_exchange_symbols
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 API_URL = 'http://data.enigma.co/api/v1'
@@ -149,7 +151,7 @@ def get_periods(start_dt, end_dt, freq):
     return len(get_periods_range(start_dt, end_dt, freq))
 
 
-def get_start_dt(end_dt, bar_count, data_frequency):
+def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     """
     The start date based on specified end date and data frequency.
 
@@ -168,6 +170,9 @@ def get_start_dt(end_dt, bar_count, data_frequency):
     if periods > 1:
         delta = get_delta(periods, data_frequency)
         start_dt = end_dt - delta
+
+        if not include_first:
+            start_dt += get_delta(1, data_frequency)
     else:
         start_dt = end_dt
 
@@ -314,3 +319,41 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
             has_data = False
 
     return has_data
+
+
+def get_assets(exchange, include_symbols, exclude_symbols):
+    """
+    Get assets from an exchange, including or excluding the specified
+    symbols.
+
+    Parameters
+    ----------
+    exchange: Exchange
+    include_symbols: str
+    exclude_symbols: str
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    if include_symbols is not None:
+        include_symbols_list = include_symbols.split(',')
+
+        return exchange.get_assets(include_symbols_list)
+
+    else:
+        all_assets = exchange.get_assets()
+
+        if exclude_symbols is not None:
+            exclude_symbols_list = exclude_symbols.split(',')
+
+            assets = []
+            for asset in all_assets:
+                if asset.symbol not in exclude_symbols_list:
+                    assets.append(asset)
+
+            return assets
+
+        else:
+            return all_assets

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -0,0 +1,600 @@
+import re
+from collections import defaultdict
+
+import ccxt
+import pandas as pd
+from ccxt import ExchangeNotAvailable
+from six import string_types
+
+from catalyst.finance.order import Order, ORDER_STATUS
+
+from catalyst.algorithm import MarketOrder
+from catalyst.assets._assets import TradingPair
+from logbook import Logger
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.exchange import Exchange, ExchangeLimitOrder
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
+    ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
+    ExchangeNotFoundError
+from catalyst.exchange.exchange_utils import mixin_market_params, \
+    from_ms_timestamp
+
+log = Logger('CCXT', level=LOG_LEVEL)
+
+SUPPORTED_EXCHANGES = dict(
+    binance=ccxt.binance,
+    bitfinex=ccxt.bitfinex,
+    bittrex=ccxt.bittrex,
+    poloniex=ccxt.poloniex,
+    bitmex=ccxt.bitmex,
+    gdax=ccxt.gdax,
+)
+
+
+class CCXT(Exchange):
+    def __init__(self, exchange_name, key, secret, base_currency,
+                 portfolio=None):
+        log.debug(
+            'finding {} in CCXT exchanges:\n{}'.format(
+                exchange_name, ccxt.exchanges
+            )
+        )
+        try:
+            # Making instantiation as explicit as possible for code tracking.
+            if exchange_name in SUPPORTED_EXCHANGES:
+                exchange_attr = SUPPORTED_EXCHANGES[exchange_name]
+
+            else:
+                exchange_attr = getattr(ccxt, exchange_name)
+
+            self.api = exchange_attr({
+                'apiKey': key,
+                'secret': secret,
+            })
+
+        except Exception:
+            raise ExchangeNotFoundError(exchange_name=exchange_name)
+
+        self._symbol_maps = [None, None]
+
+        markets_symbols = self.api.load_markets()
+        log.debug('the markets:\n{}'.format(markets_symbols))
+
+        self.name = exchange_name
+
+        self.markets = self.api.fetch_markets()
+        self.load_assets()
+
+        self.base_currency = base_currency
+        self._portfolio = portfolio
+        self.transactions = defaultdict(list)
+
+        self.num_candles_limit = 2000
+        self.max_requests_per_minute = 60
+        self.request_cpt = dict()
+
+        self.bundle = ExchangeBundle(self.name)
+
+    def account(self):
+        return None
+
+    def time_skew(self):
+        return None
+
+    def get_market(self, symbol):
+        """
+        The CCXT market.
+
+        Parameters
+        ----------
+        symbol:
+            The CCXT symbol.
+
+        Returns
+        -------
+        dict[str, Object]
+
+        """
+        s = self.get_symbol(symbol)
+        market = next(
+            (market for market in self.markets if market['symbol'] == s),
+            None,
+        )
+        return market
+
+    def get_symbol(self, asset_or_symbol):
+        """
+        The CCXT symbol.
+
+        Parameters
+        ----------
+        asset_or_symbol
+
+        Returns
+        -------
+
+        """
+        symbol = asset_or_symbol if isinstance(
+            asset_or_symbol, string_types
+        ) else asset_or_symbol.symbol
+
+        parts = symbol.split('_')
+        return '{}/{}'.format(parts[0].upper(), parts[1].upper())
+
+    def get_catalyst_symbol(self, market_or_symbol):
+        """
+        The Catalyst symbol.
+
+        Parameters
+        ----------
+        market_or_symbol
+
+        Returns
+        -------
+
+        """
+        if isinstance(market_or_symbol, string_types):
+            parts = market_or_symbol.split('/')
+            return '{}_{}'.format(parts[0].lower(), parts[1].lower())
+
+        else:
+            return '{}_{}'.format(
+                market_or_symbol['base'].lower(),
+                market_or_symbol['quote'].lower(),
+            )
+
+    def get_timeframe(self, freq):
+        """
+        The CCXT timeframe from the Catalyst frequency.
+
+        Parameters
+        ----------
+        freq: str
+            The Catalyst frequency (Pandas convention)
+
+        Returns
+        -------
+        str
+
+        """
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) \
+                if freq_match.group(1) else 1
+
+            unit = freq_match.group(2)
+
+        else:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+        if unit.lower() == 'd':
+            timeframe = '{}d'.format(candle_size)
+
+        elif unit.lower() == 'm' or unit == 'T':
+            timeframe = '{}m'.format(candle_size)
+
+        elif unit.lower() == 'h' or unit == 'T':
+            timeframe = '{}h'.format(candle_size)
+
+        return timeframe
+
+    def get_candles(self, freq, assets, bar_count=None, start_dt=None,
+                    end_dt=None):
+        symbols = self.get_symbols(assets)
+        timeframe = self.get_timeframe(freq)
+        delta = start_dt - pd.to_datetime('1970-1-1', utc=True)
+        ms = int(delta.total_seconds()) * 1000
+
+        candles = dict()
+        for asset in assets:
+            ohlcvs = self.api.fetch_ohlcv(
+                symbol=symbols[0],
+                timeframe=timeframe,
+                since=ms,
+                limit=bar_count,
+                params={}
+            )
+
+            candles[asset] = []
+            for ohlcv in ohlcvs:
+                candles[asset].append(dict(
+                    last_traded=pd.to_datetime(ohlcv[0], unit='ms', utc=True),
+                    open=ohlcv[1],
+                    high=ohlcv[2],
+                    low=ohlcv[3],
+                    close=ohlcv[4],
+                    volume=ohlcv[5]
+                ))
+
+        return candles
+
+    def _fetch_symbol_map(self, is_local):
+        try:
+            return self.fetch_symbol_map(is_local)
+        except ExchangeSymbolsNotFound:
+            return None
+
+    def get_asset_defs(self, market):
+        """
+        The local and Catalyst definitions of the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dicts.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        asset_defs = []
+
+        for is_local in (False, True):
+            asset_def = self.get_asset_def(market, is_local)
+            asset_defs.append((asset_def, is_local))
+
+        return asset_defs
+
+    def get_asset_def(self, market, is_local=False):
+        """
+        The asset definition (in symbols.json files) corresponding
+        to the the specified market.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+            The CCXT market dict.
+        is_local
+            Whether to search in local or Catalyst asset definitions.
+
+        Returns
+        -------
+        dict[str, Object]
+            The asset definition.
+
+        """
+        exchange_symbol = market['id']
+
+        symbol_map = self._fetch_symbol_map(is_local)
+        if symbol_map is not None:
+            assets_lower = {k.lower(): v for k, v in symbol_map.items()}
+            key = exchange_symbol.lower()
+
+            asset = assets_lower[key] if key in assets_lower else None
+            if asset is not None:
+                return asset
+
+            else:
+                return None
+
+        else:
+            return None
+
+    def create_trading_pair(self, market, asset_def, is_local):
+        """
+        Creating a TradingPair from market and asset data.
+
+        Parameters
+        ----------
+        market: dict[str, Object]
+        asset_def: dict[str, Object]
+        is_local: bool
+
+        Returns
+        -------
+
+        """
+        data_source = 'local' if is_local else 'catalyst'
+        params = dict(
+            exchange=self.name,
+            data_source=data_source,
+            exchange_symbol=market['id'],
+        )
+        mixin_market_params(self.name, params, market)
+
+        if asset_def is not None:
+            params['symbol'] = asset_def['symbol']
+
+            params['start_date'] = asset_def['start_date'] \
+                if 'start_date' in asset_def else None
+
+            params['end_date'] = asset_def['end_date'] \
+                if 'end_date' in asset_def else None
+
+            params['leverage'] = asset_def['leverage'] \
+                if 'leverage' in asset_def else 1.0
+
+            params['asset_name'] = asset_def['asset_name'] \
+                if 'asset_name' in asset_def else None
+
+            params['end_daily'] = asset_def['end_daily'] \
+                if 'end_daily' in asset_def \
+                   and asset_def['end_daily'] != 'N/A' else None
+
+            params['end_minute'] = asset_def['end_minute'] \
+                if 'end_minute' in asset_def \
+                   and asset_def['end_minute'] != 'N/A' else None
+
+        else:
+            params['symbol'] = self.get_catalyst_symbol(market)
+            # TODO: add as an optional column
+            params['leverage'] = 1.0
+
+        return TradingPair(**params)
+
+    def load_assets(self):
+        self.assets = []
+
+        for market in self.markets:
+            asset_defs = self.get_asset_defs(market)
+
+            for asset_def in asset_defs:
+                if asset_def[0] is not None or not asset_defs[1]:
+                    try:
+                        asset = self.create_trading_pair(
+                            market=market,
+                            asset_def=asset_def[0],
+                            is_local=asset_def[1]
+                        )
+                        self.assets.append(asset)
+
+                    except TypeError:
+                        pass
+
+    def get_balances(self):
+        try:
+            log.debug('retrieving wallets balances')
+            balances = self.api.fetch_balance()
+
+            balances_lower = dict()
+            for key in balances:
+                balances_lower[key.lower()] = balances[key]
+
+        except Exception as e:
+            log.debug('error retrieving balances: {}', e)
+            raise ExchangeRequestError(error=e)
+
+        return balances_lower
+
+    def _create_order(self, order_status):
+        """
+        Create a Catalyst order object from a CCXT order dictionary
+
+        Parameters
+        ----------
+        order_status: dict[str, Object]
+            The order dict from the CCXT api.
+
+        Returns
+        -------
+        Order
+            The Catalyst order object
+
+        """
+        if order_status['status'] == 'canceled':
+            status = ORDER_STATUS.CANCELLED
+
+        elif order_status['status'] == 'closed' and order_status['filled'] > 0:
+            log.debug('found executed order {}'.format(order_status))
+            status = ORDER_STATUS.FILLED
+
+        elif order_status['status'] == 'open':
+            status = ORDER_STATUS.OPEN
+
+        else:
+            raise ValueError('invalid state for order')
+
+        amount = order_status['amount']
+        filled = order_status['filled']
+
+        if order_status['side'] == 'sell':
+            amount = -amount
+            filled = -filled
+
+        price = order_status['price']
+        order_type = order_status['type']
+
+        limit_price = price if order_type == 'limit' else None
+        stop_price = None  # TODO: add support
+
+        executed_price = order_status['cost'] / order_status['amount']
+        commission = order_status['fee']
+        date = from_ms_timestamp(order_status['timestamp'])
+
+        # order_id = str(order_status['info']['clientOrderId'])
+        order_id = order_status['id']
+
+        # TODO: this won't work, redo the packages with a different key.
+        symbol = order_status['info']['symbol'] \
+            if 'symbol' in order_status['info'] \
+            else order_status['info']['Exchange']
+
+        order = Order(
+            dt=date,
+            asset=self.get_asset(symbol, is_exchange_symbol=True),
+            amount=amount,
+            stop=stop_price,
+            limit=limit_price,
+            filled=filled,
+            id=order_id,
+            commission=commission
+        )
+        order.status = status
+
+        return order, executed_price
+
+    def create_order(self, asset, amount, is_buy, style):
+        symbol = self.get_symbol(asset)
+
+        if isinstance(style, ExchangeLimitOrder):
+            price = style.get_limit_price(is_buy)
+            order_type = 'limit'
+
+        elif isinstance(style, MarketOrder):
+            price = None
+            order_type = 'market'
+
+        else:
+            raise InvalidOrderStyle(
+                exchange=self.name,
+                style=style.__class__.__name__
+            )
+
+        side = 'buy' if amount > 0 else 'sell'
+
+        try:
+            result = self.api.create_order(
+                symbol=symbol,
+                type=order_type,
+                side=side,
+                amount=abs(amount),
+                price=price
+            )
+        except ExchangeNotAvailable as e:
+            log.debug('unable to create order: {}'.format(e))
+            raise ExchangeRequestError(error=e)
+
+        if 'info' not in result:
+            raise ValueError('cannot use order without info attribute')
+
+        order_id = result['id']
+        order = Order(
+            dt=pd.Timestamp.utcnow(),
+            asset=asset,
+            amount=amount,
+            stop=style.get_stop_price(is_buy),
+            limit=style.get_limit_price(is_buy),
+            id=order_id
+        )
+        return order
+
+    def get_open_orders(self, asset):
+        try:
+            symbol = self.get_symbol(asset)
+            result = self.api.fetch_open_orders(
+                symbol=symbol,
+                since=None,
+                limit=None,
+                params=dict()
+            )
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        orders = []
+        for order_status in result:
+            order, executed_price = self._create_order(order_status)
+            if asset is None or asset == order.sid:
+                orders.append(order)
+
+        return orders
+
+    def _get_asset_from_order(self, order_id):
+        open_orders = self.portfolio.open_orders
+        order = next(
+            (open_orders[id] for id in open_orders if id == order_id),
+            None
+        )  # type: Order
+        return order.asset if order is not None else None
+
+    def get_order(self, order_id, asset_or_symbol=None):
+        if asset_or_symbol is None and self.portfolio is not None:
+            asset_or_symbol = self._get_asset_from_order(order_id)
+
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, the request might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            order_status = self.api.fetch_order(id=order_id, symbol=symbol)
+            order, executed_price = self._create_order(order_status)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+        return order, executed_price
+
+    def cancel_order(self, order_param, asset_or_symbol=None):
+        order_id = order_param.id \
+            if isinstance(order_param, Order) else order_param
+
+        if asset_or_symbol is None and self.portfolio is not None:
+            asset_or_symbol = self._get_asset_from_order(order_id)
+
+        if asset_or_symbol is None:
+            log.debug(
+                'order not found in memory, cancelling order might fail '
+                'on some exchanges.'
+            )
+        try:
+            symbol = self.get_symbol(asset_or_symbol) \
+                if asset_or_symbol is not None else None
+            self.api.cancel_order(id=order_id, symbol=symbol)
+
+        except Exception as e:
+            raise ExchangeRequestError(error=e)
+
+    def tickers(self, assets):
+        """
+        Retrieve current tick data for the given assets
+
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[dict[str, float]
+
+        """
+        tickers = dict()
+        for asset in assets:
+            ccxt_symbol = self.get_symbol(asset)
+            ticker = self.api.fetch_ticker(ccxt_symbol)
+
+            ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
+
+            if 'last_price' not in ticker:
+                # TODO: any more exceptions?
+                ticker['last_price'] = ticker['last']
+
+            # Using the volume represented in the base currency
+            ticker['volume'] = ticker['baseVolume'] \
+                if 'baseVolume' in ticker else 0
+
+            tickers[asset] = ticker
+
+        return tickers
+
+    def get_account(self):
+        return None
+
+    def get_orderbook(self, asset, order_type='all', limit=None):
+        ccxt_symbol = self.get_symbol(asset)
+
+        params = dict()
+        if limit is not None:
+            params['depth'] = limit
+
+        order_book = self.api.fetch_order_book(ccxt_symbol, params)
+
+        order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
+        result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
+        for index, order_type in enumerate(order_types):
+            if limit is not None and index > limit - 1:
+                break
+
+            result[order_type] = []
+            for entry in order_book[order_type]:
+                result[order_type].append(dict(
+                    rate=float(entry[0]),
+                    quantity=float(entry[1])
+                ))
+
+        return result

catalyst/exchange/exchange.py

@@ -8,15 +8,16 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.algorithm import MarketOrder
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
     get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
-    InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
+    BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange
+    NoDataAvailableOnExchange, NoValueForField
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,7 +25,6 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols, \
     get_frequency, resample_history_df
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
-from catalyst.utils.deprecate import deprecated
 
 log = Logger('Exchange', level=LOG_LEVEL)
 
@@ -34,7 +34,8 @@ class Exchange:
 
     def __init__(self):
         self.name = None
-        self.assets = {}
+        self.assets = []
+        self._symbol_maps = [None, None]
         self._portfolio = None
         self.minute_writer = None
         self.minute_reader = None
@@ -43,7 +44,7 @@ class Exchange:
         self.num_candles_limit = None
         self.max_requests_per_minute = None
         self.request_cpt = None
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     @property
     def positions(self):
@@ -132,7 +133,7 @@ class Exchange:
 
     def get_symbol(self, asset):
         """
-        The the exchange specific symbol of the specified market.
+        The exchange specific symbol of the specified market.
 
         Parameters
         ----------
@@ -145,9 +146,9 @@ class Exchange:
         """
         symbol = None
 
-        for key in self.assets:
-            if not symbol and self.assets[key].symbol == asset.symbol:
-                symbol = key
+        for a in self.assets:
+            if not symbol and a.symbol == asset.symbol:
+                symbol = a.symbol
 
         if not symbol:
             raise ValueError('Currency %s not supported by exchange %s' %
@@ -174,54 +175,104 @@ class Exchange:
 
         return symbols
 
-    def get_assets(self, symbols=None):
+    def get_assets(self, symbols=None, data_frequency=None,
+                   is_exchange_symbol=False,
+                   is_local=None):
         """
         The list of markets for the specified symbols.
 
         Parameters
         ----------
         symbols: list[str]
+        data_frequency: str
+        is_exchange_symbol: bool
+        is_local: bool
 
         Returns
         -------
         list[TradingPair]
+            A list of asset objects.
+
+        Notes
+        -----
+        See get_asset for details of each parameter.
 
         """
+        if symbols is None:
+            # Make a distinct list of all symbols
+            symbols = list(set([asset.symbol for asset in self.assets]))
+            is_exchange_symbol = False
+
         assets = []
-
-        if symbols is not None:
-            for symbol in symbols:
-                asset = self.get_asset(symbol)
-                assets.append(asset)
-        else:
-            for key in self.assets:
-                assets.append(self.assets[key])
-
+        for symbol in symbols:
+            asset = self.get_asset(
+                symbol, data_frequency, is_exchange_symbol, is_local
+            )
+            assets.append(asset)
         return assets
 
-    def get_asset(self, symbol):
+    def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
+                  is_local=None):
         """
         The market for the specified symbol.
 
         Parameters
         ----------
         symbol: str
+            The Catalyst or exchange symbol.
+
+        data_frequency: str
+            Check for asset corresponding to the specified data_frequency.
+            The same asset might exist in the Catalyst repository or
+            locally (following a CSV ingestion). Filtering by
+            data_frequency picks the right asset.
+
+        is_exchange_symbol: bool
+            Whether the symbol uses the Catalyst or exchange convention.
+
+        is_local: bool
+            For the local or Catalyst asset.
 
         Returns
         -------
         TradingPair
+            The asset object.
 
         """
         asset = None
 
-        for key in self.assets:
-            if not asset and self.assets[key].symbol.lower() == symbol.lower():
-                asset = self.assets[key]
+        log.debug(
+            'searching assets for: {} {}'.format(
+                self.name, symbol
+            )
+        )
+        for a in self.assets:
+            if asset is not None:
+                break
 
-        if not asset:
-            supported_symbols = [
-                pair.symbol for pair in list(self.assets.values())
-            ]
+            if is_local is not None:
+                data_source = 'local' if is_local else 'catalyst'
+                applies = (a.data_source == data_source)
+
+            elif data_frequency is not None:
+                applies = (
+                    (data_frequency == 'minute' and a.end_minute is not None)
+                    or (data_frequency == 'daily' and a.end_daily is not None)
+                )
+
+            else:
+                applies = True
+
+            # The symbol provided may use the Catalyst or the exchange
+            # convention
+            key = a.exchange_symbol if is_exchange_symbol else a.symbol
+            if not asset and key.lower() == symbol.lower() and applies:
+                asset = a
+
+        if asset is None:
+            supported_symbols = sorted([
+                asset.symbol for asset in self.assets
+            ])
 
             raise SymbolNotFoundOnExchange(
                 symbol=symbol,
@@ -229,12 +280,21 @@ class Exchange:
                 supported_symbols=supported_symbols
             )
 
+        log.debug('found asset: {}'.format(asset))
         return asset
 
-    def fetch_symbol_map(self):
-        return get_exchange_symbols(self.name)
+    def fetch_symbol_map(self, is_local=False):
+        index = 1 if is_local else 0
+        if self._symbol_maps[index] is not None:
+            return self._symbol_maps[index]
 
-    def load_assets(self):
+        else:
+            symbol_map = get_exchange_symbols(self.name, is_local)
+            self._symbol_maps[index] = symbol_map
+            return symbol_map
+
+    @abstractmethod
+    def load_assets(self, is_local=False):
         """
         Populate the 'assets' attribute with a dictionary of Assets.
         The key of the resulting dictionary is the exchange specific
@@ -247,63 +307,11 @@ class Exchange:
         universal symbol. This simple approach avoids maintaining a mapping
         of sids.
 
-        This method can be overridden if an exchange offers equivalent data
+        This method can be omerridden if an exchange offers equivalent data
         via its api.
 
         """
-        symbol_map = self.fetch_symbol_map()
-        for exchange_symbol in symbol_map:
-            asset = symbol_map[exchange_symbol]
-
-            if 'start_date' in asset:
-                start_date = pd.to_datetime(asset['start_date'], utc=True)
-            else:
-                start_date = None
-
-            if 'end_date' in asset:
-                end_date = pd.to_datetime(asset['end_date'], utc=True)
-            else:
-                end_date = None
-
-            if 'leverage' in asset:
-                leverage = asset['leverage']
-            else:
-                leverage = 1.0
-
-            if 'asset_name' in asset:
-                asset_name = asset['asset_name']
-            else:
-                asset_name = None
-
-            if 'min_trade_size' in asset:
-                min_trade_size = asset['min_trade_size']
-            else:
-                min_trade_size = 0.0000001
-
-            if 'end_daily' in asset and asset['end_daily'] != 'N/A':
-                end_daily = pd.to_datetime(asset['end_daily'], utc=True)
-            else:
-                end_daily = None
-
-            if 'end_minute' in asset and asset['end_minute'] != 'N/A':
-                end_minute = pd.to_datetime(asset['end_minute'], utc=True)
-            else:
-                end_minute = None
-
-            trading_pair = TradingPair(
-                symbol=asset['symbol'],
-                exchange=self.name,
-                start_date=start_date,
-                end_date=end_date,
-                leverage=leverage,
-                asset_name=asset_name,
-                min_trade_size=min_trade_size,
-                end_daily=end_daily,
-                end_minute=end_minute,
-                exchange_symbol=exchange_symbol
-            )
-
-            self.assets[exchange_symbol] = trading_pair
+        pass
 
     def check_open_orders(self):
         """
@@ -322,9 +330,11 @@ class Exchange:
                 log.debug('found open order: {}'.format(order_id))
 
                 order, executed_price = self.get_order(order_id)
-                log.debug('got updated order {} {}'.format(
-                    order, executed_price))
-
+                log.debug(
+                    'got updated order {} {}'.format(
+                        order, executed_price
+                    )
+                )
                 if order.status == ORDER_STATUS.FILLED:
                     transaction = Transaction(
                         asset=order.asset,
@@ -386,12 +396,15 @@ class Exchange:
         if field not in BASE_FIELDS:
             raise KeyError('Invalid column: {}'.format(field))
 
-        values = []
-        for asset in assets:
-            value = self.get_single_spot_value(asset, field, data_frequency)
-            values.append(value)
+        tickers = self.tickers(assets)
+        if field == 'close' or field == 'price':
+            return [tickers[asset]['last'] for asset in tickers]
 
-        return values
+        elif field == 'volume':
+            return [tickers[asset]['volume'] for asset in tickers]
+
+        else:
+            raise NoValueForField(field=field)
 
     def get_single_spot_value(self, asset, field, data_frequency):
         """
@@ -468,15 +481,14 @@ class Exchange:
 
         return series
 
-    @deprecated
-    def get_history_window_direct(self,
-                                  assets,
-                                  end_dt,
-                                  bar_count,
-                                  frequency,
-                                  field,
-                                  data_frequency=None,
-                                  ffill=True):
+    def get_history_window(self,
+                           assets,
+                           end_dt,
+                           bar_count,
+                           frequency,
+                           field,
+                           data_frequency=None,
+                           ffill=True):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -514,35 +526,46 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
-        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        freq, candle_size, unit, data_frequency = get_frequency(
+            frequency, data_frequency
+        )
+        adj_bar_count = candle_size * bar_count
+        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
 
         # The get_history method supports multiple asset
         candles = self.get_candles(
-            data_frequency=frequency,
+            freq=freq,
             assets=assets,
             bar_count=bar_count,
             start_dt=start_dt,
             end_dt=end_dt
         )
-        candle_series = self.get_series_from_candles(
-            candles=candles,
-            start_dt=start_dt,
-            end_dt=end_dt,
-            data_frequency=frequency,
-            field=field,
-        )
 
-        df = pd.DataFrame(candle_series)
+        series = dict()
+        for asset in candles:
+            asset_series = self.get_series_from_candles(
+                candles=candles[asset],
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=frequency,
+                field=field,
+            )
+            series[asset] = asset_series
+
+        df = pd.DataFrame(series)
+        df.dropna(inplace=True)
+
         return df
 
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency=None,
-                           ffill=True):
+    def get_history_window_with_bundle(self,
+                                       assets,
+                                       end_dt,
+                                       bar_count,
+                                       frequency,
+                                       field,
+                                       data_frequency=None,
+                                       ffill=True,
+                                       force_auto_ingest=False):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -590,7 +613,8 @@ class Exchange:
                 end_dt=end_dt,
                 bar_count=adj_bar_count,
                 field=field,
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                force_auto_ingest=force_auto_ingest
             )
         except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
             series = dict()
@@ -654,7 +678,7 @@ class Exchange:
         log.debug('synchronizing portfolio with exchange {}'.format(self.name))
         balances = self.get_balances()
 
-        base_position_available = balances[self.base_currency] \
+        base_position_available = balances[self.base_currency]['free'] \
             if self.base_currency in balances else None
 
         if base_position_available is None:
@@ -679,8 +703,9 @@ class Exchange:
                 # TODO: convert if the position is not in the base currency
                 ticker = tickers[asset]
                 position = portfolio.positions[asset]
+
                 position.last_sale_price = ticker['last_price']
-                position.last_sale_date = ticker['timestamp']
+                position.last_sale_date = ticker['last_traded']
 
                 portfolio.positions_value += \
                     position.amount * position.last_sale_price
@@ -737,28 +762,30 @@ class Exchange:
             log.warn('skipping order amount of 0')
             return None
 
-        if asset.base_currency != self.base_currency.lower():
+        if self.base_currency is None:
+            raise ValueError('no base_currency defined for this exchange')
+
+        if asset.quote_currency != self.base_currency.lower():
             raise MismatchingBaseCurrencies(
-                base_currency=asset.base_currency,
+                base_currency=asset.quote_currency,
                 algo_currency=self.base_currency
             )
 
         is_buy = (amount > 0)
 
         if limit_price is not None and stop_price is not None:
-            style = ExchangeStopLimitOrder(limit_price, stop_price,
-                                           exchange=self.name)
+            style = ExchangeStopLimitOrder(
+                limit_price, stop_price, exchange=self.name
+            )
+
         elif limit_price is not None:
             style = ExchangeLimitOrder(limit_price, exchange=self.name)
 
         elif stop_price is not None:
             style = ExchangeStopOrder(stop_price, exchange=self.name)
 
-        elif style is not None:
-            raise InvalidOrderStyle(exchange=self.name.title(),
-                                    style=style.__class__.__name__)
         else:
-            raise ValueError('Incomplete order data.')
+            style = MarketOrder(exchange=self.name)
 
         display_price = limit_price if limit_price is not None else stop_price
         log.debug(
@@ -767,9 +794,10 @@ class Exchange:
                 amount=amount,
                 symbol=asset.symbol,
                 type=style.__class__.__name__,
-                price='{}{}'.format(display_price, asset.base_currency)
+                price='{}{}'.format(display_price, asset.quote_currency)
             )
         )
+
         order = self.create_order(asset, amount, is_buy, style)
         if order:
             self._portfolio.create_order(order)
@@ -838,7 +866,7 @@ class Exchange:
         pass
 
     @abstractmethod
-    def get_order(self, order_id):
+    def get_order(self, order_id, symbol_or_asset=None):
         """Lookup an order based on the order id returned from one of the
         order functions.
 
@@ -846,6 +874,8 @@ class Exchange:
         ----------
         order_id : str
             The unique identifier for the order.
+        symbol_or_asset: str|TradingPair
+            The catalyst symbol, some exchanges need this
 
         Returns
         -------
@@ -857,13 +887,15 @@ class Exchange:
         pass
 
     @abstractmethod
-    def cancel_order(self, order_param):
+    def cancel_order(self, order_param, symbol_or_asset=None):
         """Cancel an open order.
 
         Parameters
         ----------
         order_param : str or Order
             The order_id or order object to cancel.
+        symbol_or_asset: str|TradingPair
+            The catalyst symbol, some exchanges need this
         """
         pass
 

catalyst/exchange/exchange_algorithm.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import pickle
 import signal
 import sys
@@ -27,8 +26,6 @@ from catalyst.assets._assets import TradingPair
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
-from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
-    BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
@@ -38,8 +35,8 @@ from catalyst.exchange.exchange_errors import (
     OrphanOrderError)
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
-    save_algo_object, get_algo_object, get_algo_folder, get_algo_df, \
+from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
+    get_algo_folder, get_algo_df, \
     save_algo_df
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
@@ -117,9 +114,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         else:
             exchange = self.exchanges[exchange_name]
 
+        data_frequency = self.data_frequency \
+            if self.sim_params.arena == 'backtest' else None
         return self.asset_finder.lookup_symbol(
             symbol=symbol_str,
             exchange=exchange,
+            data_frequency=data_frequency,
             as_of_date=_lookup_date
         )
 
@@ -182,17 +182,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
-        stats['transactions'] = dict()
+        stats['transactions'] = []
         for date in period.processed_transactions:
             if start_dt <= date < end_dt:
-                stats['transactions'][date] = \
-                    period.processed_transactions[date]
+                transactions = period.processed_transactions[date]
+                for t in transactions:
+                    stats['transactions'].append(t.to_dict())
 
-        stats['orders'] = dict()
+        stats['orders'] = []
         for date in period.orders_by_modified:
             if start_dt <= date < end_dt:
-                stats['orders'][date] = \
-                    period.orders_by_modified[date]
+                orders = period.orders_by_modified[date]
+                for order in orders:
+                    stats['orders'].append(orders[order].to_dict())
 
         return stats
 
@@ -201,6 +203,7 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
+        self.frame_stats = list()
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
@@ -245,14 +248,51 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
         else:
             return MarketOrder()
 
+    def is_last_frame_of_day(self, data):
+        # TODO: adjust here to support more intervals
+        next_frame_dt = data.current_dt + timedelta(minutes=1)
+        if next_frame_dt.date() > data.current_dt.date():
+            return True
+        else:
+            return False
+
+    def handle_data(self, data):
+        super(ExchangeTradingAlgorithmBacktest, self).handle_data(data)
+
+        if self.data_frequency == 'minute':
+            frame_stats = self.prepare_period_stats(
+                data.current_dt, data.current_dt + timedelta(minutes=1)
+            )
+            self.frame_stats.append(frame_stats)
+
+    def _create_stats_df(self):
+        stats = pd.DataFrame(self.frame_stats)
+        stats.set_index('period_close', inplace=True, drop=False)
+        return stats
+
+    def analyze(self, perf):
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
+        super(ExchangeTradingAlgorithmBacktest, self).analyze(stats)
+
+    def run(self, data=None, overwrite_sim_params=True):
+        perf = super(ExchangeTradingAlgorithmBacktest, self).run(
+            data, overwrite_sim_params
+        )
+        # Rebuilding the stats to support minute data
+        stats = self._create_stats_df() if self.data_frequency == 'minute' \
+            else perf
+        return stats
+
 
 class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
+        self.simulate_orders = kwargs.pop('simulate_orders', None)
 
         self._clock = None
-        self.minute_stats = deque(maxlen=60)
+        self.frame_stats = deque(maxlen=60)
 
         self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
 
@@ -273,34 +313,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.stats_minutes = 5
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
-        # TODO: fix precision before re-enabling
-        # self._create_minute_writer()
 
         signal.signal(signal.SIGINT, self.signal_handler)
 
         log.info('initialized trading algorithm in live mode')
 
-    def _create_minute_writer(self):
-        root = get_exchange_minute_writer_root(self.exchange.name)
-        filename = os.path.join(root, 'metadata.json')
-
-        if os.path.isfile(filename):
-            writer = BcolzMinuteBarWriter.open(
-                root, self.sim_params.end_session)
-        else:
-            # TODO: need to be able to write more precise numbers
-            writer = BcolzMinuteBarWriter(
-                rootdir=root,
-                calendar=self.trading_calendar,
-                minutes_per_day=1440,
-                start_session=self.sim_params.start_session,
-                end_session=self.sim_params.end_session,
-                write_metadata=True
-            )
-
-        self.exchange.minute_writer = writer
-        self.exchange.minute_reader = BcolzMinuteBarReader(root)
-
     def signal_handler(self, signal, frame):
         """
         Handles the keyboard interruption signal.
@@ -544,8 +561,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         )
         self.exposure_stats = pd.concat([self.exposure_stats, df])
 
-        save_algo_df(self.algo_namespace, 'exposure_stats',
-                     self.exposure_stats)
+        save_algo_df(
+            self.algo_namespace, 'exposure_stats', self.exposure_stats
+        )
 
     def handle_data(self, data):
         """
@@ -562,8 +580,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self._synchronize_portfolio()
 
         transactions = self._check_open_orders()
-        for transaction in transactions:
-            self.perf_tracker.process_transaction(transaction)
+        if len(transactions) > 0:
+            for transaction in transactions:
+                self.perf_tracker.process_transaction(transaction)
+
+            self.perf_tracker.update_performance()
 
         if self._handle_data:
             self._handle_data(self, data)
@@ -578,22 +599,22 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             # Performance tracker and keep only minute and cumulative
             self.perf_tracker.update_performance()
 
-            minute_stats = self.prepare_period_stats(
+            frame_stats = self.prepare_period_stats(
                 data.current_dt, data.current_dt + timedelta(minutes=1))
 
             # Saving the last hour in memory
-            self.minute_stats.append(minute_stats)
+            self.frame_stats.append(frame_stats)
 
-            self.add_pnl_stats(minute_stats)
+            self.add_pnl_stats(frame_stats)
             if self.recorded_vars:
-                self.add_custom_signals_stats(minute_stats)
+                self.add_custom_signals_stats(frame_stats)
                 recorded_cols = list(self.recorded_vars.keys())
             else:
                 recorded_cols = None
 
-            self.add_exposure_stats(minute_stats)
+            self.add_exposure_stats(frame_stats)
 
-            print_df = pd.DataFrame(list(self.minute_stats))
+            print_df = pd.DataFrame(list(self.frame_stats))
             log.info(
                 'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
                     stats_minutes=self.stats_minutes,

catalyst/exchange/exchange_bundle.py

@@ -1,55 +1,35 @@
 import os
 import shutil
+from datetime import datetime, timedelta
+from functools import partial
 from itertools import chain
+from operator import is_not
 
+import numpy as np
 import pandas as pd
+import pytz
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
-from pandas.tslib import Timestamp
 from pytz import UTC
 from six import itervalues
 
 from catalyst import get_calendar
+from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_delta, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label
+    get_bcolz_chunk, get_month_start_end, \
+    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
+    get_delta, get_assets
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     TempBundleNotFoundError, \
     NoDataAvailableOnExchange, \
-    PricingDataNotLoadedError
-from catalyst.exchange.exchange_utils import get_exchange_folder
-from catalyst.utils.cli import maybe_show_progress
-from catalyst.utils.paths import ensure_directory
-import os
-import shutil
-from itertools import chain
-
-import pandas as pd
-from catalyst.assets._assets import TradingPair
-from logbook import Logger
-from pandas.tslib import Timestamp
-from pytz import UTC
-from six import itervalues
-
-from catalyst import get_calendar
-from catalyst.constants import LOG_LEVEL
-from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
-    BcolzMinuteBarMetadata
-from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_delta, get_month_start_end, \
-    get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label
-from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
-    BcolzExchangeBarWriter
-from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
-    TempBundleNotFoundError, \
-    NoDataAvailableOnExchange, \
-    PricingDataNotLoadedError
-from catalyst.exchange.exchange_utils import get_exchange_folder
+    PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
+from catalyst.exchange.exchange_utils import get_exchange_folder, \
+    save_exchange_symbols, mixin_market_params
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
@@ -63,23 +43,14 @@ def _cachpath(symbol, type_):
 
 
 class ExchangeBundle:
-    def __init__(self, exchange):
-        self.exchange = exchange
+    def __init__(self, exchange_name):
+        self.exchange_name = exchange_name
         self.minutes_per_day = 1440
         self.default_ohlc_ratio = 1000000
         self._writers = dict()
         self._readers = dict()
         self.calendar = get_calendar('OPEN')
-
-    def get_assets(self, include_symbols, exclude_symbols):
-        # TODO: filter exclude symbols assets
-        if include_symbols is not None:
-            include_symbols_list = include_symbols.split(',')
-
-            return self.exchange.get_assets(include_symbols_list)
-
-        else:
-            return self.exchange.get_assets()
+        self.exchange = None
 
     def get_reader(self, data_frequency, path=None):
         """
@@ -91,7 +62,7 @@ class ExchangeBundle:
 
         """
         if path is None:
-            root = get_exchange_folder(self.exchange.name)
+            root = get_exchange_folder(self.exchange_name)
             path = BUNDLE_NAME_TEMPLATE.format(
                 root=root,
                 frequency=data_frequency
@@ -122,7 +93,7 @@ class ExchangeBundle:
         BcolzMinuteBarWriter | BcolzDailyBarWriter
 
         """
-        root = get_exchange_folder(self.exchange.name)
+        root = get_exchange_folder(self.exchange_name)
         path = BUNDLE_NAME_TEMPLATE.format(
             root=root,
             frequency=data_frequency
@@ -180,9 +151,9 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
             The assets is scope.
-        start_dt: datetime
+        start_dt: pd.Timestamp
             The chunk start date.
-        end_dt: datetime
+        end_dt: pd.Timestamp
             The chunk end date.
         data_frequency: str
 
@@ -231,8 +202,8 @@ class ExchangeBundle:
 
         Parameters
         ----------
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
         data_frequency: str
 
         Returns
@@ -244,8 +215,91 @@ class ExchangeBundle:
             if data_frequency == 'minute' \
             else self.calendar.sessions_in_range(start_dt, end_dt)
 
+    def _spot_empty_periods(self, ohlcv_df, asset, data_frequency,
+                            empty_rows_behavior):
+        problems = []
+
+        nan_rows = ohlcv_df[ohlcv_df.isnull().T.any().T].index
+        if len(nan_rows) > 0:
+            dates = []
+            for row_date in nan_rows.values:
+                row_date = pd.to_datetime(row_date, utc=True)
+                if row_date > asset.start_date:
+                    dates.append(row_date)
+
+            if len(dates) > 0:
+                end_dt = asset.end_minute if data_frequency == 'minute' \
+                    else asset.end_daily
+
+                problem = '{name} ({start_dt} to {end_dt}) has empty ' \
+                          'periods: {dates}'.format(
+                    name=asset.symbol,
+                    start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                    dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
+                )
+                if empty_rows_behavior == 'warn':
+                    log.warn(problem)
+
+                elif empty_rows_behavior == 'raise':
+                    raise EmptyValuesInBundleError(
+                        name=asset.symbol,
+                        end_minute=end_dt,
+                        dates=dates
+                    )
+
+                else:
+                    ohlcv_df.dropna(inplace=True)
+
+            else:
+                problem = None
+
+            problems.append(problem)
+
+        return problems
+
+    def _spot_duplicates(self, ohlcv_df, asset, data_frequency, threshold):
+        # TODO: work in progress
+        series = ohlcv_df.reset_index().groupby('close')['index'].apply(
+            np.array
+        )
+
+        ref_delta = timedelta(minutes=1) if data_frequency == 'minute' \
+            else timedelta(days=1)
+
+        dups = series.loc[lambda values: [len(x) > 10 for x in values]]
+
+        for index, dates in dups.iteritems():
+            prev_date = None
+            for date in dates:
+                if prev_date is not None:
+                    delta = (date - prev_date) / 1e9
+                    if delta == ref_delta.seconds:
+                        log.info('pex')
+
+                prev_date = date
+
+        problems = []
+        for index, dates in dups.iteritems():
+            end_dt = asset.end_minute if data_frequency == 'minute' \
+                else asset.end_daily
+
+            problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
+                      'identical close values on: {dates}'.format(
+                name=asset.symbol,
+                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                threshold=threshold,
+                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                       for date in dates]
+            )
+
+            problems.append(problem)
+
+        return problems
+
     def ingest_df(self, ohlcv_df, data_frequency, asset, writer,
-                  empty_rows_behavior='strip'):
+                  empty_rows_behavior='warn', duplicates_threshold=None):
         """
         Ingest a DataFrame of OHLCV data for a given market.
 
@@ -258,50 +312,16 @@ class ExchangeBundle:
         empty_rows_behavior: str
 
         """
+        problems = []
         if empty_rows_behavior is not 'ignore':
-            nan_rows = ohlcv_df[ohlcv_df.isnull().T.any().T].index
+            problems += self._spot_empty_periods(
+                ohlcv_df, asset, data_frequency, empty_rows_behavior
+            )
 
-            if len(nan_rows) > 0:
-                dates = []
-                previous_date = None
-                for row_date in nan_rows.values:
-                    row_date = pd.to_datetime(row_date)
-
-                    if previous_date is None:
-                        dates.append(row_date)
-
-                    else:
-                        seq_date = previous_date + get_delta(1, data_frequency)
-
-                        if row_date > seq_date:
-                            dates.append(previous_date)
-                            dates.append(row_date)
-
-                    previous_date = row_date
-
-                dates.append(pd.to_datetime(nan_rows.values[-1]))
-
-                name = '{} from {} to {}'.format(
-                    asset.symbol, ohlcv_df.index[0], ohlcv_df.index[-1]
-                )
-                if empty_rows_behavior == 'warn':
-                    log.warn(
-                        '\n{name} with end minute {end_minute} has empty rows '
-                        'in ranges: {dates}'.format(
-                            name=name,
-                            end_minute=asset.end_minute,
-                            dates=dates
-                        )
-                    )
-
-                elif empty_rows_behavior == 'raise':
-                    raise EmptyValuesInBundleError(
-                        name=name,
-                        end_minute=asset.end_minute,
-                        dates=dates
-                    )
-                else:
-                    ohlcv_df.dropna(inplace=True)
+        # if duplicates_threshold is not None:
+        #     problems += self._spot_duplicates(
+        #         ohlcv_df, asset, data_frequency, duplicates_threshold
+        #     )
 
         data = []
         if not ohlcv_df.empty:
@@ -310,8 +330,11 @@ class ExchangeBundle:
 
         self._write(data, writer, data_frequency)
 
+        return problems
+
     def ingest_ctable(self, asset, data_frequency, period,
-                      writer, empty_rows_behavior='strip', cleanup=False):
+                      writer, empty_rows_behavior='strip',
+                      duplicates_threshold=100, cleanup=False):
         """
         Merge a ctable bundle chunk into the main bundle for the exchange.
 
@@ -327,11 +350,17 @@ class ExchangeBundle:
         cleanup: bool
             Remove the temp bundle directory after ingestion.
 
-        :return:
+        Returns
+        -------
+        list[str]
+            A list of problems which occurred during ingestion.
+
         """
+        problems = []
+
         # Download and extract the bundle
         path = get_bcolz_chunk(
-            exchange_name=self.exchange.name,
+            exchange_name=self.exchange_name,
             symbol=asset.symbol,
             data_frequency=data_frequency,
             period=period
@@ -375,12 +404,13 @@ class ExchangeBundle:
             start_dt, end_dt, data_frequency
         )
         df = get_df_from_arrays(arrays, periods)
-        self.ingest_df(
+        problems += self.ingest_df(
             ohlcv_df=df,
             data_frequency=data_frequency,
             asset=asset,
             writer=writer,
-            empty_rows_behavior=empty_rows_behavior
+            empty_rows_behavior=empty_rows_behavior,
+            duplicates_threshold=duplicates_threshold
         )
 
         if cleanup:
@@ -390,7 +420,7 @@ class ExchangeBundle:
             )
             shutil.rmtree(reader._rootdir)
 
-        return reader._rootdir
+        return filter(partial(is_not, None), problems)
 
     def get_adj_dates(self, start, end, assets, data_frequency):
         """
@@ -399,14 +429,14 @@ class ExchangeBundle:
 
         Parameters
         ----------
-        start: datetime
-        end: datetime
+        start: pd.Timestamp
+        end: pd.Timestamp
         assets: list[TradingPair]
         data_frequency: str
 
         Returns
         -------
-        datetime, datetime
+        pd.Timestamp, pd.Timestamp
         """
         earliest_trade = None
         last_entry = None
@@ -433,9 +463,10 @@ class ExchangeBundle:
             start = earliest_trade
 
         if end is None or (last_entry is not None and end > last_entry):
-            end = last_entry
+            end = last_entry.replace(minute=59, hour=23) \
+                if data_frequency == 'minute' else last_entry
 
-        if end is None or start is None or start >= end:
+        if end is None or start is None or start > end:
             raise NoDataAvailableOnExchange(
                 exchange=[asset.exchange for asset in assets],
                 symbol=[asset.symbol for asset in assets],
@@ -453,8 +484,8 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
         data_frequency: str
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
 
         Returns
         -------
@@ -528,7 +559,8 @@ class ExchangeBundle:
         return chunks
 
     def ingest_assets(self, assets, data_frequency, start_dt=None, end_dt=None,
-                      show_progress=False, asset_chunks=False):
+                      show_progress=False, show_breakdown=False,
+                      show_report=False):
         """
         Determine if data is missing from the bundle and attempt to ingest it.
 
@@ -536,10 +568,10 @@ class ExchangeBundle:
         ----------
         assets: list[TradingPair]
         data_frequency: str
-        start_dt: datetime
-        end_dt: datetime
+        start_dt: pd.Timestamp
+        end_dt: pd.Timestamp
         show_progress: bool
-        asset_chunks: bool
+        show_breakdown: bool
 
         """
         if start_dt is None:
@@ -562,22 +594,23 @@ class ExchangeBundle:
             end_dt=end_dt
         )
 
+        problems = []
         # This is the common writer for the entire exchange bundle
         # we want to give an end_date far in time
         writer = self.get_writer(start_dt, end_dt, data_frequency)
-        if asset_chunks:
+        if show_breakdown:
             for asset in chunks:
                 with maybe_show_progress(
                         chunks[asset],
                         show_progress,
                         label='Ingesting {frequency} price data for '
                               '{symbol} on {exchange}'.format(
-                            exchange=self.exchange.name,
+                            exchange=self.exchange_name,
                             frequency=data_frequency,
                             symbol=asset.symbol
                         )) as it:
                     for chunk in it:
-                        self.ingest_ctable(
+                        problems += self.ingest_ctable(
                             asset=chunk['asset'],
                             data_frequency=data_frequency,
                             period=chunk['period'],
@@ -597,11 +630,11 @@ class ExchangeBundle:
                     show_progress,
                     label='Ingesting {frequency} price data on '
                           '{exchange}'.format(
-                        exchange=self.exchange.name,
+                        exchange=self.exchange_name,
                         frequency=data_frequency,
                     )) as it:
                 for chunk in it:
-                    self.ingest_ctable(
+                    problems += self.ingest_ctable(
                         asset=chunk['asset'],
                         data_frequency=data_frequency,
                         period=chunk['period'],
@@ -610,9 +643,148 @@ class ExchangeBundle:
                         cleanup=True
                     )
 
+        if show_report and len(problems) > 0:
+            log.info('problems during ingestion:{}\n'.format(
+                '\n'.join(problems)
+            ))
+
+    def ingest_csv(self, path, data_frequency, empty_rows_behavior='strip',
+                   duplicates_threshold=100):
+        """
+        Ingest price data from a CSV file.
+
+        Parameters
+        ----------
+        path: str
+        data_frequency: str
+
+        Returns
+        -------
+        list[str]
+            A list of potential problems detected during ingestion.
+
+        """
+        log.info('ingesting csv file: {}'.format(path))
+
+        if self.exchange is None:
+            # Avoid circular dependencies
+            from catalyst.exchange.factory import get_exchange
+            self.exchange = get_exchange(self.exchange_name)
+
+        problems = []
+        df = pd.read_csv(
+            path,
+            header=0,
+            sep=',',
+            dtype=dict(
+                symbol=np.object_,
+                last_traded=np.object_,
+                open=np.float64,
+                high=np.float64,
+                close=np.float64,
+                volume=np.float64
+            ),
+            parse_dates=['last_traded'],
+            index_col=None
+        )
+        min_start_dt = None
+        max_end_dt = None
+
+        symbols = df['symbol'].unique()
+
+        # Apply the timezone before creating an index for simplicity
+        df['last_traded'] = df['last_traded'].dt.tz_localize(pytz.UTC)
+        df.set_index(['symbol', 'last_traded'], drop=True, inplace=True)
+
+        assets = dict()
+        for symbol in symbols:
+            start_dt = df.index.get_level_values(1).min()
+            end_dt = df.index.get_level_values(1).max()
+            end_dt_key = 'end_{}'.format(data_frequency)
+
+            market = self.exchange.get_market(symbol)
+            if market is None:
+                raise ValueError('symbol not available in the exchange.')
+
+            params = dict(
+                exchange=self.exchange.name,
+                data_source='local',
+                exchange_symbol=market['id'],
+            )
+            mixin_market_params(self.exchange_name, params, market)
+
+            asset_def = self.exchange.get_asset_def(market, True)
+            if asset_def is not None:
+                params['symbol'] = asset_def['symbol']
+
+                params['start_date'] = asset_def['start_date'] \
+                    if asset_def['start_date'] < start_dt else start_dt
+
+                params['end_date'] = asset_def[end_dt_key] \
+                    if asset_def[end_dt_key] > end_dt else end_dt
+
+                params['end_daily'] = end_dt \
+                    if data_frequency == 'daily' else asset_def['end_daily']
+
+                params['end_minute'] = end_dt \
+                    if data_frequency == 'minute' else asset_def['end_minute']
+
+            else:
+                params['symbol'] = self.exchange.get_catalyst_symbol(market)
+
+                params['end_daily'] = end_dt \
+                    if data_frequency == 'daily' else 'N/A'
+                params['end_minute'] = end_dt \
+                    if data_frequency == 'minute' else 'N/A'
+
+            if min_start_dt is None or start_dt < min_start_dt:
+                min_start_dt = start_dt
+
+            if max_end_dt is None or end_dt > max_end_dt:
+                max_end_dt = end_dt
+
+            asset = TradingPair(**params)
+            assets[market['id']] = asset
+
+        save_exchange_symbols(self.exchange_name, assets, True)
+
+        writer = self.get_writer(
+            start_dt=min_start_dt.replace(hour=00, minute=00),
+            end_dt=max_end_dt.replace(hour=23, minute=59),
+            data_frequency=data_frequency
+        )
+
+        for symbol in assets:
+            asset = assets[symbol]
+            ohlcv_df = df.loc[
+                (df.index.get_level_values(0) == symbol)
+            ]  # type: pd.DataFrame
+            ohlcv_df.index = ohlcv_df.index.droplevel(0)
+
+            period_start = start_dt.replace(hour=00, minute=00)
+            period_end = end_dt.replace(hour=23, minute=59)
+            periods = self.get_calendar_periods_range(
+                period_start, period_end, data_frequency
+            )
+
+            # We're not really resampling but ensuring that each frame
+            # contains data
+            ohlcv_df = ohlcv_df.reindex(periods, method='ffill')
+            ohlcv_df['volume'] = ohlcv_df['volume'].fillna(0)
+
+            problems += self.ingest_df(
+                ohlcv_df=ohlcv_df,
+                data_frequency=data_frequency,
+                asset=asset,
+                writer=writer,
+                empty_rows_behavior=empty_rows_behavior,
+                duplicates_threshold=duplicates_threshold
+            )
+        return filter(partial(is_not, None), problems)
+
     def ingest(self, data_frequency, include_symbols=None,
-               exclude_symbols=None, start=None, end=None,
-               show_progress=True, environ=os.environ):
+               exclude_symbols=None, start=None, end=None, csv=None,
+               show_progress=True, show_breakdown=True, show_report=True):
         """
         Inject data based on specified parameters.
 
@@ -621,17 +793,34 @@ class ExchangeBundle:
         data_frequency: str
         include_symbols: str
         exclude_symbols: str
-        start: datetime
-        end: datetime
+        start: pd.Timestamp
+        end: pd.Timestamp
         show_progress: bool
         environ:
 
         """
-        assets = self.get_assets(include_symbols, exclude_symbols)
+        if csv is not None:
+            self.ingest_csv(csv, data_frequency)
 
-        for frequency in data_frequency.split(','):
-            self.ingest_assets(assets, frequency, start, end,
-                               show_progress, True)
+        else:
+            if self.exchange is None:
+                # Avoid circular dependencies
+                from catalyst.exchange.factory import get_exchange
+                self.exchange = get_exchange(self.exchange_name)
+
+            assets = get_assets(
+                self.exchange, include_symbols, exclude_symbols
+            )
+            for frequency in data_frequency.split(','):
+                self.ingest_assets(
+                    assets=assets,
+                    data_frequency=frequency,
+                    start_dt=start,
+                    end_dt=end,
+                    show_progress=show_progress,
+                    show_breakdown=show_breakdown,
+                    show_report=show_report
+                )
 
     def get_history_window_series_and_load(self,
                                            assets,
@@ -639,7 +828,9 @@ class ExchangeBundle:
                                            bar_count,
                                            field,
                                            data_frequency,
-                                           algo_end_dt=None
+                                           algo_end_dt=None,
+                                           trailing_bar_count=None,
+                                           force_auto_ingest=False
                                            ):
         """
         Retrieve price data history, ingest missing data.
@@ -647,55 +838,69 @@ class ExchangeBundle:
         Parameters
         ----------
         assets: list[TradingPair]
-        end_dt: datetime
+        end_dt: pd.Timestamp
         bar_count: int
         field: str
         data_frequency: str
-        algo_end_dt: datetime
+        algo_end_dt: pd.Timestamp
 
         Returns
         -------
         Series
 
         """
-        try:
+        if AUTO_INGEST or force_auto_ingest:
+            try:
+                series = self.get_history_window_series(
+                    assets=assets,
+                    end_dt=end_dt,
+                    bar_count=bar_count,
+                    field=field,
+                    data_frequency=data_frequency,
+                    trailing_bar_count=trailing_bar_count,
+                )
+                return pd.DataFrame(series)
+
+            except PricingDataNotLoadedError:
+                start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+                log.info(
+                    'pricing data for {symbol} not found in range '
+                    '{start} to {end}, updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        start=start_dt,
+                        end=end_dt
+                    )
+                )
+                self.ingest_assets(
+                    assets=assets,
+                    start_dt=start_dt,
+                    end_dt=algo_end_dt,  # TODO: apply trailing bars
+                    data_frequency=data_frequency,
+                    show_progress=True,
+                    show_breakdown=True
+                )
+                series = self.get_history_window_series(
+                    assets=assets,
+                    end_dt=end_dt,
+                    bar_count=bar_count,
+                    field=field,
+                    data_frequency=data_frequency,
+                    reset_reader=True,
+                    trailing_bar_count=trailing_bar_count,
+                )
+                return series
+
+        else:
             series = self.get_history_window_series(
                 assets=assets,
                 end_dt=end_dt,
                 bar_count=bar_count,
                 field=field,
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                trailing_bar_count=trailing_bar_count,
             )
             return pd.DataFrame(series)
 
-        except PricingDataNotLoadedError:
-            start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-            log.info(
-                'pricing data for {symbol} not found in range '
-                '{start} to {end}, updating the bundles.'.format(
-                    symbol=[asset.symbol for asset in assets],
-                    start=start_dt,
-                    end=end_dt
-                )
-            )
-            self.ingest_assets(
-                assets=assets,
-                start_dt=start_dt,
-                end_dt=algo_end_dt,
-                data_frequency=data_frequency,
-                show_progress=True,
-                asset_chunks=True
-            )
-            series = self.get_history_window_series(
-                assets=assets,
-                end_dt=end_dt,
-                bar_count=bar_count,
-                field=field,
-                data_frequency=data_frequency,
-                reset_reader=False
-            )
-            return series
-
     def get_spot_values(self,
                         assets,
                         field,
@@ -707,12 +912,18 @@ class ExchangeBundle:
         The spot values for the gives assets, field and date. Reads from
         the exchange data bundle.
 
-        :param assets:
-        :param field:
-        :param dt:
-        :param data_frequency:
-        :param reset_reader:
-        :return:
+        Parameters
+        ----------
+        assets: list[TradingPair]
+        field: str
+        dt: pd.Timestamp
+        data_frequency: str
+        reset_reader:
+
+        Returns
+        -------
+        float
+
         """
         values = []
         try:
@@ -736,10 +947,12 @@ class ExchangeBundle:
             raise PricingDataNotLoadedError(
                 field=field,
                 first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
+                exchange=self.exchange_name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=dt,
+                end_dt=dt
             )
 
     def get_history_window_series(self,
@@ -748,12 +961,20 @@ class ExchangeBundle:
                                   bar_count,
                                   field,
                                   data_frequency,
+                                  trailing_bar_count=None,
                                   reset_reader=False):
-        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-        start_dt, end_dt = self.get_adj_dates(
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
+        start_dt, _ = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
 
+        if trailing_bar_count:
+            delta = get_delta(trailing_bar_count, data_frequency)
+            end_dt += delta
+
+        # This is an attempt to resolve some caching with the reader
+        # when auto-ingesting data.
+        # TODO: needs more work
         reader = self.get_reader(data_frequency)
         if reset_reader:
             del self._readers[reader._rootdir]
@@ -764,59 +985,69 @@ class ExchangeBundle:
             raise PricingDataNotLoadedError(
                 field=field,
                 first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
+                exchange=self.exchange_name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=start_dt,
+                end_dt=end_dt
             )
 
+        series = dict()
         for asset in assets:
-            asset_start_dt, asset_end_dt = self.get_adj_dates(
+            asset_start_dt, _ = self.get_adj_dates(
                 start_dt, end_dt, assets, data_frequency
             )
-
             in_bundle = range_in_bundle(
-                asset, asset_start_dt, asset_end_dt, reader
+                asset, asset_start_dt, end_dt, reader
             )
             if not in_bundle:
                 raise PricingDataNotLoadedError(
                     field=field,
                     first_trading_day=asset.start_date,
-                    exchange=self.exchange.name,
+                    exchange=self.exchange_name,
                     symbols=asset.symbol,
                     symbol_list=asset.symbol,
-                    data_frequency=data_frequency
+                    data_frequency=data_frequency,
+                    start_dt=asset_start_dt,
+                    end_dt=end_dt
                 )
 
-        series = dict()
-        try:
+            periods = self.get_calendar_periods_range(
+                asset_start_dt, end_dt, data_frequency
+            )
+            # This does not behave well when requesting multiple assets
+            # when the start or end date of one asset is outside of the range
+            # looking at the logic in load_raw_arrays(), we are not achieving
+            # any performance gain by requesting multiple sids at once. It's
+            # looping through the sids and making separate requests anyway.
             arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
+                sids=[asset.sid],
                 fields=[field],
                 start_dt=start_dt,
                 end_dt=end_dt
             )
+            if len(arrays) == 0:
+                raise DataCorruptionError(
+                    exchange=self.exchange_name,
+                    symbols=asset.symbol,
+                    start_dt=asset_start_dt,
+                    end_dt=end_dt
+                )
 
-        except Exception:
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=min([asset.start_date for asset in assets]),
-                exchange=self.exchange.name,
-                symbols=symbols,
-                symbol_list=','.join(symbols),
-                data_frequency=data_frequency
-            )
+            field_values = arrays[0][:, 0]
 
-        periods = self.get_calendar_periods_range(
-            start_dt, end_dt, data_frequency
-        )
-
-        for asset_index, asset in enumerate(assets):
-            asset_values = arrays[asset_index]
-
-            value_series = pd.Series(asset_values.flatten(), index=periods)
-            series[asset] = value_series
+            try:
+                value_series = pd.Series(field_values, index=periods)
+                series[asset] = value_series
+            except ValueError as e:
+                raise PricingDataValueError(
+                    exchange=asset.exchange,
+                    symbol=asset.symbol,
+                    start_dt=asset_start_dt,
+                    end_dt=end_dt,
+                    error=e
+                )
 
         return series
 
@@ -830,14 +1061,18 @@ class ExchangeBundle:
 
         """
         log.debug('cleaning exchange {}, frequency {}'.format(
-            self.exchange.name, data_frequency
+            self.exchange_name, data_frequency
         ))
-        root = get_exchange_folder(self.exchange.name)
+        root = get_exchange_folder(self.exchange_name)
 
         symbols = os.path.join(root, 'symbols.json')
         if os.path.isfile(symbols):
             os.remove(symbols)
 
+        local_symbols = os.path.join(root, 'symbols_local.json')
+        if os.path.isfile(local_symbols):
+            os.remove(local_symbols)
+
         temp_bundles = os.path.join(root, 'temp_bundles')
 
         if os.path.isdir(temp_bundles):

catalyst/exchange/exchange_data_portal.py

@@ -6,7 +6,7 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
+from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
@@ -21,7 +21,6 @@ log = Logger('DataPortalExchange', level=LOG_LEVEL)
 class DataPortalExchangeBase(DataPortal):
     def __init__(self, *args, **kwargs):
 
-        self.exchanges = kwargs.pop('exchanges', None)
         # TODO: put somewhere accessible by each algo
         self.retry_get_history_window = 5
         self.retry_get_spot_value = 5
@@ -49,11 +48,10 @@ class DataPortalExchangeBase(DataPortal):
             if len(exchange_assets) > 1:
                 df_list = []
                 for exchange_name in exchange_assets:
-                    exchange = self.exchanges[exchange_name]
                     assets = exchange_assets[exchange_name]
 
                     df_exchange = self.get_exchange_history_window(
-                        exchange,
+                        exchange_name,
                         assets,
                         end_dt,
                         bar_count,
@@ -68,9 +66,9 @@ class DataPortalExchangeBase(DataPortal):
                 return pd.concat(df_list)
 
             else:
-                exchange = self.exchanges[list(exchange_assets.keys())[0]]
+                exchange_name = list(exchange_assets.keys())[0]
                 return self.get_exchange_history_window(
-                    exchange,
+                    exchange_name,
                     assets,
                     end_dt,
                     bar_count,
@@ -122,7 +120,7 @@ class DataPortalExchangeBase(DataPortal):
 
     @abc.abstractmethod
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -136,9 +134,8 @@ class DataPortalExchangeBase(DataPortal):
                         attempt_index=0):
         try:
             if isinstance(assets, TradingPair):
-                exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
-                    exchange, [assets], field, dt, data_frequency)
+                    assets.exchange, [assets], field, dt, data_frequency)
 
                 if not spot_values:
                     return np.nan
@@ -154,17 +151,16 @@ class DataPortalExchangeBase(DataPortal):
                     exchange_assets[asset.exchange].append(asset)
 
                 if len(list(exchange_assets.keys())) == 1:
-                    exchange = self.exchanges[list(exchange_assets.keys())[0]]
+                    exchange_name = list(exchange_assets.keys())[0]
                     return self.get_exchange_spot_value(
-                        exchange, assets, field, dt, data_frequency)
+                        exchange_name, assets, field, dt, data_frequency)
 
                 else:
                     spot_values = []
                     for exchange_name in exchange_assets:
-                        exchange = self.exchanges[exchange_name]
                         assets = exchange_assets[exchange_name]
                         exchange_spot_values = self.get_exchange_spot_value(
-                            exchange,
+                            exchange_name,
                             assets,
                             field,
                             dt,
@@ -199,7 +195,7 @@ class DataPortalExchangeBase(DataPortal):
         return self._get_spot_value(assets, field, dt, data_frequency)
 
     @abc.abstractmethod
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
         return
 
@@ -214,10 +210,11 @@ class DataPortalExchangeBase(DataPortal):
 
 class DataPortalExchangeLive(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchanges = kwargs.pop('exchanges', None)
         super(DataPortalExchangeLive, self).__init__(*args, **kwargs)
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -230,7 +227,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: Exchange
         assets: list[TradingPair]
         end_dt: datetime
         bar_count: int
@@ -244,6 +241,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
         DataFrame
 
         """
+        exchange = self.exchanges[exchange_name]
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -254,14 +252,14 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
             ffill)
         return df
 
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
+    def get_exchange_spot_value(self, exchange_name, assets, field, dt,
                                 data_frequency):
         """
         A spot value for the exchange.
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: str
         assets: list[TradingPair]
         field: str
         dt: datetime
@@ -272,6 +270,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
         float
 
         """
+        exchange = self.exchanges[exchange_name]
         exchange_spot_values = exchange.get_spot_value(
             assets, field, dt, data_frequency)
 
@@ -280,16 +279,16 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
 class DataPortalExchangeBacktest(DataPortalExchangeBase):
     def __init__(self, *args, **kwargs):
+        self.exchange_names = kwargs.pop('exchange_names', None)
+
         super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs)
 
         self.exchange_bundles = dict()
-
         self.history_loaders = dict()
         self.minute_history_loaders = dict()
 
-        for exchange_name in self.exchanges:
-            exchange = self.exchanges[exchange_name]
-            self.exchange_bundles[exchange_name] = ExchangeBundle(exchange)
+        for name in self.exchange_names:
+            self.exchange_bundles[name] = ExchangeBundle(name)
 
     def _get_first_trading_day(self, assets):
         first_date = None
@@ -299,7 +298,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         return first_date
 
     def get_exchange_history_window(self,
-                                    exchange,
+                                    exchange_name,
                                     assets,
                                     end_dt,
                                     bar_count,
@@ -326,12 +325,13 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         DataFrame
 
         """
-        bundle = self.exchange_bundles[exchange.name]  # type: ExchangeBundle
+        bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
             frequency, data_frequency
         )
         adj_bar_count = candle_size * bar_count
+        trailing_bar_count = candle_size - 1
 
         if data_frequency == 'minute' and adj_data_frequency == 'daily':
             end_dt = end_dt.floor('1D')
@@ -343,13 +343,14 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
+            trailing_bar_count=trailing_bar_count
         )
 
         df = resample_history_df(pd.DataFrame(series), freq, field)
         return df
 
     def get_exchange_spot_value(self,
-                                exchange,
+                                exchange_name,
                                 assets,
                                 field,
                                 dt,
@@ -361,7 +362,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
 
         Parameters
         ----------
-        exchange: Exchange
+        exchange_name: str
         assets: list[TradingPair]
         field: str
         dt: datetime
@@ -372,30 +373,34 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         float
 
         """
-        bundle = self.exchange_bundles[exchange.name]
+        bundle = self.exchange_bundles[exchange_name]
         if data_frequency == 'daily':
             dt = dt.floor('1D')
         else:
             dt = dt.floor('1 min')
 
-        try:
-            return bundle.get_spot_values(assets, field, dt, data_frequency)
-
-        except PricingDataNotLoadedError:
-            log.info(
-                'pricing data for {symbol} not found on {dt}'
-                ', updating the bundles.'.format(
-                    symbol=[asset.symbol for asset in assets],
-                    dt=dt
+        if AUTO_INGEST:
+            try:
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency
                 )
-            )
-            bundle.ingest_assets(
-                assets=assets,
-                start_dt=self._first_trading_day,
-                end_dt=self._last_available_session,
-                data_frequency=data_frequency,
-                show_progress=True
-            )
-            return bundle.get_spot_values(
-                assets, field, dt, data_frequency, True
-            )
+            except PricingDataNotLoadedError:
+                log.info(
+                    'pricing data for {symbol} not found on {dt}'
+                    ', updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        dt=dt
+                    )
+                )
+                bundle.ingest_assets(
+                    assets=assets,
+                    start_dt=self._first_trading_day,
+                    end_dt=self._last_available_session,
+                    data_frequency=data_frequency,
+                    show_progress=True
+                )
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency, True
+                )
+        else:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -211,12 +211,24 @@ class PricingDataBeforeTradingError(ZiplineError):
 
 
 class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Pricing data {field} for trading pairs {symbols} trading on '
-           'exchange {exchange} since {first_trading_day} is unavailable. '
-           'The bundle data is either out-of-date or has not been loaded yet. '
-           'Please ingest data using the command '
-           '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. '
-           'See catalyst documentation for details.').strip()
+    msg = ('Missing data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]'
+           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
+           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
+           'for details.').strip()
+
+class PricingDataValueError(ZiplineError):
+    msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
+           '[{start_dt} - {end_dt}]: {error}').strip()
+
+
+class DataCorruptionError(ZiplineError):
+    msg = ('Unable to validate data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]. The data is either corrupted or '
+           'unavailable. Please try deleting this bundle:'
+           '\n`catalyst clean-exchange -x {exchange}\n'
+           'Then, ingest the data again. Please contact the Catalyst team if '
+           'the issue persists.').strip()
 
 
 class ApiCandlesError(ZiplineError):
@@ -228,3 +240,7 @@ class NoDataAvailableOnExchange(ZiplineError):
         'Requested data for trading pair {symbol} is not available on exchange {exchange} '
         'in `{data_frequency}` frequency at this time. '
         'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+
+
+class NoValueForField(ZiplineError):
+    msg = ('Value not found for field: {field}.').strip()

catalyst/exchange/exchange_execution.py

@@ -1,4 +1,4 @@
-from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
+from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder, MarketOrder
 
 
 class ExchangeLimitOrder(LimitOrder):

catalyst/exchange/exchange_portfolio.py

@@ -3,7 +3,6 @@ from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
-from catalyst.utils.deprecate import deprecated
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
@@ -11,7 +10,8 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 class ExchangePortfolio(Portfolio):
     """
     Since the goal is to support multiple exchanges, it makes sense to
-    include additional stats in the portfolio object.
+    include additional stats in the portfolio object. This fills the role
+    of Blotter and Portfolio in live mode.
 
     Instead of relying on the performance tracker, each exchange portfolio
     tracks its own holding. This offers a separation between tracking an
@@ -89,32 +89,6 @@ class ExchangePortfolio(Portfolio):
 
         log.debug('updated portfolio with executed order')
 
-    @deprecated
-    def execute_transaction(self, transaction):
-        # TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
-        log.debug('executing transaction {}'.format(transaction.order_id))
-
-        order_position = self.positions[transaction.asset] \
-            if transaction.asset in self.positions else None
-
-        if order_position is None:
-            raise ValueError(
-                'Trying to execute transaction for a position not held: %s' % transaction.order_id
-            )
-
-        self.capital_used += transaction.amount * transaction.price
-
-        if transaction.amount > 0:
-            if order_position.cost_basis > 0:
-                order_position.cost_basis = np.average(
-                    [order_position.cost_basis, transaction.price],
-                    weights=[order_position.amount, transaction.amount]
-                )
-            else:
-                order_position.cost_basis = transaction.price
-
-        log.debug('updated portfolio with executed order')
-
     def remove_order(self, order):
         """
         Removing an open order.

catalyst/exchange/exchange_utils.py

@@ -1,20 +1,41 @@
+import hashlib
 import json
 import os
 import pickle
 import re
+import shutil
 from datetime import date, datetime
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
+from six import string_types
 from six.moves.urllib import request
 
+from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
 from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
     InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
-SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
-              '{exchange}/symbols.json'
+
+def get_sid(symbol):
+    """
+    Create a sid by hashing the symbol of a currency pair.
+
+    Parameters
+    ----------
+    symbol: str
+
+    Returns
+    -------
+    int
+        The resulting sid.
+
+    """
+    sid = int(
+        hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+    ) % 10 ** 6
+    return sid
 
 
 def get_exchange_folder(exchange_name, environ=None):
@@ -41,7 +62,7 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
-def get_exchange_symbols_filename(exchange_name, environ=None):
+def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
     """
     The absolute path of the exchange's symbol.json file.
 
@@ -55,8 +76,9 @@ def get_exchange_symbols_filename(exchange_name, environ=None):
     str
 
     """
+    name = 'symbols.json' if not is_local else 'symbols_local.json'
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    return os.path.join(exchange_folder, 'symbols.json')
+    return os.path.join(exchange_folder, name)
 
 
 def download_exchange_symbols(exchange_name, environ=None):
@@ -79,13 +101,28 @@ def download_exchange_symbols(exchange_name, environ=None):
     return response
 
 
-def get_exchange_symbols(exchange_name, environ=None):
+def symbols_parser(asset_def):
+    for key, value in asset_def.items():
+        match = isinstance(value, string_types) \
+                and re.search(r'(\d{4}-\d{2}-\d{2})', value)
+
+        if match:
+            try:
+                asset_def[key] = pd.to_datetime(value, utc=True)
+            except ValueError:
+                pass
+
+    return asset_def
+
+
+def get_exchange_symbols(exchange_name, is_local=False, environ=None):
     """
     The de-serialized content of the exchange's symbols.json.
 
     Parameters
     ----------
     exchange_name: str
+    is_local: bool
     environ:
 
     Returns
@@ -93,18 +130,21 @@ def get_exchange_symbols(exchange_name, environ=None):
     Object
 
     """
-    filename = get_exchange_symbols_filename(exchange_name)
+    filename = get_exchange_symbols_filename(exchange_name, is_local)
 
-    if not os.path.isfile(filename) or \
-                    pd.Timedelta(pd.Timestamp('now',
-                                              tz='UTC') - last_modified_time(
-                        filename)).days > 1:
+    if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
+                pd.Timestamp('now', tz='UTC') - last_modified_time(
+                filename)).days > 1):
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
         with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
+            try:
+                data = json.load(data_file, object_hook=symbols_parser)
+                return data
+
+            except ValueError as e:
+                return dict()
     else:
         raise ExchangeSymbolsNotFound(
             exchange=exchange_name,
@@ -112,6 +152,32 @@ def get_exchange_symbols(exchange_name, environ=None):
         )
 
 
+def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
+    """
+    Save assets into an exchange_symbols file.
+
+    Parameters
+    ----------
+    exchange_name: str
+    assets: list[dict[str, object]]
+    is_local: bool
+    environ
+
+    Returns
+    -------
+
+    """
+    asset_dicts = dict()
+    for symbol in assets:
+        asset_dicts[symbol] = assets[symbol].to_dict()
+
+    filename = get_exchange_symbols_filename(
+        exchange_name, is_local, environ
+    )
+    with open(filename, 'wt') as handle:
+        json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
+
+
 def get_symbols_string(assets):
     """
     A concatenated string of symbols from a list of assets.
@@ -158,6 +224,24 @@ def get_exchange_auth(exchange_name, environ=None):
             return data
 
 
+def delete_algo_folder(algo_name, environ=None):
+    """
+    Delete the folder containing the algo state.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
+    shutil.rmtree(folder)
+
+
 def get_algo_folder(algo_name, environ=None):
     """
     The algorithm root folder of the algorithm.
@@ -344,6 +428,25 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
     return temp_bundles
 
 
+def symbols_serial(obj):
+    """
+    JSON serializer for objects not serializable by default json code
+
+    Parameters
+    ----------
+    obj: Object
+
+    Returns
+    -------
+    str
+
+    """
+    if isinstance(obj, (datetime, date)):
+        return obj.floor('1D').strftime(DATE_FORMAT)
+
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
 def perf_serial(obj):
     """
     JSON serializer for objects not serializable by default json code
@@ -481,4 +584,46 @@ def resample_history_df(df, freq, field):
     else:
         raise ValueError('Invalid field.')
 
-    return df.resample(freq).agg(agg)
+    resampled_df = df.resample(freq).agg(agg)
+    return resampled_df
+
+
+def mixin_market_params(exchange_name, params, market):
+    """
+    Applies a CCXT market dict to parameters of TradingPair init.
+
+    Parameters
+    ----------
+    params: dict[Object]
+    market: dict[Object]
+
+    Returns
+    -------
+
+    """
+    # TODO: make this more externalized / configurable
+    if 'lot' in market:
+        params['min_trade_size'] = market['lot']
+
+    if exchange_name == 'bitfinex':
+        params['maker'] = 0.001
+        params['taker'] = 0.002
+
+    elif 'maker' in market and 'taker' in market \
+            and market['maker'] is not None and market['taker'] is not None:
+        params['maker'] = market['maker']
+        params['taker'] = market['taker']
+
+    else:
+        # TODO: default commission, make configurable
+        params['maker'] = 0.0015
+        params['taker'] = 0.0025
+
+    info = market['info'] if 'info' in market else None
+    if info:
+        if 'minimum_order_size' in info:
+            params['min_trade_size'] = float(info['minimum_order_size'])
+
+
+def from_ms_timestamp(ms):
+    return pd.to_datetime(ms, unit='ms', utc=True)

catalyst/exchange/factory.py

@@ -1,38 +1,31 @@
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.exchange_errors import ExchangeNotFoundError
-from catalyst.exchange.exchange_utils import get_exchange_auth
-from catalyst.exchange.poloniex.poloniex import Poloniex
+import os
+
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_exchange_folder
 
 
-def get_exchange(exchange_name, base_currency=None):
+def get_exchange(exchange_name, base_currency=None, portfolio=None,
+                 must_authenticate=False):
     exchange_auth = get_exchange_auth(exchange_name)
-    if exchange_name == 'bitfinex':
-        return Bitfinex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
+
+    has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
+    if must_authenticate and not has_auth:
+        raise ExchangeAuthEmpty(
+            exchange=exchange_name.title(),
+            filename=os.path.join(
+                get_exchange_folder(exchange_name), 'auth.json'
+            )
         )
 
-    elif exchange_name == 'bittrex':
-        return Bittrex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
-        )
-
-    elif exchange_name == 'poloniex':
-        return Poloniex(
-            key=exchange_auth['key'],
-            secret=exchange_auth['secret'],
-            base_currency=base_currency,
-            portfolio=None
-        )
-
-    else:
-        raise ExchangeNotFoundError(exchange_name=exchange_name)
+    return CCXT(
+        exchange_name=exchange_name,
+        key=exchange_auth['key'],
+        secret=exchange_auth['secret'],
+        base_currency=base_currency,
+        portfolio=portfolio
+    )
 
 
 def get_exchanges(exchange_names):

catalyst/exchange/poloniex/poloniex.py

@@ -35,8 +35,13 @@ class Poloniex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
         self.api = Poloniex_api(key=key, secret=secret)
         self.name = 'poloniex'
-        self.assets = {}
+
+        self.assets = dict()
         self.load_assets()
+
+        self.local_assets = dict()
+        self.load_assets(is_local=True)
+
         self.base_currency = base_currency
         self._portfolio = portfolio
         self.minute_writer = None
@@ -47,7 +52,7 @@ class Poloniex(Exchange):
         self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
-        self.bundle = ExchangeBundle(self)
+        self.bundle = ExchangeBundle(self.name)
 
     def sanitize_curency_symbol(self, exchange_symbol):
         """
@@ -226,10 +231,9 @@ class Poloniex(Exchange):
         ohlc_map = dict()
 
         for asset in asset_list:
+            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
+            end = int(delta.total_seconds())
 
-            # TODO: what's wrong with this?
-            # end = int(time.mktime(end_dt.timetuple()))
-            end = int(time.time())
             if bar_count is None:
                 start = end - 2 * frequency
             else:

catalyst/exchange/poloniex/poloniex_api.py

@@ -3,6 +3,7 @@ import json
 import time
 import hmac
 import hashlib
+import ssl
 
 from six.moves import urllib
 
@@ -104,9 +105,10 @@ class Poloniex_api(object):
         req = urllib.request.Request(
             url,
             data=post_data,
-            headers=headers
+            headers=headers,
         )
-        return json.loads(urlopen(req).read())
+        return json.loads(
+            urlopen(req, context=ssl._create_unverified_context()).read())
 
     def returnticker(self):
         return self.query('returnTicker', {})

catalyst/exchange/stats_utils.py

@@ -1,7 +1,19 @@
+import numbers
+
 import numpy as np
 import pandas as pd
 
 
+def trend_direction(series):
+    if series[-1] is np.nan or series[-1] is np.nan:
+        return None
+
+    if series[-1] > series[-2]:
+        return 'up'
+    else:
+        return 'down'
+
+
 def crossover(source, target):
     """
     The `x`-series is defined as having crossed over `y`-series if the value
@@ -18,14 +30,25 @@ def crossover(source, target):
     bool
 
     """
-    if source[-1] is np.nan or source[-2] is np.nan \
-            or target[-1] is np.nan or target[-2] is np.nan:
-        return False
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] >= target > source[-2]:
+            return True
+        else:
+            return False
 
-    if source[-1] > target[-1] and source[-2] < target[-2]:
-        return True
     else:
-        return False
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] > target[-1] and source[-2] < target[-2]:
+            return True
+        else:
+            return False
 
 
 def crossunder(source, target):
@@ -44,14 +67,56 @@ def crossunder(source, target):
     bool
 
     """
-    if source[-1] is np.nan or source[-2] is np.nan \
-            or target[-1] is np.nan or target[-2] is np.nan:
-        return False
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
 
-    if source[-1] < target[-1] and source[-2] > target[-2]:
-        return True
+        if source[-1] < target <= source[-2]:
+            return True
+        else:
+            return False
     else:
-        return False
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] < target[-1] and source[-2] >= target[-2]:
+            return True
+        else:
+            return False
+
+
+def vwap(df):
+    """
+    Volume-weighted average price (VWAP) is a ratio generally used by
+    institutional investors and mutual funds to make buys and sells so as not
+    to disturb the market prices with large orders. It is the average share
+    price of a stock weighted against its trading volume within a particular
+    time frame, generally one day.
+
+    Read more: Volume Weighted Average Price - VWAP
+    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+
+    Returns
+    -------
+
+    """
+    if 'close' not in df.columns or 'volume' not in df.columns:
+        raise ValueError('price data must include `volume` and `close`')
+
+    vol_sum = np.nansum(df['volume'].values)
+
+    try:
+        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
+    except ZeroDivisionError:
+        ret = np.nan
+
+    return ret
 
 
 def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
@@ -88,11 +153,11 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
     def format_positions(positions):
         parts = []
         for position in positions:
-            msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
+            msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format(
                 amount=position['amount'],
-                market=position['sid'].market_currency,
+                base=position['sid'].base_currency,
                 cost_basis=position['cost_basis'],
-                base=position['sid'].base_currency
+                quote=position['sid'].quote_currency
             )
             parts.append(msg)
         return ', '.join(parts)
@@ -129,3 +194,28 @@ def df_to_string(df):
     pd.set_option('display.max_colwidth', 1000)
 
     return df.to_string()
+
+
+def extract_transactions(perf):
+    """
+    Compute indexes for buy and sell transactions
+
+    Parameters
+    ----------
+    perf: DataFrame
+        The algo performance DataFrame.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of transactions.
+
+    """
+    trans_list = perf.transactions.values
+    all_trans = [t for sublist in trans_list for t in sublist]
+    all_trans.sort(key=lambda t: t['dt'])
+
+    transactions = pd.DataFrame(all_trans)
+    if not transactions.empty:
+        transactions.set_index('dt', inplace=True, drop=True)
+    return transactions

catalyst/exchange/validator.py

@@ -0,0 +1,142 @@
+import os
+import tempfile
+
+import pandas as pd
+import six
+from catalyst.assets._assets import TradingPair, get_calendar
+from logbook import Logger
+from pandas.util.testing import assert_frame_equal
+
+from catalyst.constants import LOG_LEVEL
+from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
+from catalyst.exchange.factory import get_exchanges
+from catalyst.utils.paths import ensure_directory
+
+log = Logger('Validator', level=LOG_LEVEL)
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path
+
+
+class Validator(object):
+    def __init__(self, data_portal):
+        self.data_portal = data_portal
+
+    def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
+                                     sample_minutes):
+        """
+        Creates DataFrames from the bundle and exchange for the specified
+        data set.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets
+        end_dt
+        bar_count
+        sample_minutes
+
+        Returns
+        -------
+
+        """
+        freq = '{}T'.format(sample_minutes)
+
+        log.info('creating data sample from bundle')
+        df1 = self.data_portal.get_history_window(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            frequency=freq,
+            field='close',
+            data_frequency='minute'
+        )
+        path = output_df(df1, assets, '{}_resampled'.format(freq))
+        log.info('saved resampled bundle candles: {}\n{}'.format(
+            path, df1.tail(10))
+        )
+
+        log.info('creating data sample from exchange api')
+        candles = exchange.get_candles(
+            end_dt=end_dt,
+            freq='{}T'.format(sample_minutes),
+            assets=assets,
+            bar_count=bar_count
+        )
+
+        series = dict()
+        for asset in assets:
+            series[asset] = pd.Series(
+                data=[candle['close'] for candle in candles[asset]],
+                index=[candle['last_traded'] for candle in candles[asset]]
+            )
+
+        df2 = pd.DataFrame(series)
+        path = output_df(df2, assets, '{}_api'.format(freq))
+        log.info('saved exchange api candles: {}\n{}'.format(
+            path, df2.tail(10))
+        )
+
+        try:
+            assert_frame_equal(df1, df2)
+            return True
+        except:
+            log.warn('differences found in dataframes')
+            return False
+
+
+if __name__ == '__main__':
+    exchanges = get_exchanges(['poloniex'])
+    exchange = six.next(six.itervalues(exchanges))
+    assets = exchange.get_assets(symbols=['eth_btc'])
+
+    open_calendar = get_calendar('OPEN')
+    asset_finder = AssetFinderExchange()
+    data_portal = DataPortalExchangeBacktest(
+        exchanges=exchanges,
+        asset_finder=asset_finder,
+        trading_calendar=open_calendar,
+        first_trading_day=None  # will set dynamically based on assets
+    )
+    validator = Validator(data_portal=data_portal)
+
+    validator.compare_bundle_with_exchange(
+        exchange=exchange,
+        assets=assets,
+        end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
+        bar_count=200,
+        sample_minutes=30
+    )

catalyst/finance/execution.py

@@ -77,6 +77,7 @@ class LimitOrder(ExecutionStyle):
     Execution style representing an order to be executed at a price equal to or
     better than a specified limit price.
     """
+
     def __init__(self, limit_price, exchange=None):
         """
         Store the given price.
@@ -99,6 +100,7 @@ class StopOrder(ExecutionStyle):
     Execution style representing an order to be placed once the market price
     reaches a specified stop price.
     """
+
     def __init__(self, stop_price, exchange=None):
         """
         Store the given price.
@@ -121,6 +123,7 @@ class StopLimitOrder(ExecutionStyle):
     Execution style representing a limit order to be placed with a specified
     limit price once the market reaches a specified stop price.
     """
+
     def __init__(self, limit_price, stop_price, exchange=None):
         """
         Store the given prices
@@ -144,31 +147,20 @@ class StopLimitOrder(ExecutionStyle):
 def asymmetric_round_price_to_penny(price, prefer_round_down,
                                     diff=(0.0095 - .005)):
     """
-    Asymmetric rounding function for adjusting prices to two places in a way
-    that "improves" the price.  For limit prices, this means preferring to
-    round down on buys and preferring to round up on sells.  For stop prices,
-    it means the reverse.
+    Modified the original function because we do not want to round
+    prices on crypto exchange.
 
-    If prefer_round_down == True:
-        When .05 below to .95 above a penny, use that penny.
-    If prefer_round_down == False:
-        When .95 below to .05 above a penny, use that penny.
+    Parameters
+    ----------
+    price: float
+
+    Returns
+    -------
+    float
 
-    In math-speak:
-    If prefer_round_down: [<X-1>.0095, X.0195) -> round to X.01.
-    If not prefer_round_down: (<X-1>.0005, X.0105] -> round to X.01.
     """
-    # Subtracting an epsilon from diff to enforce the open-ness of the upper
-    # bound on buys and the lower bound on sells.  Using the actual system
-    # epsilon doesn't quite get there, so use a slightly less epsilon-ey value.
-    epsilon = float_info.epsilon * 10
-    diff = diff - epsilon
-
-    # relies on rounding half away from zero, unlike numpy's bankers' rounding
-    rounded = round(price - (diff if prefer_round_down else -diff), 2)
-    if zp_math.tolerant_equals(rounded, 0.0):
-        return 0.0
-    return rounded
+    # TODO: consider overriding outside of the original function
+    return price
 
 
 def check_stoplimit_prices(price, label):

catalyst/finance/risk/cumulative.py

@@ -22,7 +22,7 @@ from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
-from . risk import (
+from .risk import (
     check_entry,
     choose_treasury
 )
@@ -37,12 +37,11 @@ from empyrical import (
     sharpe_ratio,
     sortino_ratio,
 )
-
+import warnings
 from catalyst.constants import LOG_LEVEL
 
 log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
-
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
 
@@ -145,6 +144,8 @@ class RiskMetricsCumulative(object):
         self.num_trading_days = 0
 
     def update(self, dt, algorithm_returns, benchmark_returns, leverage):
+        warnings.filterwarnings('error')
+
         # Keep track of latest dt for use in to_dict and other methods
         # that report current state.
         self.latest_dt = dt
@@ -191,9 +192,12 @@ class RiskMetricsCumulative(object):
             if len(self.benchmark_returns) == 1:
                 self.benchmark_returns = np.append(0.0, self.benchmark_returns)
 
-        self.benchmark_cumulative_returns[dt_loc] = cum_returns(
-            self.benchmark_returns
-        )[-1]
+        try:
+            self.benchmark_cumulative_returns[dt_loc] = cum_returns(
+                self.benchmark_returns
+            )[-1]
+        except Exception:
+            self.benchmark_cumulative_returns[dt_loc] = 0
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -268,10 +272,17 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.downside_risk[dt_loc] = downside_risk(
             self.algorithm_returns
         )
-        self.sortino[dt_loc] = sortino_ratio(
-            self.algorithm_returns,
-            _downside_risk=self.downside_risk[dt_loc]
-        )
+
+        try:
+            risk = self.downside_risk[dt_loc]
+            self.sortino[dt_loc] = sortino_ratio(
+                self.algorithm_returns,
+                _downside_risk=risk
+            )
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino[dt_loc] = 0
+
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
@@ -283,6 +294,8 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.
@@ -294,18 +307,18 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         rval = {
             'trading_days': self.num_trading_days,
             'benchmark_volatility':
-            self.benchmark_volatility[dt_loc],
+                self.benchmark_volatility[dt_loc],
             'algo_volatility':
-            self.algorithm_volatility[dt_loc],
+                self.algorithm_volatility[dt_loc],
             'treasury_period_return': self.treasury_period_return,
             # Though the two following keys say period return,
             # they would be more accurately called the cumulative return.
             # However, the keys need to stay the same, for now, for backwards
             # compatibility with existing consumers.
             'algorithm_period_return':
-            self.algorithm_cumulative_returns[dt_loc],
+                self.algorithm_cumulative_returns[dt_loc],
             'benchmark_period_return':
-            self.benchmark_cumulative_returns[dt_loc],
+                self.benchmark_cumulative_returns[dt_loc],
             'beta': self.beta[dt_loc],
             'alpha': self.alpha[dt_loc],
             'sharpe': self.sharpe[dt_loc],

catalyst/finance/risk/period.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import functools
+import warnings
 
 import logbook
 
@@ -23,7 +24,7 @@ import numpy as np
 import pandas as pd
 
 from . import risk
-from . risk import check_entry
+from .risk import check_entry
 
 from empyrical import (
     alpha_beta_aligned,
@@ -78,14 +79,20 @@ class RiskMetricsPeriod(object):
         self.calculate_metrics()
 
     def calculate_metrics(self):
-        self.benchmark_period_returns = \
-            cum_returns(self.benchmark_returns).iloc[-1]
+        warnings.filterwarnings('error')
+
+        try:
+            self.benchmark_period_returns = \
+                cum_returns(self.benchmark_returns).iloc[-1]
+        except Exception:
+            # TODO: why is there an error
+            self.benchmark_period_returns = 0
 
         self.algorithm_period_returns = \
             cum_returns(self.algorithm_returns).iloc[-1]
 
         if not self.algorithm_returns.index.equals(
-            self.benchmark_returns.index
+                self.benchmark_returns.index
         ):
             message = "Mismatch between benchmark_returns ({bm_count}) and \
             algorithm_returns ({algo_count}) in range {start} : {end}"
@@ -128,10 +135,17 @@ class RiskMetricsPeriod(object):
         self.downside_risk = downside_risk(
             self.algorithm_returns.values
         )
-        self.sortino = sortino_ratio(
-            self.algorithm_returns.values,
-            _downside_risk=self.downside_risk,
-        )
+
+        try:
+            risk = self.downside_risk
+            self.sortino = sortino_ratio(
+                self.algorithm_returns.values,
+                _downside_risk=risk,
+            )
+        except Exception:
+            # TODO: what causes it to error out?
+            self.sortino = 0
+
         self.information = information_ratio(
             self.algorithm_returns.values,
             self.benchmark_returns.values,
@@ -141,10 +155,12 @@ class RiskMetricsPeriod(object):
             self.benchmark_returns.values,
         )
         self.excess_return = self.algorithm_period_returns - \
-            self.treasury_period_return
+                             self.treasury_period_return
         self.max_drawdown = max_drawdown(self.algorithm_returns.values)
         self.max_leverage = self.calculate_max_leverage()
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/support/issue_47.py

@@ -1,6 +1,6 @@
 """
 Requires Catalyst version 0.3.0 or above
-Tested on Catalyst version 0.3.2
+Tested on Catalyst version 0.3.3
 
 These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
 You simply need to specify the exchange and the market that you want to focus on.
@@ -27,7 +27,7 @@ from catalyst.api import (
 def initialize(context):
     context.i = -1  # counts the minutes
     context.exchange = 'poloniex'  # must match the exchange specified in run_algorithm
-    context.base_currency = 'eth'  # must match the base currency specified in run_algorithm
+    context.base_currency = 'btc'  # must match the base currency specified in run_algorithm
 
 
 def handle_data(context, data):
@@ -56,21 +56,21 @@ def handle_data(context, data):
 
             # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
             # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
-            open = fill(data.history(coin, 'open', bar_count=lookback,
-                                     frequency='1m')).resample('30T').first()
+            opened = fill(data.history(coin, 'open', bar_count=lookback,
+                                       frequency='30T')).values
             high = fill(data.history(coin, 'high', bar_count=lookback,
-                                     frequency='1m')).resample('30T').max()
+                                     frequency='30T')).values
             low = fill(data.history(coin, 'low', bar_count=lookback,
-                                    frequency='1m')).resample('30T').min()
+                                    frequency='30T')).values
             close = fill(data.history(coin, 'price', bar_count=lookback,
-                                      frequency='1m')).resample('30T').last()
+                                      frequency='30T')).values
             volume = fill(data.history(coin, 'volume', bar_count=lookback,
-                                       frequency='1m')).resample('30T').sum()
+                                       frequency='30T')).values
 
             # close[-1] is the equivalent to current price
             # displays the minute price for each pair every 30 minutes
             print(
-            today, pair, open[-1], high[-1], low[-1], close[-1], volume[-1])
+            today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
 
             # ----------------------------------------------------------------------------------------------------------
             # -------------------------------------- Insert Your Strategy Here -----------------------------------------
@@ -82,7 +82,7 @@ def analyze(context=None, results=None):
 
 
 # Get the universe for a given exchange and a given base_currency market
-# Example: Poloniex BTC Market
+# Example: Poloniex btc Market
 def universe(context, lookback_date, current_date):
     json_symbols = get_exchange_symbols(
         context.exchange)  # get all the pairs for the exchange
@@ -103,7 +103,6 @@ def universe(context, lookback_date, current_date):
     universe_df = universe_df[universe_df.end_daily >= current_date]
     context.coins = symbols(
         *universe_df.symbol)  # convert all the pairs to symbols
-    print(universe_df.head(), len(universe_df))
     return universe_df.symbol.tolist()
 
 
@@ -119,8 +118,8 @@ def fill(series):
 
 
 if __name__ == '__main__':
-    start_date = pd.to_datetime('2017-01-01', utc=True)
-    end_date = pd.to_datetime('2017-10-15', utc=True)
+    start_date = pd.to_datetime('2017-01-08', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
 
     performance = run_algorithm(start=start_date, end=end_date,
                                 capital_base=10000.0,
@@ -129,7 +128,7 @@ if __name__ == '__main__':
                                 analyze=analyze,
                                 exchange_name='poloniex',
                                 data_frequency='minute',
-                                base_currency='eth',
+                                base_currency='btc',
                                 live=False,
                                 live_graph=False,
                                 algo_namespace='simple_universe')

catalyst/support/issue_55.py

@@ -0,0 +1,42 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('xcp_btc')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=1,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2015-3-2', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='poloniex',
+    algo_namespace='issue_55',
+    base_currency='btc'
+)

catalyst/support/issue_57.py

@@ -0,0 +1,46 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=60,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2016-2-11', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='bittrex',
+    algo_namespace='issue_57',
+    base_currency='btc'
+<<<<<<< HEAD
+)
+=======
+)
+>>>>>>> develop

catalyst/support/issue_74.py

@@ -0,0 +1,127 @@
+from __future__ import division
+import os
+import pytz
+import numpy as np
+import pandas as pd
+from scipy.optimize import minimize
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+from catalyst.api import record, symbol, symbols, order_target_percent
+from catalyst.utils.run_algo import run_algorithm
+
+np.set_printoptions(threshold='nan', suppress=True)
+
+
+def initialize(context):
+    # Portfolio assets list
+    context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
+                             'xmr_usdt')
+    context.nassets = len(context.assets)
+    # Set the time window that will be used to compute expected return
+    # and asset correlations
+    context.window = 180
+    # Set the number of days between each portfolio rebalancing
+    context.rebalance_period = 30
+    context.i = 0
+
+
+def handle_data(context, data):
+    # Only rebalance at the beggining of the algorithm execution and
+    # every multiple of the rebalance period
+    if context.i == 0 or context.i % context.rebalance_period == 0:
+        n = context.window
+        prices = data.history(context.assets, fields='price',
+                           bar_count=n + 1, frequency='daily')
+        pr = np.asmatrix(prices)
+        t_prices = prices.iloc[1:n + 1]
+        t_val = t_prices.values
+        tminus_prices = prices.iloc[0:n]
+        tminus_val = tminus_prices.values
+        # Compute daily returns (r)
+        r = np.asmatrix(t_val / tminus_val - 1)
+        # Compute the expected returns of each asset with the average
+        # daily return for the selected time window
+        m = np.asmatrix(np.mean(r, axis=0))
+        # ###
+        stds = np.std(r, axis=0)
+        # Compute excess returns matrix (xr)
+        xr = r - m
+        # Matrix algebra to get variance-covariance matrix
+        cov_m = np.dot(np.transpose(xr), xr) / n
+        # Compute asset correlation matrix (informative only)
+        corr_m = cov_m / np.dot(np.transpose(stds), stds)
+
+        # Define portfolio optimization parameters
+        n_portfolios = 50000
+        results_array = np.zeros((3 + context.nassets, n_portfolios))
+        for p in xrange(n_portfolios):
+            weights = np.random.random(context.nassets)
+            weights /= np.sum(weights)
+            w = np.asmatrix(weights)
+            p_r = np.sum(np.dot(w, np.transpose(m))) * 365
+            p_std = np.sqrt(
+                np.dot(np.dot(w, cov_m), np.transpose(w))) * np.sqrt(365)
+
+            # store results in results array
+            results_array[0, p] = p_r
+            results_array[1, p] = p_std
+            # store Sharpe Ratio (return / volatility) - risk free rate element
+            # excluded for simplicity
+            results_array[2, p] = results_array[0, p] / results_array[1, p]
+            i = 0
+            for iw in weights:
+                results_array[3 + i, p] = weights[i]
+                i += 1
+
+        # convert results array to Pandas DataFrame
+        results_frame = pd.DataFrame(np.transpose(results_array),
+                                     columns=['r', 'stdev',
+                                              'sharpe'] + context.assets)
+        # locate position of portfolio with highest Sharpe Ratio
+        max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
+        # locate positon of portfolio with minimum standard deviation
+        min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+
+        # order optimal weights for each asset
+        for asset in context.assets:
+            if data.can_trade(asset):
+                order_target_percent(asset, max_sharpe_port[asset])
+
+        # create scatter plot coloured by Sharpe Ratio
+        plt.scatter(results_frame.stdev, results_frame.r,
+                    c=results_frame.sharpe, cmap='RdYlGn')
+        plt.xlabel('Volatility')
+        plt.ylabel('Returns')
+        plt.colorbar()
+        # plot red star to highlight position of portfolio with highest Sharpe Ratio
+        plt.scatter(max_sharpe_port[1], max_sharpe_port[0], marker='o',
+                    color='b', s=200)
+        # plot green star to highlight position of minimum variance portfolio
+        plt.show()
+        print(max_sharpe_port)
+        record(pr=pr, r=r, m=m, stds=stds, max_sharpe_port=max_sharpe_port,
+               corr_m=corr_m)
+    context.i += 1
+
+
+def analyze(context=None, results=None):
+    # Form DataFrame with selected data
+    data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
+                    'portfolio_value']]
+
+    # Save results in CSV file
+    filename = os.path.splitext(os.path.basename(__file__))[0]
+    data.to_csv(filename + '.csv')
+
+
+# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
+start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
+results = run_algorithm(initialize=initialize,
+                        handle_data=handle_data,
+                        analyze=analyze,
+                        start=start,
+                        end=end,
+                        exchange_name='poloniex',
+                        capital_base=100000, )

catalyst/testing/__init__.py

@@ -55,4 +55,4 @@ from .core import (  # noqa
     write_bcolz_minute_data,
     write_compressed,
 )
-from .fixtures import ZiplineTestCase  # noqa
+from .fixtures import CatalystTestCase  # noqa

catalyst/testing/core.py

@@ -643,7 +643,7 @@ def create_data_portal_from_trade_history(asset_finder, trading_calendar,
         return DataPortal(
             asset_finder, trading_calendar,
             first_trading_day=equity_daily_reader.first_trading_day,
-            equity_daily_reader=equity_daily_reader,
+            daily_reader=equity_daily_reader,
         )
     else:
         minutes = trading_calendar.minutes_in_range(

catalyst/testing/fixtures.py

@@ -62,7 +62,7 @@ from catalyst.utils.paths import ensure_directory
 catalyst_dir = os.path.dirname(catalyst.__file__)
 
 
-class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)):
+class CatalystTestCase(with_metaclass(FinalMeta, TestCase)):
     """
     Shared extensions to core unittest.TestCase.
 
@@ -92,7 +92,7 @@ class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)):
             cls._base_init_fixtures_was_called = False
             cls.init_class_fixtures()
             assert cls._base_init_fixtures_was_called, (
-                "ZiplineTestCase.init_class_fixtures() was not called.\n"
+                "CatalystTestCase.init_class_fixtures() was not called.\n"
                 "This probably means that you overrode init_class_fixtures"
                 " without calling super()."
             )
@@ -170,7 +170,7 @@ class ZiplineTestCase(with_metaclass(FinalMeta, TestCase)):
             self._init_instance_fixtures_was_called = False
             self.init_instance_fixtures()
             assert self._init_instance_fixtures_was_called, (
-                "ZiplineTestCase.init_instance_fixtures() was not"
+                "CatalystTestCase.init_instance_fixtures() was not"
                 " called.\n"
                 "This probably means that you overrode"
                 " init_instance_fixtures without calling super()."
@@ -251,7 +251,7 @@ def alias(attr_name):
 
 class WithDefaultDateBounds(object):
     """
-    ZiplineTestCase mixin which makes it possible to synchronize date bounds
+    CatalystTestCase mixin which makes it possible to synchronize date bounds
     across fixtures.
 
     This fixture should always be the last fixture in bases of any fixture or
@@ -264,13 +264,13 @@ class WithDefaultDateBounds(object):
         The date bounds to be used for fixtures that want to have consistent
         dates.
     """
-    START_DATE = pd.Timestamp('2006-01-03', tz='utc')
-    END_DATE = pd.Timestamp('2006-12-29', tz='utc')
+    START_DATE = pd.Timestamp('2016-01-03', tz='utc')
+    END_DATE = pd.Timestamp('2016-12-29', tz='utc')
 
 
 class WithLogger(object):
     """
-    ZiplineTestCase mixin providing cls.log_handler as an instance-level
+    CatalystTestCase mixin providing cls.log_handler as an instance-level
     fixture.
 
     After init_instance_fixtures has been called `self.log_handler` will be a
@@ -295,7 +295,7 @@ class WithLogger(object):
 
 class WithAssetFinder(WithDefaultDateBounds):
     """
-    ZiplineTestCase mixin providing cls.asset_finder as a class-level fixture.
+    CatalystTestCase mixin providing cls.asset_finder as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.asset_finder` is populated
     with an AssetFinder.
@@ -402,7 +402,7 @@ class WithAssetFinder(WithDefaultDateBounds):
 
 class WithTradingCalendars(object):
     """
-    ZiplineTestCase mixin providing cls.trading_calendar,
+    CatalystTestCase mixin providing cls.trading_calendar,
     cls.all_trading_calendars, cls.trading_calendar_for_asset_type as a
     class-level fixture.
 
@@ -423,7 +423,7 @@ class WithTradingCalendars(object):
         with that asset type.
     """
     TRADING_CALENDAR_STRS = ('NYSE',)
-    TRADING_CALENDAR_FOR_ASSET_TYPE = {Equity: 'NYSE', Future: 'us_futures'}
+    TRADING_CALENDAR_FOR_ASSET_TYPE = {Equity: 'NYSE', Future: 'us_futures', }
     TRADING_CALENDAR_FOR_EXCHANGE = {}
     # For backwards compatibility, exisitng tests and fixtures refer to
     # `trading_calendar` with the assumption that the value is the NYSE
@@ -460,7 +460,7 @@ class WithTradingEnvironment(WithAssetFinder,
                              WithTradingCalendars,
                              WithDefaultDateBounds):
     """
-    ZiplineTestCase mixin providing cls.env as a class-level fixture.
+    CatalystTestCase mixin providing cls.env as a class-level fixture.
 
     After ``init_class_fixtures`` has been called, `cls.env` is populated
     with a trading environment whose `asset_finder` is the result of
@@ -560,7 +560,7 @@ class WithTradingEnvironment(WithAssetFinder,
 
 class WithSimParams(WithTradingEnvironment):
     """
-    ZiplineTestCase mixin providing cls.sim_params as a class level fixture.
+    CatalystTestCase mixin providing cls.sim_params as a class level fixture.
 
     The arguments used to construct the trading environment may be overridded
     by putting ``SIM_PARAMS_{argname}`` in the class dict except for the
@@ -615,7 +615,7 @@ class WithSimParams(WithTradingEnvironment):
 
 class WithTradingSessions(WithTradingCalendars, WithDefaultDateBounds):
     """
-    ZiplineTestCase mixin providing cls.trading_days, cls.all_trading_sessions
+    CatalystTestCase mixin providing cls.trading_days, cls.all_trading_sessions
     as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.all_trading_sessions`
@@ -668,7 +668,7 @@ class WithTradingSessions(WithTradingCalendars, WithDefaultDateBounds):
 
 class WithTmpDir(object):
     """
-    ZiplineTestCase mixing providing cls.tmpdir as a class-level fixture.
+    CatalystTestCase mixing providing cls.tmpdir as a class-level fixture.
 
     After init_class_fixtures has been called, `cls.tmpdir` is populated with
     a `testfixtures.TempDirectory` object whose path is `cls.TMP_DIR_PATH`.
@@ -691,7 +691,7 @@ class WithTmpDir(object):
 
 class WithInstanceTmpDir(object):
     """
-    ZiplineTestCase mixing providing self.tmpdir as an instance-level fixture.
+    CatalystTestCase mixing providing self.tmpdir as an instance-level fixture.
 
     After init_instance_fixtures has been called, `self.tmpdir` is populated
     with a `testfixtures.TempDirectory` object whose path is
@@ -714,7 +714,7 @@ class WithInstanceTmpDir(object):
 
 class WithEquityDailyBarData(WithTradingEnvironment):
     """
-    ZiplineTestCase mixin providing cls.make_equity_daily_bar_data.
+    CatalystTestCase mixin providing cls.make_equity_daily_bar_data.
 
     Attributes
     ----------
@@ -810,7 +810,7 @@ class WithEquityDailyBarData(WithTradingEnvironment):
 
 class WithBcolzEquityDailyBarReader(WithEquityDailyBarData, WithTmpDir):
     """
-    ZiplineTestCase mixin providing cls.bcolz_daily_bar_path,
+    CatalystTestCase mixin providing cls.bcolz_daily_bar_path,
     cls.bcolz_daily_bar_ctable, and cls.bcolz_equity_daily_bar_reader
     class level fixtures.
 
@@ -895,7 +895,7 @@ class WithBcolzEquityDailyBarReader(WithEquityDailyBarData, WithTmpDir):
 
 class WithBcolzEquityDailyBarReaderFromCSVs(WithBcolzEquityDailyBarReader):
     """
-    ZiplineTestCase mixin that provides
+    CatalystTestCase mixin that provides
     cls.bcolz_equity_daily_bar_reader from a mapping of sids to CSV
     file paths.
     """
@@ -925,7 +925,7 @@ class _WithMinuteBarDataBase(WithTradingEnvironment):
 
 class WithEquityMinuteBarData(_WithMinuteBarDataBase):
     """
-    ZiplineTestCase mixin providing cls.equity_minute_bar_days.
+    CatalystTestCase mixin providing cls.equity_minute_bar_days.
 
     After init_class_fixtures has been called:
     - `cls.equity_minute_bar_days` has the range over which data has been
@@ -984,7 +984,7 @@ class WithEquityMinuteBarData(_WithMinuteBarDataBase):
 
 class WithFutureMinuteBarData(_WithMinuteBarDataBase):
     """
-    ZiplineTestCase mixin providing cls.future_minute_bar_days.
+    CatalystTestCase mixin providing cls.future_minute_bar_days.
 
     After init_class_fixtures has been called:
     - `cls.future_minute_bar_days` has the range over which data has been
@@ -1044,7 +1044,7 @@ class WithFutureMinuteBarData(_WithMinuteBarDataBase):
 
 class WithBcolzEquityMinuteBarReader(WithEquityMinuteBarData, WithTmpDir):
     """
-    ZiplineTestCase mixin providing cls.bcolz_minute_bar_path,
+    CatalystTestCase mixin providing cls.bcolz_minute_bar_path,
     cls.bcolz_minute_bar_ctable, and cls.bcolz_equity_minute_bar_reader
     class level fixtures.
 
@@ -1103,7 +1103,7 @@ class WithBcolzEquityMinuteBarReader(WithEquityMinuteBarData, WithTmpDir):
 
 class WithBcolzFutureMinuteBarReader(WithFutureMinuteBarData, WithTmpDir):
     """
-    ZiplineTestCase mixin providing cls.bcolz_minute_bar_path,
+    CatalystTestCase mixin providing cls.bcolz_minute_bar_path,
     cls.bcolz_minute_bar_ctable, and cls.bcolz_equity_minute_bar_reader
     class level fixtures.
 
@@ -1227,7 +1227,7 @@ class WithConstantFutureMinuteBarData(WithFutureMinuteBarData):
 
 class WithAdjustmentReader(WithBcolzEquityDailyBarReader):
     """
-    ZiplineTestCase mixin providing cls.adjustment_reader as a class level
+    CatalystTestCase mixin providing cls.adjustment_reader as a class level
     fixture.
 
     After init_class_fixtures has been called, `cls.adjustment_reader` will be
@@ -1359,7 +1359,7 @@ class WithEquityPricingPipelineEngine(WithAdjustmentReader,
 
 class WithSeededRandomPipelineEngine(WithTradingSessions, WithAssetFinder):
     """
-    ZiplineTestCase mixin providing class-level fixtures for running pipelines
+    CatalystTestCase mixin providing class-level fixtures for running pipelines
     against deterministically-generated random data.
 
     Attributes
@@ -1434,7 +1434,7 @@ class WithDataPortal(WithAdjustmentReader,
                      WithBcolzEquityMinuteBarReader,
                      WithBcolzFutureMinuteBarReader):
     """
-    ZiplineTestCase mixin providing self.data_portal as an instance level
+    CatalystTestCase mixin providing self.data_portal as an instance level
     fixture.
 
     After init_instance_fixtures has been called, `self.data_portal` will be
@@ -1485,12 +1485,12 @@ class WithDataPortal(WithAdjustmentReader,
             self.env.asset_finder,
             self.trading_calendar,
             first_trading_day=self.DATA_PORTAL_FIRST_TRADING_DAY,
-            equity_daily_reader=(
+            daily_reader=(
                 self.bcolz_equity_daily_bar_reader
                 if self.DATA_PORTAL_USE_DAILY_DATA else
                 None
             ),
-            equity_minute_reader=(
+            minute_reader=(
                 self.bcolz_equity_minute_bar_reader
                 if self.DATA_PORTAL_USE_MINUTE_DATA else
                 None
@@ -1526,7 +1526,7 @@ class WithDataPortal(WithAdjustmentReader,
 
 class WithResponses(object):
     """
-    ZiplineTestCase mixin that provides self.responses as an instance
+    CatalystTestCase mixin that provides self.responses as an instance
     fixture.
 
     After init_instance_fixtures has been called, `self.responses` will be

catalyst/utils/factory.py

@@ -37,7 +37,7 @@ from catalyst.utils.input_validation import expect_types
 __all__ = ['load_from_yahoo', 'load_bars_from_yahoo']
 
 
-def create_simulation_parameters(year=2006, start=None, end=None,
+def create_simulation_parameters(year=2016, start=None, end=None,
                                  capital_base=float("1.0e5"),
                                  num_days=None,
                                  data_frequency='daily',

catalyst/utils/run_algo.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 import warnings
 from datetime import timedelta
@@ -8,9 +9,9 @@ from time import sleep
 import click
 import pandas as pd
 
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.poloniex.poloniex import Poloniex
+from catalyst.data.bundles import load
+from catalyst.data.data_portal import DataPortal
+from catalyst.exchange.factory import get_exchange
 
 try:
     from pygments import highlight
@@ -36,11 +37,9 @@ from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError, ExchangeAuthEmpty,
-    ExchangeRequestErrorTooManyAttempts,
-    BaseCurrencyNotFoundError, ExchangeNotFoundError)
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object, get_exchange_folder
+    ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
+    BaseCurrencyNotFoundError)
+from catalyst.exchange.exchange_utils import get_algo_object
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
@@ -91,7 +90,8 @@ def _run(handle_data,
          exchange,
          algo_namespace,
          base_currency,
-         live_graph):
+         live_graph,
+         simulate_orders):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
@@ -161,40 +161,15 @@ def _run(handle_data,
 
         if portfolio is None:
             portfolio = ExchangePortfolio(
-                start_date=pd.Timestamp.utcnow()
+                start if start is not None else pd.Timestamp.utcnow()
             )
 
-        # This corresponds to the json file containing api token info
-        exchange_auth = get_exchange_auth(exchange_name)
-
-        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
-            raise ExchangeAuthEmpty(
-                    exchange=exchange_name.title(), 
-                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
-
-        if exchange_name == 'bitfinex':
-            exchanges[exchange_name] = Bitfinex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'bittrex':
-            exchanges[exchange_name] = Bittrex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        elif exchange_name == 'poloniex':
-            exchanges[exchange_name] = Poloniex(
-                key=exchange_auth['key'],
-                secret=exchange_auth['secret'],
-                base_currency=base_currency,
-                portfolio=portfolio
-            )
-        else:
-            raise ExchangeNotFoundError(exchange_name=exchange_name)
+        exchanges[exchange_name] = get_exchange(
+            exchange_name=exchange_name,
+            base_currency=base_currency,
+            portfolio=portfolio,
+            must_authenticate=live,
+        )
 
     open_calendar = get_calendar('OPEN')
 
@@ -258,17 +233,35 @@ def _run(handle_data,
                     )
 
             if base_currency in balances:
-                return balances[base_currency]
+                base_currency_available = balances[base_currency]['free']
+                log.info(
+                    'base currency available in the account: {} {}'.format(
+                        base_currency_available, base_currency
+                    )
+                )
+
+                if capital_base is not None \
+                        and capital_base < base_currency_available:
+                    log.info(
+                        'using capital base limit: {} {}'.format(
+                            capital_base, base_currency
+                        )
+                    )
+                    amount = capital_base
+                else:
+                    amount = base_currency_available
+
+                return amount
             else:
                 raise BaseCurrencyNotFoundError(
                     base_currency=base_currency,
                     exchange=exchange_name
                 )
 
-        capital_base = 0
+        combined_capital_base = 0
         for exchange_name in exchanges:
             exchange = exchanges[exchange_name]
-            capital_base += fetch_capital_base(exchange)
+            combined_capital_base += fetch_capital_base(exchange)
 
         sim_params = create_simulation_parameters(
             start=start,
@@ -285,9 +278,10 @@ def _run(handle_data,
             ExchangeTradingAlgorithmLive,
             exchanges=exchanges,
             algo_namespace=algo_namespace,
-            live_graph=live_graph
+            live_graph=live_graph,
+            simulate_orders=simulate_orders
         )
-    else:
+    elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
         # We can add back the complexity if required.
 
@@ -297,7 +291,7 @@ def _run(handle_data,
         # can handle this later.
 
         data = DataPortalExchangeBacktest(
-            exchanges=exchanges,
+            exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,
             trading_calendar=open_calendar,
             first_trading_day=start,
@@ -317,6 +311,36 @@ def _run(handle_data,
             exchanges=exchanges
         )
 
+    elif bundle is not None:
+        bundle_data = load(
+            bundle,
+            environ,
+            bundle_timestamp,
+        )
+
+        prefix, connstr = re.split(
+            r'sqlite:///',
+            str(bundle_data.asset_finder.engine.url),
+            maxsplit=1,
+        )
+        if prefix:
+            raise ValueError(
+                "invalid url %r, must begin with 'sqlite:///'" %
+                str(bundle_data.asset_finder.engine.url),
+            )
+
+        env = TradingEnvironment(asset_db_path=connstr, environ=environ)
+        first_trading_day = \
+            bundle_data.equity_minute_bar_reader.first_trading_day
+
+        data = DataPortal(
+            env.asset_finder, open_calendar,
+            first_trading_day=first_trading_day,
+            equity_minute_reader=bundle_data.equity_minute_bar_reader,
+            equity_daily_reader=bundle_data.equity_daily_bar_reader,
+            adjustment_reader=bundle_data.adjustment_reader,
+        )
+
     perf = algorithm_class(
         namespace=namespace,
         env=env,
@@ -416,7 +440,9 @@ def run_algorithm(initialize,
                   exchange_name=None,
                   base_currency=None,
                   algo_namespace=None,
-                  live_graph=False):
+                  live_graph=False,
+                  simulate_orders=True,
+                  output=os.devnull):
     """Run a trading algorithm.
 
     Parameters
@@ -486,7 +512,9 @@ def run_algorithm(initialize,
     --------
     catalyst.data.bundles.bundles : The available data bundles.
     """
-    load_extensions(default_extension, extensions, strict_extensions, environ)
+    load_extensions(
+        default_extension, extensions, strict_extensions, environ
+    )
 
     # I'm not sure that we need this since the modified DataPortal
     # does not require extensions to be explicitly loaded.
@@ -527,7 +555,7 @@ def run_algorithm(initialize,
         bundle_timestamp=bundle_timestamp,
         start=start,
         end=end,
-        output=os.devnull,
+        output=output,
         print_algo=False,
         local_namespace=False,
         environ=environ,
@@ -535,5 +563,6 @@ def run_algorithm(initialize,
         exchange=exchange_name,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
-        live_graph=live_graph
+        live_graph=live_graph,
+        simulate_orders=simulate_orders
     )

docs/source/beginner-tutorial.rst

@@ -5,9 +5,8 @@ Basics
 ~~~~~~
 
 Catalyst is an open-source algorithmic trading simulator for crypto
-assets written in Python.
-
-The source can be found at: https://github.com/enigmampc/catalyst
+assets written in Python. The source code can be found at: 
+https://github.com/enigmampc/catalyst
 
 Some benefits include:
 
@@ -25,8 +24,7 @@ Some benefits include:
    build profitable, data-driven investment strategies.
 
 This tutorial assumes that you have Catalyst correctly installed, see the
-:doc:`installation instructions <install>` if you haven't set up 
-Catalyst yet.
+:doc:`Install<install>` section if you haven't set up Catalyst yet.
 
 Every ``catalyst`` algorithm consists of at least two functions you have to
 define:
@@ -40,10 +38,12 @@ Before the start of the algorithm, ``catalyst`` calls the
 need to access from one algorithm iteration to the next.
 
 After the algorithm has been initialized, ``catalyst`` calls the
-``handle_data()`` function once for each event. At every call, it passes
-the same ``context`` variable and an event-frame called ``data``
-containing the current trading bar with open, high, low, and close
-(OHLC) prices as well as volume for each crypto asset in your universe. 
+``handle_data()`` function on each iteration, that's one per day (daily) or 
+once every minute (minute), depending on the frequency we choose to run our 
+simulation. On every iteration, ``handle_data()`` passes the same ``context`` 
+variable and an event-frame called ``data`` containing the current trading bar 
+with open, high, low, and close (OHLC) prices as well as volume for each 
+crypto asset in your universe. 
 
 .. For more information on these functions, see the `relevant part of the
 .. Quantopian docs <https://www.quantopian.com/help#api-toplevel>`.
@@ -51,8 +51,8 @@ containing the current trading bar with open, high, low, and close
 My first algorithm
 ~~~~~~~~~~~~~~~~~~
 
-Lets take a look at a very simple algorithm from the ``examples``
-directory: `buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_:
+Lets take a look at a very simple algorithm from the ``examples`` directory: 
+`buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_:
 
 .. code-block:: python
 
@@ -70,9 +70,9 @@ directory: `buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master
 
 As you can see, we first have to import some functions we would like to
 use. All functions commonly used in your algorithm can be found in
-``catalyst.api``. Here we are using :func:`~catalyst.api.order()` which takes two
-arguments: a cryptoasset object, and a number specifying how many assets you would
-like to order (if negative, :func:`~catalyst.api.order()` will sell/short
+``catalyst.api``. Here we are using :func:`~catalyst.api.order()` which takes 
+twoarguments: a cryptoasset object, and a number specifying how many assets you 
+wouldlike to order (if negative, :func:`~catalyst.api.order()` will sell/short
 assets). In this case we want to order 1 bitcoin at each iteration. 
 
 .. For more documentation on ``order()``, see the `Quantopian docs
@@ -88,61 +88,102 @@ a bitcoin in the ``data`` event frame.
 
 .. (for more information see `here <https://www.quantopian.com/help#api-event-properties>`__.
 
-Running the algorithm
-~~~~~~~~~~~~~~~~~~~~~
-
-To can now test this algorithm on crypto data, ``catalyst`` provides three
-interfaces: 
-
--  A command-line interface,
--  ``IPython Notebook`` magic, 
--  and :func:`~catalyst.run_algorithm`.
-
 Ingesting data
-^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~
 
-In previous versions of Catalyst you needed to manually ingest data before running
-your algorithm to make it available at runtime. Starting with version 0.3, the
-algorithm will automagically ingest the data it needs the first time that encounters 
-a data request for data that it doesn't have.
+Before you can backtest your algorithm, you first need to load the historical 
+pricing data that Catalyst needs to run your simulation through a process called
+``ingestion``. When you ingest data, Catalyst downloads that data in compressed 
+form from the Enigma servers (which eventually will migrate to the Enigma Data 
+Marketplace), and stores it locally to make it available at runtime. 
 
-Still, we believe it is important for you to have a high-level understanding
-of how data is managed:
+In order to ingest data, you need to run a command like the following:
+
+.. code-block:: bash
+
+  catalyst ingest-exchange -x bitfinex -i btc_usd
+
+This instructs Catalyst to download pricing data from the ``Bitfinex`` exchange 
+for the ``btc_usd`` currency pair (this follows from the simple algorithm 
+presented above where we want to trade ``btc_usd``), and we're choosing to test
+our algorithm using historical pricing data from the Bitfinex exchange. By 
+default, Catalyst assumes that you want data with ``daily`` frequency (one candle
+bar per day). If you want instead ``minute`` frequency (one candle bar for every
+minute), you would need to specify it as follows:
+
+.. code-block:: bash
+
+  catalyst ingest-exchange -x bitfinex -i btc_usd -f minute
+
+.. parsed-literal::
+
+  Ingesting exchange bundle bitfinex...
+    [====================================]  Ingesting daily price data on bitfinex:  100%
+
+We believe it is important for you to have a high-level understanding of how 
+data is managed, hence the following overview:
 
 -  Pricing data is split and packaged into ``bundles``: chunks of data organized 
    as time series that are kept up to date daily on Enigma's servers. Catalyst 
-   downloads the bundles that needs at any given time, and reconstructs the whole
-   dataset in your hard drive.
+   downloads the requested bundles and reconstructs the full dataset in your 
+   hard drive.
 
--  Pricing data is provided in ``daily`` and ``minute`` resolution. Those are different
-   bundle datasets, and are managed separately.
+-  Pricing data is provided in ``daily`` and ``minute`` resolution. Those are 
+   different bundle datasets, and are managed separately.
 
--  Bundles are exchange-specific, as the pricing data is specific to the trades that
-   happen in each exchange. You can optionally specify which exchange you want pricing
-   data from.
+-  Bundles are exchange-specific, as the pricing data is specific to the trades 
+   that happen in each exchange. As a result, you can must specify which 
+   exchange you want pricing data from when ingesting data
 
--  Catalyst keeps track of all the downloaded bundles, so that it only has to download
-   them once, and will do incremental updates as needed.
+-  Catalyst keeps track of all the downloaded bundles, so that it only has to 
+   download them once, and will do incremental updates as needed.
 
--  When running in ``live trading`` mode, Catalyst will first look for historical 
-   pricing data in the locally stored bundles. If there is anything missing, Catalyst will
-   hit the exchange for the most recent data, and merge it with the local bundle to make
-   it available for future iterations.
+-  When running in ``live trading`` mode, Catalyst will first look for 
+   historical pricing data in the locally stored bundles. If there is anything 
+   missing, Catalyst will hit the exchange for the most recent data, and merge 
+   it with the local bundle to optimize the number of requests it needs to make 
+   to the exchange.
 
-If you want to learn more, check out the :ref:`ingesting data <ingesting-data>` section
-for more detail.
+The ``ingest-exchange`` command in catalyst offers additional parameters to 
+further tweak the data ingestion process. You can learn more by running the
+following from the command line:
+
+.. code-block:: bash
+
+  catalyst ingest-exchange --help
+
+Running the algorithm
+~~~~~~~~~~~~~~~~~~~~~
+
+You can now test your algorithm using cryptoassets' historical pricing data, 
+``catalyst`` provides three interfaces: 
+
+-  A command-line interface (CLI),
+-  a :func:`~catalyst.run_algorithm()` that you can call from other 
+   Python scripts,
+-  and the ``Jupyter Notebook`` magic.
+
+
+We'll start with the CLI, and introduce the ``run_algorithm()`` in the last 
+example of this tutorial. Some of the :doc:`example algorithms <example-algos>` 
+provide instructions on how to run them both from the CLI, and using the 
+:func:`~catalyst.run_algorithm` function. For the third method, refer to the 
+corresponding section on :doc:`Catalyst & Jupyter Notebook <jupyter>` after you 
+have assimilated the contents of this tutorial.
 
 Command line interface
 ^^^^^^^^^^^^^^^^^^^^^^
 
-After you installed Catalyst you should be able to execute the following
-from your command line (e.g. ``cmd.exe`` on Windows, or the Terminal app
-on OSX). Displaying here a simplified output for eductional purposes:
+After you installed Catalyst, you should be able to execute the following
+from your command line (e.g. ``cmd.exe`` or the ``Anaconda Prompt`` on Windows, 
+or the Terminal application on MacOS). 
 
 .. code-block:: bash
 
    $ catalyst --help
 
+This is the resulting output, simplified for eductional purposes:
+
 .. parsed-literal::
 
      Usage: catalyst [OPTIONS] COMMAND [ARGS]...
@@ -158,10 +199,11 @@ on OSX). Displaying here a simplified output for eductional purposes:
        live             Trade live with the given algorithm.
        run              Run a backtest for the given algorithm.
 
-There are three main modes you can run on Catalyst. The first being ``ingest-exchange`` 
-for data ingestion, which we have summarized in the previous section. The second 
-is ``live`` to use your algorithm to trade live against a given exchange, and the 
-third mode ``run`` is to backtest your algorithm before trading live with it.
+There are three main modes you can run on Catalyst. The first being 
+``ingest-exchange`` for data ingestion, which we have covered in the previous 
+section. The second is ``live`` to use your algorithm to trade live against a 
+given exchange, and the third mode ``run`` is to backtest your algorithm before 
+trading live with it.
 
 Let's start with backtesting, so run this other command to learn more about 
 the available options:
@@ -210,22 +252,24 @@ the available options:
 
 
 As you can see there are a couple of flags that specify where to find your
-algorithm (``-f``) as well as a parameter to specify which exchange to use. 
-There are also arguments for the date range to run the algorithm over 
-(``--start`` and ``--end``). Finally, you'll want to save the performance 
-metrics of your algorithm so that you can analyze how it performed. This is 
-done via the ``--output`` flag and will cause it to write the performance 
-``DataFrame`` in the pickle Python file format. Note that you can also define 
-a configuration file with these parameters that you can then conveniently pass 
-to the ``-c`` option so that you don't have to supply the command line args 
-all the time (see the .conf files in the examples directory).
+algorithm (``-f``) as well as a the ``-x`` flag to specify which exchange to 
+use. There are also arguments for the date range to run the algorithm over 
+(``--start`` and ``--end``). You also need to set the base currency for your 
+algorithm through the ``-c`` flag, and the ``--capital_base``. All the 
+aforementioned parameters are required. Optionally, you will want to save the 
+performance metrics of your algorithm so that you can analyze how it performed. 
+This is done via the ``--output`` flag and will cause it to write the 
+performance ``DataFrame`` in the pickle Python file format. Note that you can 
+also define a configuration file with these parameters that you can then 
+conveniently pass to the ``-c`` option so that you don't have to supply the 
+command line args all the time.
 
 Thus, to execute our algorithm from above and save the results to
 ``buy_btc_simple_out.pickle`` we would call ``catalyst run`` as follows:
 
-.. code-block:: python
+.. code-block:: bash
 
-    catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+    catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -c usd --capital-base 100000 -o buy_btc_simple_out.pickle
 
 
 .. parsed-literal:: 
@@ -253,17 +297,25 @@ slippage model that ``catalyst`` uses).
 .. see the `Quantopian docs <https://www.quantopian.com/help#ide-slippage>`__
 .. for more information).
 
-Let's take a quick look at the performance ``DataFrame``. For this, we
-use ``pandas`` from inside the IPython Notebook and print the first ten
-rows. Note that ``catalyst`` makes heavy usage of 
-`pandas <http://pandas.pydata.org/>`_, especially for data input and 
-outputting so it's worth spending some time to learn it.
+
+Let's take a quick look at the performance ``DataFrame``. For this, we write 
+different Python script--let's call it ``print_results.py``--and we make use of 
+the fantastic ``pandas`` library to print the first ten rows. Note that 
+``catalyst`` makes heavy usage of `pandas <http://pandas.pydata.org/>`_, 
+especially for data analysis and outputting so it's worth spending some time to 
+learn it.
 
 .. code-block:: python
 
     import pandas as pd
     perf = pd.read_pickle('buy_btc_simple_out.pickle') # read in perf DataFrame
-    perf.head()
+    print(perf.head())
+
+Which we execute by running:
+
+.. code-block:: bash
+
+   $ python print_results.py
 
 .. raw:: html
 
@@ -429,30 +481,48 @@ and allows us to plot the price of bitcoin. For example, we could easily
 examine now how our portfolio value changed over time compared to the
 bitcoin price.
 
-.. code-block:: python
-
-    %load_ext catalyst
+Now we will run the simulation again, but this time we extend our original 
+algorithm with the addition of the ``analyze()`` function. Somewhat analogously 
+as how ``initialize()`` gets called once before the start of the algorith, 
+``analyze()`` gets called once at the end of the algorithm, and receives two 
+variables: ``context``, which we discussed at the very beginning, and ``perf``, 
+which is the pandas dataframe containing the performance data for our algorithm 
+that we reviewed above. Inside the ``analyze()`` function is where we can 
+analyze and visualize the results of our strategy. Here's the revised simple 
+algorithm (note the addition of Line 1, and Lines 11-18)
 
 .. code-block:: python
 
-    %pylab inline
-    figsize(12, 12)
     import matplotlib.pyplot as plt
+    from catalyst.api import order, record, symbol
 
-    ax1 = plt.subplot(211)
-    perf.portfolio_value.plot(ax=ax1)
-    ax1.set_ylabel('portfolio value')
-    ax2 = plt.subplot(212, sharex=ax1)
-    perf.btc.plot(ax=ax2)
-    ax2.set_ylabel('bitcoin price')
+    def initialize(context):
+        context.asset = symbol('btc_usd')
 
-.. parsed-literal::
+    def handle_data(context, data):
+        order(context.asset, 1)
+        record(btc = data.current(context.asset, 'price'))
 
-    Populating the interactive namespace from numpy and matplotlib
+    def analyze(context, perf):
+        ax1 = plt.subplot(211)
+        perf.portfolio_value.plot(ax=ax1)
+        ax1.set_ylabel('portfolio value')
+        ax2 = plt.subplot(212, sharex=ax1)
+        perf.btc.plot(ax=ax2)
+        ax2.set_ylabel('bitcoin price')
+        plt.show()
 
-.. parsed-literal::
+Here we make use of the external visualization library called 
+`matplotlib <https://matplotlib.org/>`_, which you might recall we installed 
+alongside enigma-catalyst (with the exception of the ``Conda`` install, where it
+was included by default inside the conda environment we created). If for any 
+reason you don't have it installed, you can add it by running:
 
-    <matplotlib.text.Text at 0x10eaeadd0>
+.. code-block:: python
+
+  (catalyst)$ pip install matplotlib
+
+If everything works well, you'll see the following chart:
 
 .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/buy_btc_simple_graph.png
 
@@ -460,6 +530,23 @@ Our algorithm performance as assessed by the ``portfolio_value`` closely
 matches that of the bitcoin price. This is not surprising as our algorithm 
 only bought bitcoin every chance it got.
 
+  If you get an error when invoking matplotlib to visualize the performance 
+  results refer to `MacOS + Matplotlib <install.html#macos-virtualenv-matplotlib>`_. 
+  Alternatively, some users have reported the following error when running an algo 
+  in a Linux environment:
+
+  .. parsed-literal::
+
+      ImportError: No module named _tkinter, please install the python-tk package
+
+  Which can easily solved by running (in Ubuntu/Debian-based systems):
+
+  .. code-block:: python
+
+      sudo apt install python-tk
+
+
+.. _history:
 
 Access to previous prices using ``history``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -478,78 +565,235 @@ If the short-mavg crosses from above we exit the positions as we assume
 the stock to go down further.
 
 As we need to have access to previous prices to implement this strategy
-we need a new concept: History
+we need a new concept: History. ``data.history()`` is a convenience function 
+that keeps a rolling window of data for you. The first argument is the number 
+of bars you want to collect, the second argument is the unit (either ``'1d'`` 
+for daily or ``'1m'`` for minute frequency, but note that you need to have 
+minute-level data when using ``1m``). This is a function we use in the 
+``handle_data()`` section.
 
-``data.history()`` is a convenience function that keeps a rolling window of
-data for you. The first argument is the number of bars you want to
-collect, the second argument is the unit (either ``'1d'`` for ``'1m'``
-but note that you need to have minute-level data for using ``1m``). This is
-a function we use in the ``handle_data()`` section:
-
-.. code-block:: python
-  
-  %load_ext catalyst
+You will note that the code below is substantially longer than the previous 
+examples. Don't get overwhelmed by it as the logic is fairly simple and easy to 
+follow. Most of the added some complexity has been added to beautify the output, 
+which you can skim through for now. A copy of this algorithm is available in 
+the ``examples`` directory:
+`dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
 
 .. code-block:: python
 
-    %%catalyst --start 2016-4-1 --end 2017-9-30 -x bitfinex
+    import numpy as np
+    import pandas as pd
+    from logbook import Logger
+    import matplotlib.pyplot as plt 
 
-    from catalyst.api import order, record, symbol, order_target
+    from catalyst import run_algorithm
+    from catalyst.api import (order, record, symbol, order_target_percent, 
+            get_open_orders)
+    from catalyst.exchange.stats_utils import extract_transactions
+
+    NAMESPACE = 'dual_moving_average'
+    log = Logger(NAMESPACE)
 
     def initialize(context):
-       context.i = 0
-       context.asset = symbol('btc_usd')
+        context.i = 0
+        context.asset = symbol('ltc_usd')
+        context.base_price = None
+
 
     def handle_data(context, data):
-       # Skip first 150 days to get full windows
-       context.i += 1
-       if context.i < 150:
+        # define the windows for the moving averages 
+        short_window = 50
+        long_window = 200
+
+        # Skip as many bars as long_window to properly compute the average
+        context.i += 1
+        if context.i < long_window:
            return
 
-       # Compute averages
-       # data.history() has to be called with the same params
-       # from above and returns a pandas dataframe.
-       short_mavg = data.history(context.asset, 'price', bar_count=50, frequency="1d").mean()
-       long_mavg = data.history(context.asset, 'price', bar_count=150, frequency="1d").mean()
+        # Compute moving averages calling data.history() for each 
+        # moving average with the appropriate parameters. We choose to use
+        # minute bars for this simulation -> freq="1m"
+        # Returns a pandas dataframe.
+        short_mavg = data.history(context.asset, 'price', 
+                            bar_count=short_window, frequency="1m").mean()
+        long_mavg = data.history(context.asset, 'price',
+                            bar_count=long_window, frequency="1m").mean()
 
-       # Trading logic
-       if short_mavg > long_mavg:
-           # order_target orders as many shares as needed to
-           # achieve the desired number of shares.
-           order_target(context.asset, 100)
-       elif short_mavg < long_mavg:
-           order_target(context.asset, 0)
+        # Let's keep the price of our asset in a more handy variable
+        price = data.current(context.asset, 'price')
+
+        # If base_price is not set, we use the current value. This is the
+        # price at the first bar which we reference to calculate price_change.
+        if context.base_price is None:
+            context.base_price = price
+        price_change = (price - context.base_price) / context.base_price
+
+        # Save values for later inspection
+        record(price=price,
+               cash=context.portfolio.cash,
+               price_change=price_change,
+               short_mavg=short_mavg,
+               long_mavg=long_mavg)
+
+        # Since we are using limit orders, some orders may not execute immediately
+        # we wait until all orders are executed before considering more trades.
+        orders = get_open_orders(context.asset)
+        if len(orders) > 0:
+            return
+
+        # Exit if we cannot trade
+        if not data.can_trade(context.asset):
+            return
+
+        # We check what's our position on our portfolio and trade accordingly
+        pos_amount = context.portfolio.positions[context.asset].amount
+
+        # Trading logic
+        if short_mavg > long_mavg and pos_amount == 0:
+           # we buy 100% of our portfolio for this asset
+           order_target_percent(context.asset, 1)
+        elif short_mavg < long_mavg and pos_amount > 0:
+           # we sell all our positions for this asset
+           order_target_percent(context.asset, 0)
 
-       # Save values for later inspection
-       record(btc=data.current(context.asset, 'price'),
-              short_mavg=short_mavg,
-              long_mavg=long_mavg)
 
     def analyze(context, perf):
-       import matplotlib.pyplot as plt
-       fig = plt.figure(figsize=(12,12))
-       ax1 = fig.add_subplot(211)
-       perf.portfolio_value.plot(ax=ax1)
-       ax1.set_ylabel('portfolio value in $')
 
-       ax2 = fig.add_subplot(212)
-       perf['btc'].plot(ax=ax2)
-       perf[['short_mavg', 'long_mavg']].plot(ax=ax2)
+        # Get the base_currency that was passed as a parameter to the simulation
+        base_currency = context.exchanges.values()[0].base_currency.upper()
 
-       perf_trans = perf.ix[[t != [] for t in perf.transactions]]
-       buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]]
-       sells = perf_trans.ix[
-           [t[0]['amount'] < 0 for t in perf_trans.transactions]]
-       ax2.plot(buys.index, perf.short_mavg.ix[buys.index],
-                '^', markersize=10, color='m')
-       ax2.plot(sells.index, perf.short_mavg.ix[sells.index],
-                'v', markersize=10, color='k')
-       ax2.set_ylabel('price in $')
-       plt.legend(loc=0)
-       plt.show()
+        # First chart: Plot portfolio value using base_currency
+        ax1 = plt.subplot(411)
+        perf.loc[:, ['portfolio_value']].plot(ax=ax1)
+        ax1.legend_.remove()
+        ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
+        start, end = ax1.get_ylim()
+        ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
 
-Here we are explicitly defining an ``analyze()`` function that gets
-automatically called once the backtest is done.
+        # Second chart: Plot asset price, moving averages and buys/sells
+        ax2 = plt.subplot(412, sharex=ax1)
+        perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
+        ax2.legend_.remove()
+        ax2.set_ylabel('{asset}\n({base})'.format(
+            asset = context.asset.symbol,
+            base = base_currency
+            ))
+        start, end = ax2.get_ylim()
+        ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+        transaction_df = extract_transactions(perf)
+        if not transaction_df.empty:
+            buy_df = transaction_df[transaction_df['amount'] > 0]
+            sell_df = transaction_df[transaction_df['amount'] < 0]
+            ax2.scatter(
+                buy_df.index.to_pydatetime(),
+                perf.loc[buy_df.index, 'price'],
+                marker='^',
+                s=100,
+                c='green',
+                label=''
+            )
+            ax2.scatter(
+                sell_df.index.to_pydatetime(),
+                perf.loc[sell_df.index, 'price'],
+                marker='v',
+                s=100,
+                c='red',
+                label=''
+            )
+
+        # Third chart: Compare percentage change between our portfolio
+        # and the price of the asset
+        ax3 = plt.subplot(413, sharex=ax1)
+        perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
+        ax3.legend_.remove()
+        ax3.set_ylabel('Percent Change')
+        start, end = ax3.get_ylim()
+        ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+        # Fourth chart: Plot our cash
+        ax4 = plt.subplot(414, sharex=ax1)
+        perf.cash.plot(ax=ax4)
+        ax4.set_ylabel('Cash\n({})'.format(base_currency))
+        start, end = ax4.get_ylim()
+        ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+
+        plt.show()
+
+
+    if __name__ == '__main__':
+        run_algorithm(
+                capital_base=1000,
+                data_frequency='minute',
+                initialize=initialize,
+                handle_data=handle_data,
+                analyze=analyze,
+                exchange_name='bitfinex',
+                algo_namespace=NAMESPACE,
+                base_currency='usd',
+                start=pd.to_datetime('2017-9-22', utc=True),
+                end=pd.to_datetime('2017-9-23', utc=True),
+            )
+
+In order to run the code above, you have to ingest the needed data first:
+
+.. code-block:: bash
+
+  catalyst ingest-exchange -x bitfinex -f minute -i ltc_usd
+
+And then run the code above with the following command:
+
+.. code-block:: bash
+
+  catalyst run -f dual_moving_average.py -x bitfinex -s 2017-9-22 -e 2017-9-23 --capital-base 1000 --base-currency usd --data-frequency minute -o out.pickle
+
+Alternatively, we can make use of the ``run_algorithm()`` function included at 
+the end of the file, where we can specify all the simulation parameters, and 
+execute this file as a Python script:
+
+.. code-block:: bash
+
+  python dual_moving_average.py
+
+Either way, we obtain the following charts:
+
+.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
+
+
+A few comments on the code above:
+
+  At the beginning of our code, we import a number of Python libraries that we
+  will be using in different parts of our script. It's good practice to keep all
+  imports at the beginning of the file, as they are available globally 
+  throughout our script. All the libraries imported in this example are already
+  present in your environment since they are prerequisites for the Catalyst 
+  installation.
+
+  Focus on the code that is inside ``handle_data()`` that is where all the 
+  trading logic occurs. You can safely dismiss most of the code in the 
+  ``analyze()`` section, which is mostly to customize the visualization of the 
+  performance of our algorithm using the matplotlib library. You can copy and
+  paste this whole section into other algorithms to obtain a similar display.
+
+  Inside the ``handle_data()``, we also used the ``order_target_percent()`` 
+  function above. This and other functions like it can make order management 
+  and portfolio rebalancing much easier.
+
+  The ``ltc_usd`` asset was arbitrarily chosen. The values of 50 and 200 for the 
+  ``short_window`` and ``long_window`` parameters are fairly common for a dual 
+  moving average crossover strategy from the world of traditional stocks (but 
+  bear in mind that they are usually used with daily bars instead of minute 
+  bars). The ``start`` and ``end`` dates have been chosen so as to demonstrate 
+  how our strategy can both perform better (blue line above green line on the 
+  ``Percent Change`` chart) and worse (green line above blue line towards the end) than the
+  price of the asset we are trading. 
+
+  You can change any of these parameters: ``asset``, ``short_window``, 
+  ``long_window``, ``start_date`` and ``end_date`` and compare the results, and 
+  you will see that in most cases, the performance is either worse than the 
+  price of the asset, or you are overfitting to one specific case. As we said 
+  at the beginning of this section, this strategy is probably not used by any 
+  serious trader anymore, but its educational purpose.
 
 Although it might not be directly apparent, the power of ``history()``
 (pun intended) can not be under-estimated as most algorithms make use of
@@ -561,21 +805,13 @@ the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
 ``pandas.DataFrame``\ s, so you can simply pass the underlying
 ``ndarray`` of a ``DataFrame`` via ``.values``).
 
-We also used the ``order_target()`` function above. This and other
-functions like it can make order management and portfolio rebalancing
-much easier.
 
-
-Conclusions
-~~~~~~~~~~~
+Next steps
+~~~~~~~~~~
 
 We hope that this tutorial gave you a little insight into the
-architecture, API, and features of ``catalyst``. For next steps, check
-out some of the
-`examples <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`__.
-The natural next step would be too look into the 
-`buy_and_hodl <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_ 
-example, which is a more elaborated and realistic version of the ``buy_btc_simple`` example presented in this tutorial.
+architecture, API, and features of Catalyst. For next steps, check
+out some of the other :doc:`example algorithms<example-algos>`.
 
 Feel free to ask questions on the ``#catalyst_dev`` channel of our 
 `Discord group <https://discord.gg/SJK32GY>`__ and report

docs/source/example-algos.rst

@@ -0,0 +1,901 @@
+|
+Example Algorithms
+==================
+
+This section documents a number of example algorithms to complement the 
+beginner tutorial, and show how other trading algorithms can be implemented 
+using Catalyst.
+
+Overview
+~~~~~~~~
+
+- :ref:`Buy BTC Simple<buy_btc_simple>`: The simplest algorithm that introduces
+  the ``initialize()`` and ``handle_data()`` functions, and is used in the 
+  :doc:`beginner tutorial<beginner-tutorial>` to show how to run catalyst 
+  for the first time.
+
+- :ref:`Buy and Hodl <buy_and_hodl>`: A very straightforward *buy and hold* that 
+  makes one single buy at the very beginning. Introduces the notions of 
+  ``cash``, management of outstanding ``orders``, and ``order_target_value`` 
+  to place orders. It also introduces the ``analyze()`` function to visualize 
+  the performance of our strategy using the external library ``matplotlib``.
+
+- :ref:`Dual Moving Average Crossover<dual_moving_average>`: A classic momentum 
+  strategy used in the second part of the 
+  `beginner tutorial <beginner-tutorial.html#history>`_ to introduce the 
+  ``data.history()`` function. It makes a heavy use of ``matplotlib`` library 
+  in the ``analyze()`` function to chart the performance of the algorithm.
+
+- :ref:`Mean Reversion Algorithm <mean_reversion>`: Another simple momentum 
+  strategy that is used in our 
+  `two-part video tutorial <videos.html#backtesting-a-strategy>`_ to show how 
+  to get started in backtesting and live trading with Catalyst.
+
+- :ref:`Portfolio Optimization <portfolio_optimization>`: Use this code to 
+  execute a portfolio optimization model. This strategy will select the 
+  portfolio with the maximum Sharpe Ratio. The parameters are set to use 180 
+  days of historical data and rebalance every 30 days. This code was used in 
+  writting the following article: 
+  `Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
+
+
+.. _buy_btc_simple:
+
+Buy BTC Simple Algorithm
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
+
+.. code-block:: python
+
+  '''
+    Run this example, by executing the following from your terminal:
+      catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
+      catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+    If you want to run this code using another exchange, make sure that 
+    the asset is available on that exchange. For example, if you were to run 
+    it for exchange Poloniex, you would need to edit the following line:
+
+      context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
+
+    and specify exchange poloniex as follows:
+    catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
+    catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+    To see which assets are available on each exchange, visit:
+    https://www.enigma.co/catalyst/status
+  '''
+
+  from catalyst.api import order, record, symbol
+
+  def initialize(context):
+      context.asset = symbol('btc_usd')
+
+  def handle_data(context, data):
+      order(context.asset, 1)
+      record(btc = data.current(context.asset, 'price'))
+
+This simple algorithm does not produce any output nor displays any chart.
+
+
+.. _buy_and_hodl:
+
+Buy and Hodl Algorithm
+~~~~~~~~~~~~~~~~~~~~~~
+
+Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
+
+First ingest the historical pricing data needed to run this algorithm:
+
+.. code-block:: bash
+
+   catalyst ingest-exchange -x bitfinex -f daily -i btc_usd
+
+Then, you can run the code below with the following command:
+
+.. code-block:: bash
+
+   catalyst run -f buy_and_hodl.py --start 2015-3-1 --end 2017-10-31 --capital-base 100000 -x bitfinex -c btc -o bah.pickle
+
+or using the same parameters specified in the run_algorithm() function at the 
+end of the file:
+
+.. code-block:: bash
+
+  python buy_and_hodl.py
+
+
+This command will run the trading algorithm in the specified time range and 
+plot the resulting performance using the matplotlib library. You can choose any 
+date interval with the ``--start`` and ``--end`` parameters, but bear in mind 
+that 2015-3-1 is the earliest date that Catalyst supports (if you choose an 
+earlier date, you'll get an error), and the most recent date you can choose is 
+one day prior to the current date. 
+
+
+.. code-block:: python
+
+  #!/usr/bin/env python
+  #
+  # Copyright 2017 Enigma MPC, Inc.
+  # Copyright 2015 Quantopian, Inc.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  import pandas as pd
+  import matplotlib.pyplot as plt
+
+  from catalyst import run_algorithm
+  from catalyst.api import (order_target_value, symbol, record,
+      cancel_order, get_open_orders, )
+
+
+  def initialize(context):
+      context.ASSET_NAME = 'btc_usd'
+      context.TARGET_HODL_RATIO = 0.8
+      context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
+
+      context.is_buying = True
+      context.asset = symbol(context.ASSET_NAME)
+
+      context.i = 0
+
+
+  def handle_data(context, data):
+      context.i += 1
+
+      starting_cash = context.portfolio.starting_cash
+      target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
+      reserve_value = context.RESERVE_RATIO * starting_cash
+
+      # Cancel any outstanding orders
+      orders = get_open_orders(context.asset) or []
+      for order in orders:
+          cancel_order(order)
+
+      # Stop buying after passing the reserve threshold
+      cash = context.portfolio.cash
+      if cash <= reserve_value:
+          context.is_buying = False
+
+      # Retrieve current asset price from pricing data
+      price = data.current(context.asset, 'price')
+
+      # Check if still buying and could (approximately) afford another purchase
+      if context.is_buying and cash > price:
+          print('buying')
+          # Place order to make position in asset equal to target_hodl_value
+          order_target_value(
+              context.asset,
+              target_hodl_value,
+              limit_price=price * 1.1,
+              stop_price=price * 0.9,
+          )
+
+      record(
+          price=price,
+          volume=data.current(context.asset, 'volume'),
+          cash=cash,
+          starting_cash=context.portfolio.starting_cash,
+          leverage=context.account.leverage,
+      )
+
+
+  def analyze(context=None, results=None):
+
+      # Plot the portfolio and asset data.
+      ax1 = plt.subplot(611)
+      results[['portfolio_value']].plot(ax=ax1)
+      ax1.set_ylabel('Portfolio Value (USD)')
+
+      ax2 = plt.subplot(612, sharex=ax1)
+      ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
+      results[['price']].plot(ax=ax2)
+
+      trans = results.ix[[t != [] for t in results.transactions]]
+      buys = trans.ix[
+          [t[0]['amount'] > 0 for t in trans.transactions]
+      ]
+      ax2.scatter(
+          buys.index.to_pydatetime(),
+          results.price[buys.index],
+          marker='^',
+          s=100,
+          c='g',
+          label=''
+      )
+
+      ax3 = plt.subplot(613, sharex=ax1)
+      results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
+      ax3.set_ylabel('Leverage ')
+
+      ax4 = plt.subplot(614, sharex=ax1)
+      results[['starting_cash', 'cash']].plot(ax=ax4)
+      ax4.set_ylabel('Cash (USD)')
+
+      results[[
+          'treasury',
+          'algorithm',
+          'benchmark',
+      ]] = results[[
+          'treasury_period_return',
+          'algorithm_period_return',
+          'benchmark_period_return',
+      ]]
+
+      ax5 = plt.subplot(615, sharex=ax1)
+      results[[
+          'treasury',
+          'algorithm',
+          'benchmark',
+      ]].plot(ax=ax5)
+      ax5.set_ylabel('Percent Change')
+
+      ax6 = plt.subplot(616, sharex=ax1)
+      results[['volume']].plot(ax=ax6)
+      ax6.set_ylabel('Volume (mCoins/5min)')
+
+      plt.legend(loc=3)
+
+      # Show the plot.
+      plt.gcf().set_size_inches(18, 8)
+      plt.show()
+
+
+  if __name__ == '__main__':
+      run_algorithm(
+          capital_base=10000,
+          data_frequency='daily',
+          initialize=initialize,
+          handle_data=handle_data,
+          analyze=analyze,
+          exchange_name='bitfinex',
+          algo_namespace='buy_and_hodl',
+          base_currency='usd',
+          start=pd.to_datetime('2015-03-01', utc=True),
+          end=pd.to_datetime('2017-10-31', utc=True),
+      )
+
+.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
+
+.. _dual_moving_average:
+
+Dual Moving Average Crossover
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
+
+This strategy is covered in detail in the last part of 
+`this tutorial <beginner-tutorial.html#history>`_.
+
+.. code-block:: python
+
+  import numpy as np
+  import pandas as pd
+  from logbook import Logger
+  import matplotlib.pyplot as plt
+
+  from catalyst import run_algorithm
+  from catalyst.api import (order, record, symbol, order_target_percent,
+          get_open_orders)
+  from catalyst.exchange.stats_utils import extract_transactions
+
+  NAMESPACE = 'dual_moving_average'
+  log = Logger(NAMESPACE)
+
+  def initialize(context):
+      context.i = 0
+      context.asset = symbol('ltc_usd')
+      context.base_price = None
+
+
+  def handle_data(context, data):
+      # define the windows for the moving averages
+      short_window = 50
+      long_window = 200
+
+      # Skip as many bars as long_window to properly compute the average
+      context.i += 1
+      if context.i < long_window:
+         return
+
+      # Compute moving averages calling data.history() for each
+      # moving average with the appropriate parameters. We choose to use
+      # minute bars for this simulation -> freq="1m"
+      # Returns a pandas dataframe.
+      short_mavg = data.history(context.asset, 'price',
+                          bar_count=short_window, frequency="1m").mean()
+      long_mavg = data.history(context.asset, 'price',
+                          bar_count=long_window, frequency="1m").mean()
+
+      # Let's keep the price of our asset in a more handy variable
+      price = data.current(context.asset, 'price')
+
+      # If base_price is not set, we use the current value. This is the
+      # price at the first bar which we reference to calculate price_change.
+      if context.base_price is None:
+          context.base_price = price
+      price_change = (price - context.base_price) / context.base_price
+
+      # Save values for later inspection
+      record(price=price,
+             cash=context.portfolio.cash,
+             price_change=price_change,
+             short_mavg=short_mavg,
+             long_mavg=long_mavg)
+
+      # Since we are using limit orders, some orders may not execute immediately
+      # we wait until all orders are executed before considering more trades.
+      orders = get_open_orders(context.asset)
+      if len(orders) > 0:
+          return
+
+      # Exit if we cannot trade
+      if not data.can_trade(context.asset):
+          return
+
+      # We check what's our position on our portfolio and trade accordingly
+      pos_amount = context.portfolio.positions[context.asset].amount
+
+      # Trading logic
+      if short_mavg > long_mavg and pos_amount == 0:
+         # we buy 100% of our portfolio for this asset
+         order_target_percent(context.asset, 1)
+      elif short_mavg < long_mavg and pos_amount > 0:
+         # we sell all our positions for this asset
+         order_target_percent(context.asset, 0)
+
+
+  def analyze(context, perf):
+
+      # Get the base_currency that was passed as a parameter to the simulation
+      base_currency = context.exchanges.values()[0].base_currency.upper()
+
+      # First chart: Plot portfolio value using base_currency
+      ax1 = plt.subplot(411)
+      perf.loc[:, ['portfolio_value']].plot(ax=ax1)
+      ax1.legend_.remove()
+      ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
+      start, end = ax1.get_ylim()
+      ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+      # Second chart: Plot asset price, moving averages and buys/sells
+      ax2 = plt.subplot(412, sharex=ax1)
+      perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
+      ax2.legend_.remove()
+      ax2.set_ylabel('{asset}\n({base})'.format(
+          asset = context.asset.symbol,
+          base = base_currency
+          ))
+      start, end = ax2.get_ylim()
+      ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+      transaction_df = extract_transactions(perf)
+      if not transaction_df.empty:
+          buy_df = transaction_df[transaction_df['amount'] > 0]
+          sell_df = transaction_df[transaction_df['amount'] < 0]
+          ax2.scatter(
+              buy_df.index.to_pydatetime(),
+              perf.loc[buy_df.index, 'price'],
+              marker='^',
+              s=100,
+              c='green',
+              label=''
+          )
+          ax2.scatter(
+              sell_df.index.to_pydatetime(),
+              perf.loc[sell_df.index, 'price'],
+              marker='v',
+              s=100,
+              c='red',
+              label=''
+          )
+
+      # Third chart: Compare percentage change between our portfolio
+      # and the price of the asset
+      ax3 = plt.subplot(413, sharex=ax1)
+      perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
+      ax3.legend_.remove()
+      ax3.set_ylabel('Percent Change')
+      start, end = ax3.get_ylim()
+      ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
+
+      # Fourth chart: Plot our cash
+      ax4 = plt.subplot(414, sharex=ax1)
+      perf.cash.plot(ax=ax4)
+      ax4.set_ylabel('Cash\n({})'.format(base_currency))
+      start, end = ax4.get_ylim()
+      ax4.yaxis.set_ticks(np.arange(0, end, end/5))
+
+      plt.show()
+
+
+  if __name__ == '__main__':
+      run_algorithm(
+              capital_base=1000,
+              data_frequency='minute',
+              initialize=initialize,
+              handle_data=handle_data,
+              analyze=analyze,
+              exchange_name='bitfinex',
+              algo_namespace=NAMESPACE,
+              base_currency='usd',
+              start=pd.to_datetime('2017-9-22', utc=True),
+              end=pd.to_datetime('2017-9-23', utc=True),
+          )
+
+.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
+
+
+.. _mean_reversion:
+
+Mean Reversion Algorithm
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
+
+This algorithm is based on a simple momentum strategy. When the cryptoasset goes
+up quickly, we're going to buy; when it goes down quickly, we're going to sell. 
+Hopefully, we'll ride the waves.
+
+We are choosing to backtest this trading algorithm with the ``neo_usd`` currency 
+pairon the ``Bitfinex`` exchange. Thus, first ingest the historical pricing data
+that we need, with minute resolution:
+
+.. code-block:: bash
+
+   catalyst ingest-exchange -x bitfinex -f minute -i neo_usd
+
+To run this algorithm, we are opting for the Python interpreter, instead of the 
+command line (CLI). All of the parameters for the simulation are specified in 
+lines 218-245, so in order to run the algorithm we just type:
+
+.. code-block:: bash
+
+   python mean_reversion_simple.py
+
+.. code-block:: python
+
+  import os
+  import tempfile
+  import time
+
+  import numpy as np
+  import pandas as pd
+  import talib
+  from logbook import Logger
+
+  from catalyst import run_algorithm
+  from catalyst.api import symbol, record, order_target_percent, get_open_orders
+  from catalyst.exchange.stats_utils import extract_transactions
+  # We give a name to the algorithm which Catalyst will use to persist its state.
+  # In this example, Catalyst will create the `.catalyst/data/live_algos`
+  # directory. If we stop and start the algorithm, Catalyst will resume its
+  # state using the files included in the folder.
+  from catalyst.utils.paths import ensure_directory
+
+  NAMESPACE = 'mean_reversion_simple'
+  log = Logger(NAMESPACE)
+
+
+  # To run an algorithm in Catalyst, you need two functions: initialize and
+  # handle_data.
+
+  def initialize(context):
+      # This initialize function sets any data or variables that you'll use in
+      # your algorithm.  For instance, you'll want to define the trading pair (or
+      # trading pairs) you want to backtest.  You'll also want to define any
+      # parameters or values you're going to use.
+
+      # In our example, we're looking at Neo in USD.
+      context.neo_eth = symbol('neo_usd')
+      context.base_price = None
+      context.current_day = None
+
+      context.RSI_OVERSOLD = 30
+      context.RSI_OVERBOUGHT = 80
+      context.CANDLE_SIZE = '15T'
+
+      context.start_time = time.time()
+
+
+  def handle_data(context, data):
+      # This handle_data function is where the real work is done.  Our data is
+      # minute-level tick data, and each minute is called a frame.  This function
+      # runs on each frame of the data.
+
+      # We flag the first period of each day.
+      # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+      # would only execute once. This method works with minute and daily
+      # frequencies.
+      today = data.current_dt.floor('1D')
+      if today != context.current_day:
+          context.traded_today = False
+          context.current_day = today
+
+      # We're computing the volume-weighted-average-price of the security
+      # defined above, in the context.neo_eth variable.  For this example, we're 
+      # using three bars on the 15 min bars.
+
+      # The frequency attribute determine the bar size. We use this convention
+      # for the frequency alias:
+      # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+      prices = data.history(
+          context.neo_eth,
+          fields='close',
+          bar_count=50,
+          frequency=context.CANDLE_SIZE
+      )
+
+      # Ta-lib calculates various technical indicator based on price and
+      # volume arrays.
+
+      # In this example, we are comp
+      rsi = talib.RSI(prices.values, timeperiod=14)
+
+      # We need a variable for the current price of the security to compare to
+      # the average. Since we are requesting two fields, data.current()
+      # returns a DataFrame with
+      current = data.current(context.neo_eth, fields=['close', 'volume'])
+      price = current['close']
+
+      # If base_price is not set, we use the current value. This is the
+      # price at the first bar which we reference to calculate price_change.
+      if context.base_price is None:
+          context.base_price = price
+
+      price_change = (price - context.base_price) / context.base_price
+      cash = context.portfolio.cash
+
+      # Now that we've collected all current data for this frame, we use
+      # the record() method to save it. This data will be available as
+      # a parameter of the analyze() function for further analysis.
+      record(
+          price=price,
+          volume=current['volume'],
+          price_change=price_change,
+          rsi=rsi[-1],
+          cash=cash
+      )
+
+      # We are trying to avoid over-trading by limiting our trades to
+      # one per day.
+      if context.traded_today:
+          return
+
+      # Since we are using limit orders, some orders may not execute immediately
+      # we wait until all orders are executed before considering more trades.
+      orders = get_open_orders(context.neo_eth)
+      if len(orders) > 0:
+          return
+
+      # Exit if we cannot trade
+      if not data.can_trade(context.neo_eth):
+          return
+
+      # Another powerful built-in feature of the Catalyst backtester is the
+      # portfolio object.  The portfolio object tracks your positions, cash,
+      # cost basis of specific holdings, and more.  In this line, we calculate
+      # how long or short our position is at this minute.   
+      pos_amount = context.portfolio.positions[context.neo_eth].amount
+
+      if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+          log.info(
+              '{}: buying - price: {}, rsi: {}'.format(
+                  data.current_dt, price, rsi[-1]
+              )
+          )
+          # Set a style for limit orders,
+          limit_price = price * 1.005
+          order_target_percent(
+              context.neo_eth, 1, limit_price=limit_price
+          )
+          context.traded_today = True
+
+      elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+          log.info(
+              '{}: selling - price: {}, rsi: {}'.format(
+                  data.current_dt, price, rsi[-1]
+              )
+          )
+          limit_price = price * 0.995
+          order_target_percent(
+              context.neo_eth, 0, limit_price=limit_price
+          )
+          context.traded_today = True
+
+
+  def analyze(context=None, perf=None):
+      end = time.time()
+      log.info('elapsed time: {}'.format(end - context.start_time))
+
+      import matplotlib.pyplot as plt
+      # The base currency of the algo exchange
+      base_currency = context.exchanges.values()[0].base_currency.upper()
+
+      # Plot the portfolio value over time.
+      ax1 = plt.subplot(611)
+      perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+      ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+      # Plot the price increase or decrease over time.
+      ax2 = plt.subplot(612, sharex=ax1)
+      perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+      ax2.set_ylabel('{asset}\n({base})'.format(
+          asset=context.neo_eth.symbol, base=base_currency
+      ))
+
+      transaction_df = extract_transactions(perf)
+      if not transaction_df.empty:
+          buy_df = transaction_df[transaction_df['amount'] > 0]
+          sell_df = transaction_df[transaction_df['amount'] < 0]
+          ax2.scatter(
+              buy_df.index.to_pydatetime(),
+              perf.loc[buy_df.index.floor('1 min'), 'price'],
+              marker='^',
+              s=100,
+              c='green',
+              label=''
+          )
+          ax2.scatter(
+              sell_df.index.to_pydatetime(),
+              perf.loc[sell_df.index.floor('1 min'), 'price'],
+              marker='v',
+              s=100,
+              c='red',
+              label=''
+          )
+
+      ax4 = plt.subplot(613, sharex=ax1)
+      perf.loc[:, 'cash'].plot(
+          ax=ax4, label='Base Currency ({})'.format(base_currency)
+      )
+      ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+      perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+      ax5 = plt.subplot(614, sharex=ax1)
+      perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+      ax5.set_ylabel('Percent\nChange')
+
+      ax6 = plt.subplot(615, sharex=ax1)
+      perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+      ax6.set_ylabel('RSI')
+      ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+      ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+      if not transaction_df.empty:
+          ax6.scatter(
+              buy_df.index.to_pydatetime(),
+              perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+              marker='^',
+              s=100,
+              c='green',
+              label=''
+          )
+          ax6.scatter(
+              sell_df.index.to_pydatetime(),
+              perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+              marker='v',
+              s=100,
+              c='red',
+              label=''
+          )
+      plt.legend(loc=3)
+      start, end = ax6.get_ylim()
+      ax6.yaxis.set_ticks(np.arange(0, end, end/5))
+
+      # Show the plot.
+      plt.gcf().set_size_inches(18, 8)
+      plt.show()
+      pass
+
+
+  if __name__ == '__main__':
+      # The execution mode: backtest or live
+      MODE = 'backtest'
+
+      if MODE == 'backtest':
+          folder = os.path.join(
+              tempfile.gettempdir(), 'catalyst', NAMESPACE
+          )
+          ensure_directory(folder)
+
+          timestr = time.strftime('%Y%m%d-%H%M%S')
+          out = os.path.join(folder, '{}.p'.format(timestr))
+          # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+          run_algorithm(
+              capital_base=10000,
+              data_frequency='minute',
+              initialize=initialize,
+              handle_data=handle_data,
+              analyze=analyze,
+              exchange_name='bitfinex',
+              algo_namespace=NAMESPACE,
+              base_currency='usd',
+              start=pd.to_datetime('2017-10-01', utc=True),
+              end=pd.to_datetime('2017-11-10', utc=True),
+              output=out
+          )
+          log.info('saved perf stats: {}'.format(out))
+
+      elif MODE == 'live':
+          run_algorithm(
+              capital_base=0.5,
+              initialize=initialize,
+              handle_data=handle_data,
+              analyze=analyze,
+              exchange_name='bittrex',
+              live=True,
+              algo_namespace=NAMESPACE,
+              base_currency='usd',
+              live_graph=False
+          )
+
+.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
+
+Notice the difference in performance between the charts above and those seen on 
+`this video tutorial <https://youtu.be/JOBRwst9jUY>`_ at 
+minute 8:10. The buy and sell orders are triggered at the same exact times, but
+the differences result from a more realistic slippage model 
+implemented after the video was recorded, which executes the orders at slighlty
+different prices, but resulting in significant changes in performance of our 
+strategy.
+
+.. _portfolio_optimization:
+
+Portfolio Optimization
+~~~~~~~~~~~~~~~~~~~~~~
+
+Use this code to execute a portfolio optimization model. This strategy will 
+select the portfolio with the maximum Sharpe Ratio. The parameters are set to 
+use 180 days of historical data and rebalance every 30 days. This code was used 
+in writting the following article: 
+`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
+
+.. code-block:: python
+
+  '''
+     You can run this code using the Python interpreter:
+
+     $ python portfolio_optimization.py
+  '''
+
+  from __future__ import division
+  import os
+  import pytz
+  import numpy as np
+  import pandas as pd
+  from scipy.optimize import minimize
+  import matplotlib.pyplot as plt
+  from datetime import datetime
+
+  from catalyst.api import record, symbol, symbols, order_target_percent
+  from catalyst.utils.run_algo import run_algorithm
+
+  np.set_printoptions(threshold='nan', suppress=True)
+
+
+  def initialize(context):
+     # Portfolio assets list
+     context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
+                              'xmr_usdt')
+     context.nassets = len(context.assets)
+     # Set the time window that will be used to compute expected return 
+     # and asset correlations
+     context.window = 180
+     # Set the number of days between each portfolio rebalancing
+     context.rebalance_period = 30                   
+     context.i = 0
+
+     
+  def handle_data(context, data):
+     # Only rebalance at the beggining of the algorithm execution and 
+     # every multiple of the rebalance period
+     if context.i == 0 or context.i%context.rebalance_period == 0:
+         n = context.window
+         prices = data.history(context.assets, fields='price', 
+                               bar_count=n+1, frequency='1d') 
+         pr = np.asmatrix(prices)
+         t_prices = prices.iloc[1:n+1]
+         t_val = t_prices.values
+         tminus_prices = prices.iloc[0:n]
+         tminus_val = tminus_prices.values
+         # Compute daily returns (r)
+         r = np.asmatrix(t_val/tminus_val-1)
+         # Compute the expected returns of each asset with the average 
+         # daily return for the selected time window
+         m = np.asmatrix(np.mean(r, axis=0))
+         # ###
+         stds = np.std(r, axis=0)
+         # Compute excess returns matrix (xr)
+         xr = r - m
+         # Matrix algebra to get variance-covariance matrix
+         cov_m = np.dot(np.transpose(xr),xr)/n
+         # Compute asset correlation matrix (informative only)
+         corr_m = cov_m/np.dot(np.transpose(stds),stds)
+         
+         # Define portfolio optimization parameters
+         n_portfolios = 50000
+         results_array = np.zeros((3+context.nassets,n_portfolios))
+         for p in xrange(n_portfolios):
+             weights = np.random.random(context.nassets)
+             weights /= np.sum(weights)
+             w = np.asmatrix(weights)
+             p_r = np.sum(np.dot(w,np.transpose(m)))*365
+             p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
+             
+             #store results in results array
+             results_array[0,p] = p_r
+             results_array[1,p] = p_std
+             #store Sharpe Ratio (return / volatility) - risk free rate element 
+             #excluded for simplicity
+             results_array[2,p] = results_array[0,p] / results_array[1,p]
+             i = 0
+             for iw in weights:
+                 results_array[3+i,p] = weights[i]
+                 i += 1
+         
+         #convert results array to Pandas DataFrame
+         results_frame = pd.DataFrame(np.transpose(results_array),
+                            columns=['r','stdev','sharpe']+context.assets)
+         #locate position of portfolio with highest Sharpe Ratio
+         max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
+         #locate positon of portfolio with minimum standard deviation
+         min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
+         
+         #order optimal weights for each asset
+         for asset in context.assets:
+             if data.can_trade(asset):
+                 order_target_percent(asset, max_sharpe_port[asset])
+         
+         #create scatter plot coloured by Sharpe Ratio
+         plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
+         plt.xlabel('Volatility')
+         plt.ylabel('Returns')
+         plt.colorbar()
+         #plot red star to highlight position of portfolio with highest Sharpe Ratio
+         plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
+         #plot green star to highlight position of minimum variance portfolio
+         plt.show()
+         print(max_sharpe_port)
+         record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
+     context.i += 1
+     
+         
+  def analyze(context=None, results=None):
+     # Form DataFrame with selected data
+     data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
+     
+     # Save results in CSV file
+     filename = os.path.splitext(os.path.basename(__file__))[0]
+     data.to_csv(filename + '.csv')
+
+
+  # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
+  start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+  end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) 
+  results = run_algorithm(initialize=initialize,
+                          handle_data=handle_data,
+                          analyze=analyze,
+                          start=start,
+                          end=end,
+                          exchange_name='poloniex',
+                          capital_base=100000, )
+
+.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
+   :align: center
+
+
+

docs/source/index.rst

@@ -12,6 +12,8 @@ Table of Contents
    jupyter
    live-trading
    naming-convention
+   example-algos
+   utilities
    videos
    resources
    development-guidelines

docs/source/install.rst

@@ -6,7 +6,154 @@ Like any other piece of software, Catalyst has a number of dependencies
 (other software on which it depends to run) that you will need to install, as 
 well. We recommend using a software named ``Conda`` that will manage all 
 these dependencies for you, and set up the environment needed to get you up 
-and running as easily as possible. See :ref:`Installing with Conda <conda>`.
+and running as easily as possible. This is the recommended installation method
+for Windows, MacOS and Linux. See :ref:`Installing with Conda <conda>`.
+
+What conda does is create a pre-configured environment, and inside that 
+environment install Catalyst using ``pip``, Python's package manager. Thus, 
+as an alternative installation method for MacOS and Linux, you can install 
+Catalyst directly with ``pip`` (we recommend in combination with a virtual 
+environemnt). See :ref:`Installing with pip <pip>`.
+
+Regardless of the method, each operating system (OS), has its own 
+prerequisites, make sure to review the corresponding sections for your system:
+:ref:`Linux <linux>`, :ref:`MacOS <macos>` and :ref:`Windows <windows>`.
+
+.. _conda:
+
+Installing with ``conda``
+-------------------------
+
+The preferred method to install Catalyst is via the ``conda`` package manager, 
+which comes as part of Continuum Analytics' `Anaconda
+<http://continuum.io/downloads>`_ distribution.
+
+The primary advantage of using Conda over ``pip`` is that conda natively
+understands the complex binary dependencies of packages like ``numpy`` and
+``scipy``.  This means that ``conda`` can install Catalyst and its 
+dependencies without requiring the use of a second tool to acquire Catalyst's 
+non-Python dependencies.
+
+  For Windows, you will first need to install the *Microsoft Visual C++ 
+  Compiler for Python 2.7*. Follow the instructions on the :ref:`Windows 
+  <windows>` section and come back here.
+
+For instructions on how to install ``conda``, see the `Conda Installation
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
+you can install MiniConda, which is a smaller footprint (fewer packages and 
+smaller size) than its big brother Anaconda, but it still contains all the 
+main packages needed. To install MiniConda, you can follow these steps:
+
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
+   for your Operating System.
+2. Install MiniConda. See the `Installation Instructions 
+   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal 
+   window, which should print the list of packages installed with Conda.
+
+  For Windows, if you accepted the default installation options, you didn't 
+  check an option to add Conda to the PATH, so trying to run ``conda`` from
+  a regular ``Command Prompt`` will result in the following error: ``'conda' 
+  is no recognized as an internal or external command, operatble program or 
+  batch file``. That's to be expected. You will nee to launch an ``Anaconda 
+  Prompt`` that was added at installation time to your list of programs 
+  available from the Start menu. 
+
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+
+     To download, simply click on the 'Raw' button and save the file locally 
+     to a folder you can remember. Make sure that the file gets saved with the
+     ``.yml`` extension, and nothing like a ``.txt`` file or anything else.
+
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
+   saved the above ``python2.7-environment.yml`` file.
+
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new 
+   session to run Catalyst):
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+5. Verify that Catalyst is install correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
+above failed for any reason, you can try setting up the environment manually 
+with the following steps:
+
+1. If the above installation failed, and you have a partially set up catalyst
+   environment, remove it first. If you are starting from scratch, proceed to 
+   step #2:
+
+   .. code-block:: bash
+
+      conda env remove --name catalyst
+
+2. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy zlib
+
+3. Activate the environment:
+
+   **Linux or MacOS:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+4. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+5. Verify that Catalyst is installed correctly:
+
+   .. code-block:: bash
+
+     catalyst --version
+
+   which should display the current version.
+
+Congratulations! You now have Catalyst properly installed.
+
+.. _pip:
 
 Installing with ``pip``
 -----------------------
@@ -28,15 +175,21 @@ Because LAPACK and the CPython headers are non-Python dependencies, the
 correctway to install them varies from platform to platform.  If you'd rather 
 use a single tool to install Python and non-Python dependencies, or if you're 
 already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
-distribution, you can skip to the :ref:`Installing with Conda <conda>` 
-section.
+distribution, refer to the :ref:`Installing with Conda <conda>` section.
 
-Once you've installed the necessary additional dependencies (see below for
-your particular platform), you should be able to simply run
+Once you've installed the necessary additional dependencies for your system 
+(see below for your particular platform: :ref:`Linux`, :ref:`MacOS` or 
+:ref:`Windows`), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install enigma-catalyst
+   $ pip install enigma-catalyst matplotlib
+
+Note that in the command above we install two different packages. The second 
+one, ``matplotlib`` is a visualization library. While it's not strictly 
+required to run catalyst simulations or live trading, it comes in very handy
+to visualize the performance of your algorithms, and for this reason we 
+recommend you install it, as well.
 
 If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
@@ -50,153 +203,7 @@ summarized version:
    $ pip install virtualenv
    $ virtualenv catalyst-venv
    $ source ./catalyst-venv/bin/activate
-   $ pip install enigma-catalyst
-
-Though not required by Catalyst directly, our example algorithms use 
-matplotlib to visually display the results of the trading algorithms. If you 
-wish to run any examples or use matplotlib during development, it can be 
-installed using:
-
-.. code-block:: bash
-
-    $ pip install matplotlib
-
-GNU/Linux
-~~~~~~~~~
-
-On `Debian-derived`_ Linux distributions, you can acquire all the necessary
-binary dependencies from ``apt`` by running:
-
-.. code-block:: bash
-
-   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
-
-On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
-following should be sufficient to acquire the necessary additional
-dependencies:
-
-.. code-block:: bash
-
-   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
-
-On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
-
-.. code-block:: bash
-
-   $ pacman -S lapack gcc gcc-fortran pkg-config
-
-.. Commenting it out until Catalyst fully supports Python 3.X
-..
-.. There are also AUR packages available for installing `Python 3.4
-.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
-.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
-.. Python 2 is also installable via:
-
-.. 
-
-..   $ pacman -S python2
-
-OSX
-~~~
-
-The version of Python shipped with OSX by default is generally out of date, 
-and has a number of quirks because it's used directly by the operating system.
-For these reasons, many developers choose to install and use a separate Python
-installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
-to `Installing Python on OSX <http://docs.python-guide.org/en/latest/>`_, 
-which explains how to install Python with the `Homebrew`_ manager.
-
-Assuming you've installed Python with Homebrew, you'll also likely need the
-following brew packages:
-
-.. code-block:: bash
-
-   $ brew install freetype pkg-config gcc openssl
-
-OSX + virtualenv + matplotlib
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-A note about using matplotlib in virtual enviroments on OSX: it may be 
-necessary to run
-
-.. code-block:: bash
-
-   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
-
-in order to override the default ``macosx`` backend for your system, which 
-may not be accessible from inside the virtual environment. This will allow 
-Catalyst to open matplotlib charts from within a virtual environment, which 
-is useful for displaying the performance of your backtests.  To learn more 
-about matplotlib backends, please refer to the
-`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
-
-.. _windows:
-
-Windows
-~~~~~~~
-
-In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
-<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
-package contains the compiler and the set of system headers necessary for 
-producing binary wheels for Python 2.7 packages. If it's not already in your 
-system, download it and install it before proceeding to the next step.
-
-For windows, the easiest and best supported way to install Catalyst is to use
-:ref:`Conda <conda>`.
-
-Some problems we have encountered installing the **Visual C++ Compiler** 
-mentioned above are as follows:
-
-- **The system administrator has set policies to prevent this installation**.
-  
-  In some systems, there is a default *Windows Software Restriction* policy 
-  that prevents the installation of some software packages like this one. 
-  You'll have to change the Registry to circumvent this:
-
-  - Click ``Start``, and search for ``regedit`` and launch the 
-    ``Registry Editor``
-  - Navigate to the following folder:
-    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
-  - If there is an entry for ``DisableMSI``, set the Value data to 0.
-  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
-    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
-    default you get 0 as the Value Data)
-
-|
-- **The installer has encountered an unexpected error installing this package. 
-  This may indicate a problem with this package. The error code is 2503.**
-
-  We have observed this when trying to install a package without enough 
-  administrator permissions. Even when you are logged in as an Administrator, 
-  you have to explictily install this package with administrator privileges:
-
-  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
-  - Right click on it and choose ``Run as administrator``
-  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
-  - Run ``msiexec /i VCForPython27.msi``
-
-
-Amazon Linux AMI
-~~~~~~~~~~~~~~~~
-
-The packages ``pip`` and ``setuptools`` that come shipped by default are very 
-outdated. Thus, you first need to run:
-
-.. code-block:: bash
-
-   pip install --upgrade pip setuptools
-
-The default installation is also missing the C and C++ compilers, which you 
-install by:
-
-.. code-block:: bash
-
-   sudo yum install gcc gcc-c++
-
-Then you should follow the regular installation instructions outlined at the 
-beginning of this page.
-
+   $ pip install enigma-catalyst matplotlib
 
 Troubleshooting ``pip`` Install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -287,99 +294,150 @@ Troubleshooting ``pip`` Install
       sudo apt-get install python-dev
 
 
-.. _conda:
+.. _linux:
 
-Installing with ``conda``
--------------------------
+GNU/Linux Requirements
+----------------------
 
-Another way to install Catalyst is via the ``conda`` package manager, which
-comes as part of Continuum Analytics' `Anaconda
-<http://continuum.io/downloads>`_ distribution.
+On `Debian-derived`_ Linux distributions, you can acquire all the necessary
+binary dependencies from ``apt`` by running:
 
-The primary advantage of using Conda over ``pip`` is that conda natively
-understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Catalyst and its 
-dependencies without requiring the use of a second tool to acquire Catalyst's 
-non-Python dependencies.
+.. code-block:: bash
 
-  For Windows, you will need the *Microsoft Visual C++ Compiler for Python 
-  2.7*. Follow the instructions on the :ref:`Windows` section and come back 
-  here.
+   $ sudo apt-get install libatlas-base-dev python-dev gfortran pkg-config libfreetype6-dev
 
-For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
-you can install MiniConda, which is a smaller footprint (fewer packages and 
-smaller size) than its big brother Anaconda, but it still contains all the 
-main packages needed. To install MiniConda, you can follow these steps:
+On recent `RHEL-derived`_ derived Linux distributions (e.g. Fedora), the
+following should be sufficient to acquire the necessary additional
+dependencies:
 
-1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
-   for your Operating System.
-2. Install MiniConda. See the `Installation Instructions 
-   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
-3. Ensure the correct installation by running ``conda list`` in a Terminal 
-   window, which should print the list of packages installed with Conda.
+.. code-block:: bash
 
-Once either Conda or MiniConda has been set up you can install Catalyst:
+   $ sudo dnf install atlas-devel gcc-c++ gcc-gfortran libgfortran python-devel redhat-rep-config
 
-1. Download the file `python2.7-environment.yml 
-   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
-2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
-   saved the above ``python2.7-environment.yml`` file.
-3. Install using this file. This step can take about 5-10 minutes to install.
+On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
-   .. code-block:: bash
+.. code-block:: bash
 
-      conda env create -f python2.7-environment.yml
+   $ pacman -S lapack gcc gcc-fortran pkg-config
 
-4. Activate the environment (which you need to do every time you start a new 
-   session to run Catalyst):
+.. Commenting it out until Catalyst fully supports Python 3.X
+..
+.. There are also AUR packages available for installing `Python 3.4
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-   **Linux or OSX:**
+.. 
 
-   .. code-block:: bash
+..   $ pacman -S python2
 
-      source activate catalyst
+Amazon Linux AMI Notes
+~~~~~~~~~~~~~~~~~~~~~~
 
-   **Windows:**
+The packages ``pip`` and ``setuptools`` that come shipped by default are very 
+outdated. Thus, you first need to run:
 
-   .. code-block:: bash
+.. code-block:: bash
 
-      activate catalyst
+   pip install --upgrade pip setuptools
 
-Congratulations! You now have Catalyst installed.
+The default installation is also missing the C and C++ compilers, which you 
+install by:
 
-Troubleshooting ``conda`` Install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. code-block:: bash
 
-If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
-above failed for any reason, you can try setting up the environment manually 
-with the following steps:
+   sudo yum install gcc gcc-c++
 
-1. Create the environment:
+Then you should follow the regular installation instructions outlined at the 
+beginning of this page.
 
-   .. code-block:: bash
 
-      conda create --name catalyst python=2.7 scipy zlib
+.. _MacOS:
 
-2. Activate the environment:
+MacOS Requirements
+------------------
 
-   **Linux or OSX:**
+The version of Python shipped with MacOS by default is generally out of date, 
+and has a number of quirks because it's used directly by the operating system.
+For these reasons, many developers choose to install and use a separate Python
+installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
+to `Installing Python on MacOS <http://docs.python-guide.org/en/latest/>`_, 
+which explains how to install Python with the `Homebrew`_ manager.
 
-   .. code-block:: bash
+Assuming you've installed Python with Homebrew, you'll also likely need the
+following brew packages:
 
-      source activate catalyst
+.. code-block:: bash
 
-   **Windows:**
+   $ brew install freetype pkg-config gcc openssl
 
-   .. code-block:: bash
+MacOS + virtualenv + matplotlib
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-      activate catalyst
+A note about using matplotlib in virtual enviroments on MacOS: it may be 
+necessary to run
 
-3. Install the Catalyst inside the environment:
+.. code-block:: bash
 
-   .. code-block:: bash
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
 
-      pip install enigma-catalyst matplotlib
+in order to override the default ``MacOS`` backend for your system, which 
+may not be accessible from inside the virtual environment. This will allow 
+Catalyst to open matplotlib charts from within a virtual environment, which 
+is useful for displaying the performance of your backtests.  To learn more 
+about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+.. _windows:
+
+Windows Requirements
+--------------------
+
+In Windows, you will first need to install the `Microsoft Visual C++ Compiler 
+for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
+package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python 2.7 packages. If it's not already in your 
+system, download it and install it before proceeding to the next step.
+
+Once you have the above compiler installed, the easiest and best supported way 
+to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't 
+any problems installing the compiler, jump to the :ref:`Conda <conda>` section, 
+otherwise keep on reading to troubleshoot the C++ compiler installtion.
+
+Some problems we have encountered installing the **Visual C++ Compiler** 
+mentioned above are as follows:
+
+- **The system administrator has set policies to prevent this installation**.
+  
+  In some systems, there is a default *Windows Software Restriction* policy 
+  that prevents the installation of some software packages like this one. 
+  You'll have to change the Registry to circumvent this:
+
+  - Click ``Start``, and search for ``regedit`` and launch the 
+    ``Registry Editor``
+  - Navigate to the following folder:
+    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
+  - If the last folder does not exist, create it by right-clicking on the 
+    parent folder and choosing -> ``New`` -> ``Key`` and typing ``Installer``
+  - If there is an entry for ``DisableMSI``, set the Value data to 0.
+  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
+    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
+    default you get 0 as the Value Data)
+
+|
+- **The installer has encountered an unexpected error installing this package. 
+  This may indicate a problem with this package. The error code is 2503.**
+
+  We have observed this when trying to install a package without enough 
+  administrator permissions. Even when you are logged in as an Administrator, 
+  you have to explictily install this package with administrator privileges:
+
+  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
+  - Right click on it and choose ``Run as administrator``
+  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
+  - Run ``msiexec /i VCForPython27.msi``
 
 Getting Help
 ------------

docs/source/releases.rst

@@ -2,9 +2,72 @@
 Release Notes
 =============
 
+Version 0.3.10
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with fetching assets with daily frequency
+
+Version 0.3.9
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-28
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed sortino warning issues (:issue:`77`)
+- Adjusted computation of last candle of data.history (:issue:`71`)
+
+Build
+~~~~~
+- Added capital_base parameter to live mode to limit cash (:issue:`79`)
+- Added support for csv ingestion (:issue:`65`)
+- Improved cash display in running stats (:issue:`80`)
+
+
+Version 0.3.8
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed a warning filter issue introduced with the latest release
+
+Version 0.3.7
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an SSL cert issue (:issue:`64`)
+- Fixed cumulative stats warnings (:issue:`63`)
+- Disabled auto-ingestion because of unresolved caching issues (:issue:`47`)
+- Standardized live-trading stats (:issue:`61`)
+
+Build
+~~~~~
+
+- Added a mean-reversion sample algo
+- Added minutely stats in the analyze() function (:issue:`62`)
+- Added specificity to some error messages
+
+Version 0.3.6
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an issue with single bar data.history() (:issue:`55`)
+
 Version 0.3.5
 ^^^^^^^^^^^^^
-**Release Date**: 2017-11-2
+**Release Date**: 2017-11-4
 
 Bug Fixes
 ~~~~~~~~~
@@ -22,7 +85,8 @@ Bug Fixes
 - Fixed issue with sell orders in backtesting
 - Fixed data frequency issues with data.history() in backtesting
 - Fixed an issue with can_trade()
-- Reduced the commission and slippage values to account for lower volume transactions
+- Reduced the commission and slippage values to account for lower volume
+  transactions
 
 Build
 ~~~~~
@@ -33,12 +97,18 @@ Documentation
 ~~~~~~~~~~~~~
 
 - Improved installation notes for Windows C++ compiler and Conda
-- Addition of `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
-- Addition of `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
-- Addition of `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
-- Addition of `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
-- Addition of `Development Guidelines <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
-- Addition of `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
+- Addition of
+  `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
+- Addition of
+  `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
+- Addition of
+  `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
+- Addition of
+  `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
+- Addition of `Development Guidelines
+  <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
+- Addition of
+  `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
 - Updated code docstrings
 
 
@@ -88,9 +158,11 @@ Bug Fixes
 ~~~~~~~~~
 
 - Fixed OS-dependent path issue in data bundle
-- Changed handling of empty ``auth.json``, instead of throwing an error for missing file
+- Changed handling of empty ``auth.json``, instead of throwing an error for
+  missing file
 - Updated ``etc/python2.7-environment.yml`` to work with Catalyst version 0.3
-- Updated ``catalyst/examples/buy_and_hodl.py``  and ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
+- Updated ``catalyst/examples/buy_and_hodl.py``  and
+  ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
 
 
 Version 0.3
@@ -109,15 +181,19 @@ Version 0.2.dev5
 ^^^^^^^^^^^^^^^^
 **Release Date**: 2017-10-03
 
-- Fixes bug in data.history function that was formatting 'volume' data as integers, now they are returned as floats with up to 9 decimals of precision. Data bundles redone.
+- Fixes bug in data.history function that was formatting 'volume' data as
+  integers, now they are returned as floats with up to 9 decimals of precision.
+  Data bundles redone.
 
-Version 0.2.dev4 
+Version 0.2.dev4
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-09-20
 
-- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal places. Pricing resolution of daily data remains set to 9 decimal places.
-- The current data bundle takes 340MB compressed for download, and 460MB uncompressed on disk for Catalyst to use.
+- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal
+  places. Pricing resolution of daily data remains set to 9 decimal places.
+- The current data bundle takes 340MB compressed for download, and 460MB
+  uncompressed on disk for Catalyst to use.
 
 Version 0.2.dev3
 ^^^^^^^^^^^^^^^^
@@ -126,11 +202,14 @@ Version 0.2.dev3
 
 - 1-minute resolution OHLCV data bundle for backtesting from Poloniex exchange
 - Implementation of trading of fractional crypto assets (i.e. 0.01 BTC)
-- Minimum trade size of a coin can be configured on a per-coin basis, defaults to 0.00000001 in backtesting (most exchanges set the minimum trade to larger amounts, which will impact live trading)
+- Minimum trade size of a coin can be configured on a per-coin basis, defaults
+  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger
+  amounts, which will impact live trading)
 - Increased pricing resolution from 3 to 9 decimal places
-- The current data bundle takes 40MB compressed for download, and 99MB uncompressed on disk for Catalyst to use.
+- The current data bundle takes 40MB compressed for download, and 99MB
+  uncompressed on disk for Catalyst to use.
 
-Version 0.2.dev2 
+Version 0.2.dev2
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-09-07
@@ -146,24 +225,29 @@ Version 0.2.dev1
 
   - Comprehensive trading functionality against exchanges Bitfinex and Bittrex.
   - Support for all trading pairs available on each exchange.
-  - Multiple algorithms can trade simultaneously against a single exchange using the same account.
-  - Each algorithm has a persisted state (i.e. algorithm can be stopped and restarted preserving the state without data loss) that tracks all open orders, executed transactions and portfolio positions.
+  - Multiple algorithms can trade simultaneously against a single exchange
+    using the same account.
+  - Each algorithm has a persisted state (i.e. algorithm can be stopped and
+    restarted preserving the state without data loss) that tracks all open
+    orders, executed transactions and portfolio positions.
 
 - Minute by minute portfolio performance metrics.
 
-  - Daily summary performance statistics compatible with pyfolio, a Python library for performance and risk analysis of financial portfolios
+  - Daily summary performance statistics compatible with pyfolio, a Python
+    library for performance and risk analysis of financial portfolios
 
 Version 0.1.dev9
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-08-28
 
-- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex exchange directly
+- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex
+  exchange directly
 - Change of bundle storage provider from Dropbox to AWS
 - Fix issue with 1/1000 scaling issue of prices in bundle
 
 Version 0.1.dev8
-^^^^^^^^^^^^^^^^ 
+^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-08-18
 
@@ -183,4 +267,3 @@ Version 0.1.dev6
 **Release Date**: 2017-07-13
 
 - Initial public release
-

docs/source/utilities.rst

@@ -0,0 +1,149 @@
+Utilities
+=========
+
+This section covers a variety of utilites that provide complimentary 
+functionality to your trading algorithms. These are code snippets that you can
+add to any algorithm to add the desired functionality.
+
+If you are looking for example trading algorithms, see the corresponding section.
+
+Output to CSV file
+~~~~~~~~~~~~~~~~~~
+
+Add this script to the analyze method to create and save a CSV file with the 
+results from the trading algorithm. This file will include the default 
+parameters of the results DataFrame plus any recorded variables and will be 
+saved in the same location where your trading algorithm is saved. The exact 
+script that you need to use depends on the interface that you are using to run 
+your trading algorithm, which could be the CLI or a Python Interpreter.
+
+1. Script to use with CLI:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import sys
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(basename(sys.argv[3]))[0]
+          results.to_csv(filename + '.csv')
+
+2. Script to use with Python Interpreter:
+
+   .. code-block:: python
+
+      def analyze(context=None, results=None):
+          import os
+          from os.path import basename
+          
+          # Save results in CSV file
+          filename = os.path.splitext(os.path.basename(__file__))[0]
+          results.to_csv(filename + '.csv')
+
+Extracting market data
+~~~~~~~~~~~~~~~~~~~~~~
+
+Use this script to save the price and volume data of one cryptoasset in a CSV 
+file, which will be saved in the same location and with the same name as your 
+Python file. To get custom data, simply modify the asset's symbol and the dates. 
+Run this script directly from your development environment: python scriptname.py, 
+where the contents of 'scriptname.py' are as follows. Two different version are 
+provided as an example for daily- and minute-resolution data respectively:
+
+Simpler case for daily data
+
+.. code-block:: python
+
+   import os
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+   def initialize(context):
+      # Portfolio assets list
+      context.asset = symbol('btc_usdt') # Bitcoin on Poloniex
+                          
+   def handle_data(context, data):
+      # Variables to record for a given asset: price and volume
+      price = data.current(context.asset, 'price')
+      volume = data.current(context.asset, 'volume')
+      record(price=price, volume=volume)
+      
+   def analyze(context=None, results=None):
+      
+      # Generate DataFrame with Price and Volume only
+      data = results[['price','volume']]
+      
+      # Save results in CSV file
+      filename = os.path.splitext(os.path.basename(__file__))[0]
+      data.to_csv(filename + '.csv')
+      
+   ''' Bitcoin data is available on Poloniex since 2015-3-1. 
+       Dates vary for other tokens. In the example below, we choose the 
+       full month of July of 2017.
+   '''
+   start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data, 
+                           analyze=analyze, 
+                           start=start, 
+                           end=end, 
+                           exchange_name='poloniex', 
+                           capital_base=10000, 
+                           base_currency = 'usdt')
+
+More versatile case for minute data
+
+.. code-block:: python
+
+   import os
+   import csv
+   import pytz
+   from datetime import datetime
+
+   from catalyst.api import record, symbol, symbols
+   from catalyst.utils.run_algo import run_algorithm
+
+
+   def initialize(context):
+   # Portfolio assets list
+   context.asset     = symbol('btc_usdt') # Bitcoin on Poloniex
+
+   # Creates a .CSV file with the same name as this script to store results
+   context.csvfile   = open(os.path.splitext(
+                            os.path.basename(__file__))[0]+'.csv', 'w+')
+   context.csvwriter = csv.writer(context.csvfile)
+
+   def handle_data(context, data):
+   # Variables to record for a given asset: price and volume
+   # Other options include 'open', 'high', 'open', 'close'
+   # Please note that 'price' equals 'close'
+   date   = context.blotter.current_dt     # current time in each iteration
+   price  = data.current(context.asset, 'price')
+   volume = data.current(context.asset, 'volume')
+
+   # Writes one line to CSV on each iteration with the chosen variables    
+   context.csvwriter.writerow([date,price,volume])
+
+   def analyze(context=None, results=None):
+   # Close open file properly at the end
+   context.csvfile.close()
+
+
+   # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.    
+   start = datetime(2017, 7, 30, 0, 0, 0, 0, pytz.utc)
+   end = datetime(2017, 7, 31, 0, 0, 0, 0, pytz.utc) 
+   results = run_algorithm(initialize=initialize, 
+                           handle_data=handle_data,
+                           analyze=analyze, 
+                           start=start, 
+                           end=end,
+                           exchange_name='poloniex', 
+                           data_frequency='minute',
+                           base_currency ='usdt', 
+                           capital_base=10000 )

docs/source/videos.rst

@@ -22,5 +22,37 @@ Where things go smoothly:
 
 |
 Where things don't:
+
+.. raw:: html
    
-   Coming up next!
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/qLkQcWlUBy8" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Backtesting a Strategy
+----------------------
+
+This is the first video of a two-part series on using Catalyst for algorithmic 
+trading. This video implements a simple momentum strategy based on 
+`mean reversion <example-algos.html#mean-reversion>`_: when the cryptoasset
+goes up quickly, we’re going to buy; when it goes down quickly, we’re going to 
+sell. Hopefully, we’ll ride the waves.
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/JOBRwst9jUY" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Live Trading a Strategy
+-----------------------
+
+This is the second part of the two-part series on using Catalyst for algorithmic
+trading. Having backtested `our strategy <example-algos.html#mean-reversion>`_ 
+in the previous video, we now take it to trade live against the Bittrex exchange.
+
+.. raw:: html
+
+	<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
+|
+|

docs/source/whatsnew/0.9.0.txt

@@ -105,7 +105,7 @@ None
 Miscellaneous
 ~~~~~~~~~~~~~
 
-* Adds :class:`~zipline.testing.fixtures.ZiplineTestCase` which provides hooks
+* Adds :class:`~zipline.testing.fixtures.CatalystTestCase` which provides hooks
   to consume test fixtures. Fixtures are things like:
   :class:`~zipline.testing.fixtures.WithAssetFinder` which will make
   ``self.asset_finder`` available to your test with some mock data

etc/requirements.txt

@@ -80,3 +80,6 @@ empyrical==0.2.1
 
 tables==3.3.0
 
+#Catalyst dependencies
+ccxt==1.10.251
+

tests/calendars/test_calendar_dispatcher.py

@@ -6,12 +6,12 @@ from catalyst.errors import (
     CyclicCalendarAlias,
     InvalidCalendarName,
 )
-from catalyst.testing import ZiplineTestCase
+from catalyst.testing import CatalystTestCase
 from catalyst.utils.calendars.calendar_utils import TradingCalendarDispatcher
 from catalyst.utils.calendars.exchange_calendar_ice import ICEExchangeCalendar
 
 
-class CalendarAliasTestCase(ZiplineTestCase):
+class CalendarAliasTestCase(CatalystTestCase):
 
     @classmethod
     def init_class_fixtures(cls):

tests/data/bundles/test_core.py

@@ -22,7 +22,7 @@ from catalyst.testing import (
     subtest,
     str_to_seconds,
 )
-from catalyst.testing.fixtures import WithInstanceTmpDir, ZiplineTestCase, \
+from catalyst.testing.fixtures import WithInstanceTmpDir, CatalystTestCase, \
     WithDefaultDateBounds
 from catalyst.testing.predicates import (
     assert_equal,
@@ -45,7 +45,7 @@ _1_ns = pd.Timedelta(1, unit='ns')
 
 class BundleCoreTestCase(WithInstanceTmpDir,
                          WithDefaultDateBounds,
-                         ZiplineTestCase):
+                         CatalystTestCase):
 
     START_DATE = pd.Timestamp('2014-01-06', tz='utc')
     END_DATE = pd.Timestamp('2014-01-10', tz='utc')

tests/data/bundles/test_quandl.py

@@ -17,14 +17,14 @@ from catalyst.testing import (
     tmp_dir,
     patch_read_csv,
 )
-from catalyst.testing.fixtures import ZiplineTestCase
+from catalyst.testing.fixtures import CatalystTestCase
 from catalyst.testing.predicates import (
     assert_equal,
 )
 from catalyst.utils.functional import apply
 
 
-class QuandlBundleTestCase(ZiplineTestCase):
+class QuandlBundleTestCase(CatalystTestCase):
     symbols = 'AAPL', 'BRK_A', 'MSFT', 'ZEN'
     asset_start = pd.Timestamp('2014-01', tz='utc')
     asset_end = pd.Timestamp('2015-01', tz='utc')

tests/data/bundles/test_yahoo.py

@@ -10,12 +10,12 @@ from catalyst.data.bundles.core import _make_bundle_core
 from catalyst.data.bundles import yahoo_equities
 from catalyst.lib.adjustment import Float64Multiply
 from catalyst.testing import test_resource_path, tmp_dir, read_compressed
-from catalyst.testing.fixtures import WithResponses, ZiplineTestCase
+from catalyst.testing.fixtures import WithResponses, CatalystTestCase
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.calendars import get_calendar
 
 
-class YahooBundleTestCase(WithResponses, ZiplineTestCase):
+class YahooBundleTestCase(WithResponses, CatalystTestCase):
     symbols = 'AAPL', 'IBM', 'MSFT'
     columns = 'open', 'high', 'low', 'close', 'volume'
     asset_start = pd.Timestamp('2014-01-02', tz='utc')

tests/data/test_dispatch_bar_reader.py

@@ -11,6 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+
+'''
+#   ZIPLINE legacy test: Catalyst does not use DispatchBarReader, and thus
+#   this test suite is irrelevant, and is commented out in its entirety
+
 from numpy import array, nan
 from numpy.testing import assert_almost_equal
 from pandas import DataFrame, Timestamp
@@ -31,7 +37,7 @@ from catalyst.testing.fixtures import (
     WithBcolzEquityDailyBarReader,
     WithBcolzFutureMinuteBarReader,
     WithTradingSessions,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 OHLC = ['open', 'high', 'low', 'close']
@@ -40,7 +46,7 @@ OHLC = ['open', 'high', 'low', 'close']
 class AssetDispatchSessionBarTestCase(WithBcolzEquityDailyBarReader,
                                       WithBcolzFutureMinuteBarReader,
                                       WithTradingSessions,
-                                      ZiplineTestCase):
+                                      CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('us_futures', 'NYSE')
     TRADING_CALENDAR_PRIMARY_CAL = 'us_futures'
@@ -169,7 +175,7 @@ class AssetDispatchSessionBarTestCase(WithBcolzEquityDailyBarReader,
 
 class AssetDispatchMinuteBarTestCase(WithBcolzEquityMinuteBarReader,
                                      WithBcolzFutureMinuteBarReader,
-                                     ZiplineTestCase):
+                                     CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('us_futures', 'NYSE')
     TRADING_CALENDAR_PRIMARY_CAL = 'us_futures'
@@ -330,3 +336,4 @@ class AssetDispatchMinuteBarTestCase(WithBcolzEquityMinuteBarReader,
         for i, (sid, expected, msg) in enumerate(expected_per_sid):
             for j, result in enumerate(results):
                 assert_almost_equal(result[:, i], expected[j], err_msg=msg)
+'''

tests/data/test_minute_bars.py

@@ -38,8 +38,8 @@ from pandas import (
 from catalyst.data.bar_reader import NoDataOnDate
 from catalyst.data.minute_bars import (
     BcolzMinuteBarMetadata,
-    BcolzMinuteBarWriter,
-    BcolzMinuteBarReader,
+#    BcolzMinuteBarWriter,
+#    BcolzMinuteBarReader,
     BcolzMinuteOverlappingData,
     US_EQUITIES_MINUTES_PER_DAY,
     BcolzMinuteWriterColumnMismatch,
@@ -47,24 +47,29 @@ from catalyst.data.minute_bars import (
     H5MinuteBarUpdateReader,
 )
 
+from catalyst.exchange.exchange_bcolz import (
+    BcolzExchangeBarWriter,
+    BcolzExchangeBarReader,
+)
+
 from catalyst.testing.fixtures import (
     WithAssetFinder,
     WithInstanceTmpDir,
     WithTradingCalendars,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 # Calendar is set to cover several half days, to check a case where half
 # days would be read out of order in cases of windows which spanned over
 # multiple half days.
-TEST_CALENDAR_START = Timestamp('2014-06-02', tz='UTC')
-TEST_CALENDAR_STOP = Timestamp('2015-12-31', tz='UTC')
+TEST_CALENDAR_START = Timestamp('2015-06-02', tz='UTC')
+TEST_CALENDAR_STOP = Timestamp('2016-12-31', tz='UTC')
 
 
 class BcolzMinuteBarTestCase(WithTradingCalendars,
                              WithAssetFinder,
                              WithInstanceTmpDir,
-                             ZiplineTestCase):
+                             CatalystTestCase):
 
     ASSET_FINDER_EQUITY_SIDS = 1, 2
 
@@ -87,14 +92,14 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
 
         self.dest = self.instance_tmpdir.getpath('minute_bars')
         os.makedirs(self.dest)
-        self.writer = BcolzMinuteBarWriter(
-            self.dest,
-            self.trading_calendar,
-            TEST_CALENDAR_START,
-            TEST_CALENDAR_STOP,
-            US_EQUITIES_MINUTES_PER_DAY,
+        self.writer = BcolzExchangeBarWriter(
+            rootdir=self.dest,
+            calendar=self.trading_calendar,
+            start_session=TEST_CALENDAR_START,
+            end_session=TEST_CALENDAR_STOP,
+            data_frequency='minute',
         )
-        self.reader = BcolzMinuteBarReader(self.dest)
+        self.reader = BcolzExchangeBarReader(self.dest)
 
     def test_version(self):
         metadata = self.reader._get_metadata()
@@ -152,7 +157,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
         )
 
         # Create a new writer with `ohlc_ratios_per_sid` defined.
-        writer_with_ratios = BcolzMinuteBarWriter(
+        writer_with_ratios = BcolzExchangeBarWriter(
             self.dest,
             self.trading_calendar,
             TEST_CALENDAR_START,
@@ -161,7 +166,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
             ohlc_ratios_per_sid={sid: 25},
         )
         writer_with_ratios.write_sid(sid, data)
-        reader = BcolzMinuteBarReader(self.dest)
+        reader = BcolzExchangeBarReader(self.dest)
 
         open_price = reader.get_value(sid, minute, 'open')
         self.assertEquals(10.0, open_price)
@@ -449,7 +454,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
         # of appending new days will be writing to an existing directory.
         cday = self.trading_calendar.schedule.index.freq
         new_end_session = TEST_CALENDAR_STOP + cday
-        writer = BcolzMinuteBarWriter.open(self.dest, new_end_session)
+        writer = BcolzExchangeBarWriter.open(self.dest, new_end_session)
         next_day_minute = dt + cday
         new_data = DataFrame(
             data=ohlcv,
@@ -457,7 +462,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
         writer.write_sid(sid, new_data)
 
         # Get a new reader to test updated calendar.
-        reader = BcolzMinuteBarReader(self.dest)
+        reader = BcolzExchangeBarReader(self.dest)
 
         second_minute = dt + Timedelta(minutes=1)
 
@@ -802,7 +807,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
             index=minutes)
         self.writer.write_sid(sids[1], data_2)
 
-        reader = BcolzMinuteBarReader(self.dest)
+        reader = BcolzExchangeBarReader(self.dest)
 
         columns = ['open', 'high', 'low', 'close', 'volume']
         sids = [sids[0], sids[1]]
@@ -854,7 +859,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
             index=minutes)
         self.writer.write_sid(sids[1], data_2)
 
-        reader = BcolzMinuteBarReader(self.dest)
+        reader = BcolzExchangeBarReader(self.dest)
 
         columns = ['open', 'high', 'low', 'close', 'volume']
         sids = [sids[0], sids[1]]
@@ -877,6 +882,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
                 assert_almost_equal(data[sid].loc[minutes, col],
                                     arrays[i][j][minute_locs])
 
+    '''
     def test_adjust_non_trading_minutes(self):
         start_day = Timestamp('2015-06-01', tz='UTC')
         end_day = Timestamp('2015-06-02', tz='UTC')
@@ -922,7 +928,9 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
                 Timestamp('2015-06-02 20:01:00', tz='UTC'),
                 'open'
             )
+    '''
 
+    '''
     def test_adjust_non_trading_minutes_half_days(self):
         # half day
         start_day = Timestamp('2015-11-27', tz='UTC')
@@ -978,6 +986,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
                 Timestamp('2015-11-30 21:01:00', tz='UTC'),
                 'open'
             )
+    '''
 
     def test_set_sid_attrs(self):
         """Confirm that we can set the attributes of a sid's file correctly.
@@ -1023,13 +1032,13 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
 
         # Open a new writer to cover `open` method, also truncating only
         # applies to an existing directory.
-        writer = BcolzMinuteBarWriter.open(self.dest)
+        writer = BcolzExchangeBarWriter.open(self.dest)
 
         # Truncate to first day with data.
         writer.truncate(days[0])
 
         # Refresh the reader since truncate update the metadata.
-        self.reader = BcolzMinuteBarReader(self.dest)
+        self.reader = BcolzExchangeBarReader(self.dest)
 
         self.assertEqual(self.writer.last_date_in_output_for_sid(sid), days[0])
 
@@ -1087,7 +1096,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
         self.writer.truncate(self.test_calendar_start)
 
         # Refresh the reader since truncate update the metadata.
-        self.reader = BcolzMinuteBarReader(self.dest)
+        self.reader = BcolzExchangeBarReader(self.dest)
 
         self.assertEqual(
             self.writer.last_date_in_output_for_sid(sid),
@@ -1198,7 +1207,7 @@ class BcolzMinuteBarTestCase(WithTradingCalendars,
         self.writer.write(update_reader.read(minutes, sids))
 
         # Refresh the reader since truncate update the metadata.
-        reader = BcolzMinuteBarReader(self.dest)
+        reader = BcolzExchangeBarReader(self.dest)
 
         columns = ['open', 'high', 'low', 'close', 'volume']
         sids = [sids[0], sids[1]]

tests/data/test_resample.py

@@ -35,7 +35,7 @@ from catalyst.testing.fixtures import (
     WithBcolzEquityMinuteBarReader,
     WithBcolzEquityDailyBarReader,
     WithBcolzFutureMinuteBarReader,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 OHLC = ['open', 'high', 'low', 'close']
@@ -254,7 +254,7 @@ EXPECTED_SESSIONS = {
 
 class MinuteToDailyAggregationTestCase(WithBcolzEquityMinuteBarReader,
                                        WithBcolzFutureMinuteBarReader,
-                                       ZiplineTestCase):
+                                       CatalystTestCase):
 
     #    March 2016
     # Su Mo Tu We Th Fr Sa
@@ -525,7 +525,7 @@ class MinuteToDailyAggregationTestCase(WithBcolzEquityMinuteBarReader,
 
 
 class TestMinuteToSession(WithEquityMinuteBarData,
-                          ZiplineTestCase):
+                          CatalystTestCase):
 
     #    March 2016
     # Su Mo Tu We Th Fr Sa
@@ -565,7 +565,7 @@ class TestMinuteToSession(WithEquityMinuteBarData,
 
 
 class TestResampleSessionBars(WithBcolzFutureMinuteBarReader,
-                              ZiplineTestCase):
+                              CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('us_futures',)
     TRADING_CALENDAR_PRIMARY_CAL = 'us_futures'
@@ -667,7 +667,7 @@ class TestResampleSessionBars(WithBcolzFutureMinuteBarReader,
 
 
 class TestReindexMinuteBars(WithBcolzEquityMinuteBarReader,
-                            ZiplineTestCase):
+                            CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('us_futures', 'NYSE')
     TRADING_CALENDAR_PRIMARY_CAL = 'us_futures'
@@ -736,7 +736,7 @@ class TestReindexMinuteBars(WithBcolzEquityMinuteBarReader,
 
 
 class TestReindexSessionBars(WithBcolzEquityDailyBarReader,
-                             ZiplineTestCase):
+                             CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('us_futures', 'NYSE')
     TRADING_CALENDAR_PRIMARY_CAL = 'us_futures'

tests/data/test_us_equity_pricing.py

@@ -50,7 +50,7 @@ from catalyst.testing.fixtures import (
     WithBcolzEquityDailyBarReader,
     WithTmpDir,
     WithTradingCalendars,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.calendars import get_calendar
 
@@ -86,7 +86,7 @@ EQUITY_INFO['symbol'] = [chr(ord('A') + n) for n in range(len(EQUITY_INFO))]
 TEST_QUERY_ASSETS = EQUITY_INFO.index
 
 
-class BcolzDailyBarTestCase(WithBcolzEquityDailyBarReader, ZiplineTestCase):
+class BcolzDailyBarTestCase(WithBcolzEquityDailyBarReader, CatalystTestCase):
     EQUITY_DAILY_BAR_START_DATE = TEST_CALENDAR_START
     EQUITY_DAILY_BAR_END_DATE = TEST_CALENDAR_STOP
 
@@ -372,7 +372,7 @@ class BcolzDailyBarNeverReadAllTestCase(BcolzDailyBarTestCase):
 class BcolzDailyBarWriterMissingDataTestCase(WithAssetFinder,
                                              WithTmpDir,
                                              WithTradingCalendars,
-                                             ZiplineTestCase):
+                                             CatalystTestCase):
     # Sid 3 is active from 2015-06-02 to 2015-06-30.
     MISSING_DATA_SID = 3
     # Leave out data for a day in the middle of the query range.

tests/events/test_events_cme.py

@@ -12,6 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+'''
+#   ZIPLINE legacy test: Catalyst only uses OPEN calendar, and thus 
+#   this test suite is irrelevant, and is commented out in its entirety
+
 from unittest import TestCase
 import pandas as pd
 
@@ -41,3 +46,5 @@ class TestStatelessRulesCME(StatelessRulesTests, TestCase):
 
 class TestStatefulRulesCME(StatefulRulesTests, TestCase):
     CALENDAR_STRING = "CME"
+
+'''

tests/events/test_events_nyse.py

@@ -12,6 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+'''
+#   ZIPLINE legacy test: Catalyst only uses OPEN calendar, and thus 
+#   this test suite is irrelevant, and is commented out in its entirety
+
 from unittest import TestCase
 from datetime import timedelta
 import pandas as pd
@@ -162,3 +167,5 @@ class TestStatelessRulesNYSE(StatelessRulesTests, TestCase):
 
 class TestStatefulRulesNYSE(StatefulRulesTests, TestCase):
     CALENDAR_STRING = "NYSE"
+
+'''

tests/exchange/test_bundle.py

@@ -42,17 +42,16 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
         assets = [
-            exchange.get_asset('xmr_btc')
+            exchange.get_asset('eth_btc')
         ]
 
-        # start = pd.to_datetime('2017-09-01', utc=True)
-        start = pd.to_datetime('2016-01-01', utc=True)
-        end = pd.to_datetime('2017-9-30', utc=True)
+        start = pd.to_datetime('2016-03-01', utc=True)
+        end = pd.to_datetime('2017-11-1', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -122,8 +121,8 @@ class TestExchangeBundle:
 
     def test_ingest_daily(self):
         exchange_name = 'bitfinex'
-        data_frequency = 'daily'
-        include_symbols = 'btc_usd'
+        data_frequency = 'minute'
+        include_symbols = 'neo_btc'
 
         # exchange_name = 'poloniex'
         # data_frequency = 'daily'
@@ -422,7 +421,8 @@ class TestExchangeBundle:
                 data_frequency=data_frequency,
                 asset=asset,
                 writer=writer,
-                empty_rows_behavior='raise'
+                empty_rows_behavior='raise',
+                duplicates_behavior='raise'
             )
 
         bundle_series = bundle.get_history_window_series(
@@ -438,26 +438,30 @@ class TestExchangeBundle:
         pass
 
     def main_bundle_to_csv(self):
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neo_usd')
+        asset = exchange.get_asset('eth_btc')
 
+        start_dt = pd.to_datetime('2016-5-31', utc=True)
+        end_dt = pd.to_datetime('2016-6-1', utc=True)
         self._bundle_to_csv(
             asset=asset,
-            exchange=exchange,
+            exchange_name=exchange.name,
             data_frequency=data_frequency,
             filename='{}_{}_{}'.format(
                 exchange_name, data_frequency, asset.symbol
-            )
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
         )
 
     def bundle_to_csv(self):
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
         data_frequency = 'minute'
-        period = '2017-10'
-        symbol = 'neo_btc'
+        period = '2017-01'
+        symbol = 'eth_btc'
 
         exchange = get_exchange(exchange_name)
         asset = exchange.get_asset(symbol)
@@ -470,20 +474,23 @@ class TestExchangeBundle:
         )
         self._bundle_to_csv(
             asset=asset,
-            exchange=exchange,
+            exchange_name=exchange.name,
             data_frequency=data_frequency,
             path=path,
             filename=period
         )
         pass
 
-    def _bundle_to_csv(self, asset, exchange, data_frequency, filename,
-                       path=None):
-        bundle = ExchangeBundle(exchange)
+    def _bundle_to_csv(self, asset, exchange_name, data_frequency, filename,
+                       path=None, start_dt=None, end_dt=None):
+        bundle = ExchangeBundle(exchange_name)
         reader = bundle.get_reader(data_frequency, path=path)
 
-        start_dt = reader.first_trading_day
-        end_dt = reader.last_available_dt
+        if start_dt is None:
+            start_dt = reader.first_trading_day
+
+        if end_dt is None:
+            end_dt = reader.last_available_dt
 
         if data_frequency == 'daily':
             end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
@@ -507,14 +514,39 @@ class TestExchangeBundle:
         df = get_df_from_arrays(arrays, periods)
 
         folder = os.path.join(
-            tempfile.gettempdir(), 'catalyst', exchange.name, asset.symbol
+            tempfile.gettempdir(), 'catalyst', exchange_name, asset.symbol
         )
         ensure_directory(folder)
 
         path = os.path.join(folder, filename + '.csv')
 
         log.info('creating csv file: {}'.format(path))
-        print('HEAD\n{}'.format(df.head(10)))
-        print('TAIL\n{}'.format(df.tail(10)))
+        print('HEAD\n{}'.format(df.head(100)))
+        print('TAIL\n{}'.format(df.tail(100)))
         df.to_csv(path)
         pass
+
+    def test_ingest_csv(self):
+        data_frequency = 'minute'
+        exchange_name = 'bittrex'
+        path = '/Users/fredfortier/Dropbox/Enigma/Data/bittrex_bat_eth.csv'
+
+        exchange_bundle = ExchangeBundle(exchange_name)
+        exchange_bundle.ingest_csv(path, data_frequency)
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset('bat_eth')
+
+        start_dt = pd.to_datetime('2017-6-3', utc=True)
+        end_dt = pd.to_datetime('2017-8-3 19:24', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange_name=exchange.name,
+            data_frequency=data_frequency,
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
+        pass

tests/exchange/test_ccxt.py

@@ -0,0 +1,96 @@
+import os
+import tempfile
+
+import pandas as pd
+from catalyst.exchange.ccxt.ccxt_exchange import CCXT
+from catalyst.finance.order import Order
+from base import BaseExchangeTestCase
+from logbook import Logger
+from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.utils.paths import ensure_directory
+
+log = Logger('test_ccxt')
+
+
+class TestCCXT(BaseExchangeTestCase):
+    @classmethod
+    def setup(self):
+        exchange_name = 'gdax'
+        auth = get_exchange_auth(exchange_name)
+        self.exchange = CCXT(
+            exchange_name=exchange_name,
+            key=auth['key'],
+            secret=auth['secret'],
+            base_currency='eth',
+            portfolio=None
+        )
+
+    def test_order(self):
+        log.info('creating order')
+        asset = self.exchange.get_asset('neo_eth')
+        order_id = self.exchange.order(
+            asset=asset,
+            limit_price=0.07,
+            amount=1,
+        )
+        log.info('order created {}'.format(order_id))
+        assert order_id is not None
+        pass
+
+    def test_open_orders(self):
+        log.info('retrieving open orders')
+        asset = self.exchange.get_asset('neo_eth')
+        orders = self.exchange.get_open_orders(asset)
+        pass
+
+    def test_get_order(self):
+        log.info('retrieving order')
+        order = self.exchange.get_order('2631386', 'neo_eth')
+        # order = self.exchange.get_order('2631386')
+        assert isinstance(order, Order)
+        pass
+
+    def test_cancel_order(self, ):
+        log.info('cancel order')
+        self.exchange.cancel_order('2631386', 'neo_eth')
+        pass
+
+    def test_get_candles(self):
+        log.info('retrieving candles')
+        candles = self.exchange.get_candles(
+            freq='5T',
+            assets=[self.exchange.get_asset('eth_btc')],
+            bar_count=200,
+            start_dt=pd.to_datetime('2017-01-01', utc=True)
+        )
+
+        for asset in candles:
+            df = pd.DataFrame(candles[asset])
+            df.set_index('last_traded', drop=True, inplace=True)
+        pass
+
+    def test_tickers(self):
+        log.info('retrieving tickers')
+        tickers = self.exchange.tickers([
+            self.exchange.get_asset('eth_btc'),
+        ])
+        assert len(tickers) == 1
+        pass
+
+    def test_get_balances(self):
+        log.info('testing wallet balances')
+        balances = self.exchange.get_balances()
+        pass
+
+    def test_get_account(self):
+        log.info('testing account data')
+        pass
+
+    def test_orderbook(self):
+        log.info('testing order book for bittrex')
+        asset = self.exchange.get_asset('eth_btc')
+        orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
+        pass
+
+    def test_get_fees(self):
+        pass

tests/exchange/test_data_portal.py

@@ -1,16 +1,13 @@
 import pandas as pd
-from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
-    DataPortalExchangeLive
 from logbook import Logger
-from test_utils import rnd_history_date_days, rnd_bar_count
 
 from catalyst import get_calendar
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_common_assets
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
+    DataPortalExchangeLive
+from catalyst.exchange.exchange_utils import get_common_assets
 from catalyst.exchange.factory import get_exchange, get_exchanges
+from test_utils import rnd_history_date_days, rnd_bar_count, output_df
 
 log = Logger('test_bitfinex')
 
@@ -113,3 +110,6 @@ class TestExchangeDataPortal:
         )
 
         log.info('found history window: {}'.format(data))
+
+    def test_validate_resample(self):
+        pass

tests/exchange/test_poloniex.py

@@ -1,9 +1,10 @@
-from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.exchange.poloniex.poloniex import Poloniex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+import pandas as pd
+from test_utils import output_df
 
 log = Logger('test_poloniex')
 
@@ -51,18 +52,20 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
+        assets = self.exchange.get_asset('eth_btc')
+        ohlcv = self.exchange.get_candles(
+            # end_dt=pd.to_datetime('2017-11-01', utc=True),
+            end_dt=None,
             freq='5T',
-            assets=self.exchange.get_asset('eth_btc')
-        )
-        ohlcv_neo_ubq = self.exchange.get_candles(
-            freq='5T',
-            assets=[
-                self.exchange.get_asset('neos_btc'),
-                self.exchange.get_asset('via_btc')
-            ],
-            bar_count=14
+            assets=assets,
+            bar_count=200
         )
+        df = pd.DataFrame(ohlcv)
+        df.set_index('last_traded', drop=True, inplace=True)
+        log.info(df.tail(25))
+
+        path = output_df(df, assets, '5min_candles')
+        log.info('saved candles: {}'.format(path))
         pass
 
     def test_tickers(self):

tests/exchange/test_utils.py

@@ -1,17 +1,66 @@
+import os
+import tempfile
 from datetime import timedelta
 from random import randint
 
 import pandas as pd
+from catalyst.assets._assets import TradingPair
+
+from catalyst.utils.paths import ensure_directory
 
 
-def rnd_history_date_days(max_days=30):
-    now = pd.Timestamp.utcnow()
+def rnd_history_date_days(max_days=30, last_dt=None):
+    if last_dt is None:
+        last_dt = pd.Timestamp.utcnow()
+
     days = randint(0, max_days)
 
-    return now - timedelta(days=days)
+    return last_dt - timedelta(days=days)
+
+
+def rnd_history_date_minutes(max_minutes=1440):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_minutes)
+
+    return now - timedelta(minutes=days)
 
 
 def rnd_bar_count(max_bars=21):
     now = pd.Timestamp.utcnow()
 
     return randint(0, max_bars)
+
+
+def output_df(df, assets, name=None):
+    """
+    Outputs a price DataFrame to a temp folder.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+    assets
+    name
+
+    Returns
+    -------
+
+    """
+    if isinstance(assets, TradingPair):
+        exchange_folder = assets.exchange
+        asset_folder = assets.symbol
+    else:
+        exchange_folder = ','.join([asset.exchange for asset in assets])
+        asset_folder = ','.join([asset.symbol for asset in assets])
+
+    folder = os.path.join(
+        tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
+    )
+    ensure_directory(folder)
+
+    if name is None:
+        name = 'output'
+
+    path = os.path.join(folder, '{}.csv'.format(name))
+    df.to_csv(path)
+
+    return path

tests/finance/test_blotter.py

@@ -37,7 +37,7 @@ from catalyst.testing.fixtures import (
     WithDataPortal,
     WithLogger,
     WithSimParams,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.classproperty import classproperty
 
@@ -46,7 +46,7 @@ class BlotterTestCase(WithCreateBarData,
                       WithLogger,
                       WithDataPortal,
                       WithSimParams,
-                      ZiplineTestCase):
+                      CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-05', tz='utc')
     END_DATE = pd.Timestamp('2006-01-06', tz='utc')
     ASSET_FINDER_EQUITY_SIDS = 24, 25

tests/finance/test_commissions.py

@@ -19,7 +19,7 @@ from catalyst.finance.commission import (
 )
 from catalyst.finance.order import Order
 from catalyst.finance.transaction import Transaction
-from catalyst.testing import ZiplineTestCase, trades_by_sid_to_dfs
+from catalyst.testing import CatalystTestCase, trades_by_sid_to_dfs
 from catalyst.testing.fixtures import (
     WithAssetFinder,
     WithSimParams,
@@ -28,7 +28,7 @@ from catalyst.testing.fixtures import (
 from catalyst.utils import factory
 
 
-class CommissionUnitTests(WithAssetFinder, ZiplineTestCase):
+class CommissionUnitTests(WithAssetFinder, CatalystTestCase):
     ASSET_FINDER_EQUITY_SIDS = 1, 2
 
     @classmethod
@@ -272,7 +272,7 @@ class CommissionUnitTests(WithAssetFinder, ZiplineTestCase):
         self.assertAlmostEqual(15.3, model.calculate(order, txns[2]))
 
 
-class CommissionAlgorithmTests(WithDataPortal, WithSimParams, ZiplineTestCase):
+class CommissionAlgorithmTests(WithDataPortal, WithSimParams, CatalystTestCase):
     # make sure order commissions are properly incremented
 
     sidint, = ASSET_FINDER_EQUITY_SIDS = (133,)

tests/finance/test_slippage.py

@@ -48,7 +48,7 @@ from catalyst.testing.fixtures import (
     WithDataPortal,
     WithSimParams,
     WithTradingEnvironment,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.classproperty import classproperty
 
@@ -59,7 +59,7 @@ TestOrder = namedtuple('TestOrder', 'limit direction')
 class SlippageTestCase(WithCreateBarData,
                        WithSimParams,
                        WithDataPortal,
-                       ZiplineTestCase):
+                       CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-05 14:31', tz='utc')
     END_DATE = pd.Timestamp('2006-01-05 14:36', tz='utc')
     SIM_PARAMS_CAPITAL_BASE = 1.0e5
@@ -566,7 +566,7 @@ class SlippageTestCase(WithCreateBarData,
 class VolumeShareSlippageTestCase(WithCreateBarData,
                                   WithSimParams,
                                   WithDataPortal,
-                                  ZiplineTestCase):
+                                  CatalystTestCase):
 
     START_DATE = pd.Timestamp('2006-01-05 14:31', tz='utc')
     END_DATE = pd.Timestamp('2006-01-05 14:36', tz='utc')
@@ -743,7 +743,7 @@ class VolumeShareSlippageTestCase(WithCreateBarData,
 class VolatilityVolumeShareTestCase(WithCreateBarData,
                                     WithSimParams,
                                     WithDataPortal,
-                                    ZiplineTestCase):
+                                    CatalystTestCase):
 
     ASSET_START_DATE = pd.Timestamp('2006-02-10')
 
@@ -890,7 +890,7 @@ class VolatilityVolumeShareTestCase(WithCreateBarData,
         self.assertIsNone(amount)
 
 
-class MarketImpactTestCase(WithCreateBarData, ZiplineTestCase):
+class MarketImpactTestCase(WithCreateBarData, CatalystTestCase):
 
     ASSET_FINDER_EQUITY_SIDS = (1,)
 
@@ -947,7 +947,7 @@ class MarketImpactTestCase(WithCreateBarData, ZiplineTestCase):
 
 class OrdersStopTestCase(WithSimParams,
                          WithTradingEnvironment,
-                         ZiplineTestCase):
+                         CatalystTestCase):
 
     START_DATE = pd.Timestamp('2006-01-05 14:31', tz='utc')
     END_DATE = pd.Timestamp('2006-01-05 14:36', tz='utc')

tests/pipeline/base.py

@@ -18,7 +18,7 @@ from catalyst.testing import (
 from catalyst.testing.fixtures import (
     WithAssetFinder,
     WithTradingSessions,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 from catalyst.utils.functional import dzip_exact
@@ -54,7 +54,7 @@ with_default_shape = with_defaults(shape=lambda self: self.default_shape)
 
 class BasePipelineTestCase(WithTradingSessions,
                            WithAssetFinder,
-                           ZiplineTestCase):
+                           CatalystTestCase):
     START_DATE = Timestamp('2014', tz='UTC')
     END_DATE = Timestamp('2014-12-31', tz='UTC')
     ASSET_FINDER_EQUITY_SIDS = list(range(20))

tests/pipeline/test_blaze.py

@@ -34,7 +34,7 @@ from catalyst.pipeline.loaders.blaze.core import (
     NonPipelineField,
 )
 from catalyst.testing import (
-    ZiplineTestCase,
+    CatalystTestCase,
     parameter_space,
     tmp_asset_finder,
 )
@@ -77,7 +77,7 @@ def _utc_localize_index_level_0(df):
     return df
 
 
-class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase):
+class BlazeToPipelineTestCase(WithAssetFinder, CatalystTestCase):
     START_DATE = pd.Timestamp(0)
     END_DATE = pd.Timestamp('2015')
 
@@ -1927,7 +1927,7 @@ class BlazeToPipelineTestCase(WithAssetFinder, ZiplineTestCase):
         )
 
 
-class MiscTestCase(ZiplineTestCase):
+class MiscTestCase(CatalystTestCase):
     def test_exprdata_repr(self):
         strd = set()
 

tests/pipeline/test_classifier.py

@@ -7,7 +7,7 @@ import pandas as pd
 from catalyst.lib.labelarray import LabelArray
 from catalyst.pipeline import Classifier
 from catalyst.testing import parameter_space
-from catalyst.testing.fixtures import ZiplineTestCase
+from catalyst.testing.fixtures import CatalystTestCase
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.numpy_utils import (
     categorical_dtype,
@@ -585,7 +585,7 @@ class ClassifierTestCase(BasePipelineTestCase):
         self.assertEqual(result, expected)
 
 
-class TestPostProcessAndToWorkSpaceValue(ZiplineTestCase):
+class TestPostProcessAndToWorkSpaceValue(CatalystTestCase):
     def test_reversability_categorical(self):
         class F(Classifier):
             inputs = ()

tests/pipeline/test_downsampling.py

@@ -13,7 +13,7 @@ from catalyst.pipeline import (
 from catalyst.pipeline.data.testing import TestingDataSet
 from catalyst.pipeline.factors.equity import SimpleMovingAverage
 from catalyst.pipeline.filters.smoothing import All
-from catalyst.testing import ZiplineTestCase, parameter_space
+from catalyst.testing import CatalystTestCase, parameter_space
 from catalyst.testing.fixtures import (
     WithTradingSessions,
     WithSeededRandomPipelineEngine,
@@ -44,7 +44,7 @@ class NDaysAgoClassifier(CustomClassifier):
         out[:] = cats[0]
 
 
-class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
+class ComputeExtraRowsTestcase(WithTradingSessions, CatalystTestCase):
 
     DATA_MIN_DAY = pd.Timestamp('2012-06', tz='UTC')
     DATA_MAX_DAY = pd.Timestamp('2015', tz='UTC')
@@ -555,7 +555,7 @@ class ComputeExtraRowsTestcase(WithTradingSessions, ZiplineTestCase):
 
 
 class DownsampledPipelineTestCase(WithSeededRandomPipelineEngine,
-                                  ZiplineTestCase):
+                                  CatalystTestCase):
 
     # Extend into the last few days of 2013 to test year/quarter boundaries.
     START_DATE = pd.Timestamp('2013-12-15', tz='UTC')

tests/pipeline/test_engine.py

@@ -81,7 +81,7 @@ from catalyst.testing.fixtures import (
     WithEquityPricingPipelineEngine,
     WithSeededRandomPipelineEngine,
     WithTradingEnvironment,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.memoize import lazyval
@@ -199,7 +199,7 @@ class WithConstantInputs(WithTradingEnvironment):
         cls.assets = cls.asset_finder.retrieve_all(cls.asset_ids)
 
 
-class ConstantInputTestCase(WithConstantInputs, ZiplineTestCase):
+class ConstantInputTestCase(WithConstantInputs, CatalystTestCase):
     def test_bad_dates(self):
         loader = self.loader
         engine = SimplePipelineEngine(
@@ -816,7 +816,7 @@ class ConstantInputTestCase(WithConstantInputs, ZiplineTestCase):
                                                   Loader2DataSet.col2)})
 
 
-class FrameInputTestCase(WithTradingEnvironment, ZiplineTestCase):
+class FrameInputTestCase(WithTradingEnvironment, CatalystTestCase):
     asset_ids = ASSET_FINDER_EQUITY_SIDS = 1, 2, 3
     start = START_DATE = Timestamp('2015-01-01', tz='utc')
     end = END_DATE = Timestamp('2015-01-31', tz='utc')
@@ -921,7 +921,7 @@ class FrameInputTestCase(WithTradingEnvironment, ZiplineTestCase):
 
 
 class SyntheticBcolzTestCase(WithAdjustmentReader,
-                             ZiplineTestCase):
+                             CatalystTestCase):
     first_asset_start = Timestamp('2015-04-01', tz='UTC')
     START_DATE = Timestamp('2015-01-01', tz='utc')
     END_DATE = Timestamp('2015-08-01', tz='utc')
@@ -1079,7 +1079,7 @@ class SyntheticBcolzTestCase(WithAdjustmentReader,
         assert_frame_equal(expected, result)
 
 
-class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase):
+class ParameterizedFactorTestCase(WithTradingEnvironment, CatalystTestCase):
     sids = ASSET_FINDER_EQUITY_SIDS = Int64Index([1, 2, 3])
     START_DATE = Timestamp('2015-01-31', tz='UTC')
     END_DATE = Timestamp('2015-03-01', tz='UTC')
@@ -1297,7 +1297,7 @@ class ParameterizedFactorTestCase(WithTradingEnvironment, ZiplineTestCase):
 
 
 class StringColumnTestCase(WithSeededRandomPipelineEngine,
-                           ZiplineTestCase):
+                           CatalystTestCase):
 
     def test_string_classifiers_produce_categoricals(self):
         """
@@ -1327,7 +1327,7 @@ class StringColumnTestCase(WithSeededRandomPipelineEngine,
 
 
 class WindowSafetyPropagationTestCase(WithSeededRandomPipelineEngine,
-                                      ZiplineTestCase):
+                                      CatalystTestCase):
 
     SEEDED_RANDOM_PIPELINE_SEED = 5
 
@@ -1378,7 +1378,7 @@ class WindowSafetyPropagationTestCase(WithSeededRandomPipelineEngine,
             assert_equal(expected_result, results[colname])
 
 
-class PopulateInitialWorkspaceTestCase(WithConstantInputs, ZiplineTestCase):
+class PopulateInitialWorkspaceTestCase(WithConstantInputs, CatalystTestCase):
 
     @parameter_space(window_length=[3, 5], pipeline_length=[5, 10])
     def test_populate_initial_workspace(self, window_length, pipeline_length):
@@ -1503,7 +1503,7 @@ class PopulateInitialWorkspaceTestCase(WithConstantInputs, ZiplineTestCase):
 
 
 class ChunkedPipelineTestCase(WithEquityPricingPipelineEngine,
-                              ZiplineTestCase):
+                              CatalystTestCase):
 
     PIPELINE_START_DATE = Timestamp('2006-01-05', tz='UTC')
     END_DATE = Timestamp('2006-12-29', tz='UTC')

tests/pipeline/test_events.py

@@ -24,7 +24,7 @@ from catalyst.pipeline.loaders.utils import (
     normalize_timestamp_to_query_time,
     previous_event_indexer,
 )
-from catalyst.testing import check_arrays, ZiplineTestCase
+from catalyst.testing import check_arrays, CatalystTestCase
 from catalyst.testing.fixtures import (
     WithAssetFinder,
     WithTradingSessions,
@@ -148,7 +148,7 @@ def make_events(add_nulls):
     return pd.concat(event_frames, ignore_index=True)
 
 
-class EventIndexerTestCase(ZiplineTestCase):
+class EventIndexerTestCase(CatalystTestCase):
 
     @classmethod
     def init_class_fixtures(cls):
@@ -269,7 +269,7 @@ class EventIndexerTestCase(ZiplineTestCase):
 
 class EventsLoaderEmptyTestCase(WithAssetFinder,
                                 WithTradingSessions,
-                                ZiplineTestCase):
+                                CatalystTestCase):
     START_DATE = pd.Timestamp('2014-01-01')
     END_DATE = pd.Timestamp('2014-01-30')
 
@@ -351,7 +351,7 @@ class EventsLoaderEmptyTestCase(WithAssetFinder,
 
 class EventsLoaderTestCase(WithAssetFinder,
                            WithTradingSessions,
-                           ZiplineTestCase):
+                           CatalystTestCase):
 
     START_DATE = pd.Timestamp('2014-01-01')
     END_DATE = pd.Timestamp('2014-01-30')
@@ -586,7 +586,7 @@ class BlazeEventsLoaderTestCase(EventsLoaderTestCase):
         )
 
 
-class EventLoaderUtilsTestCase(ZiplineTestCase):
+class EventLoaderUtilsTestCase(CatalystTestCase):
     # These cases test the following:
     # 1. Shuffling timestamps in DST/EST produces the correct normalized
     # timestamps

tests/pipeline/test_factor.py

@@ -40,7 +40,7 @@ from catalyst.testing import (
     parameter_space,
     permute_rows,
 )
-from catalyst.testing.fixtures import ZiplineTestCase
+from catalyst.testing.fixtures import CatalystTestCase
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.numpy_utils import (
     categorical_dtype,
@@ -1246,7 +1246,7 @@ class TestWindowSafety(TestCase):
         )
 
 
-class TestPostProcessAndToWorkSpaceValue(ZiplineTestCase):
+class TestPostProcessAndToWorkSpaceValue(CatalystTestCase):
     @parameter_space(dtype_=(float64_dtype, datetime64ns_dtype))
     def test_reversability(self, dtype_):
         class F(Factor):

tests/pipeline/test_filter.py

@@ -37,7 +37,7 @@ from catalyst.pipeline.filters import (
     StaticAssets,
     StaticSids,
 )
-from catalyst.testing import parameter_space, permute_rows, ZiplineTestCase
+from catalyst.testing import parameter_space, permute_rows, CatalystTestCase
 from catalyst.testing.fixtures import WithSeededRandomPipelineEngine
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.numpy_utils import float64_dtype, int64_dtype
@@ -841,7 +841,7 @@ class SidFactor(CustomFactor):
 
 
 class SpecificAssetsTestCase(WithSeededRandomPipelineEngine,
-                             ZiplineTestCase):
+                             CatalystTestCase):
 
     ASSET_FINDER_EQUITY_SIDS = tuple(range(10))
 
@@ -887,7 +887,7 @@ class SpecificAssetsTestCase(WithSeededRandomPipelineEngine,
         )
 
 
-class TestPostProcessAndToWorkSpaceValue(ZiplineTestCase):
+class TestPostProcessAndToWorkSpaceValue(CatalystTestCase):
     def test_reversability(self):
         class F(Filter):
             inputs = ()

tests/pipeline/test_pipeline_algo.py

@@ -60,7 +60,7 @@ from catalyst.testing.fixtures import (
     WithAdjustmentReader,
     WithBcolzEquityDailyBarReaderFromCSVs,
     WithDataPortal,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.calendars import get_calendar
 
@@ -84,7 +84,7 @@ def rolling_vwap(df, length):
     return Series(out, index=df.index)
 
 
-class ClosesOnly(WithDataPortal, ZiplineTestCase):
+class ClosesOnly(WithDataPortal, CatalystTestCase):
     sids = 1, 2, 3
     START_DATE = pd.Timestamp('2014-01-01', tz='utc')
     END_DATE = pd.Timestamp('2014-02-01', tz='utc')
@@ -355,7 +355,7 @@ class MockDailyBarSpotReader(object):
 
 class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs,
                                 WithAdjustmentReader,
-                                ZiplineTestCase):
+                                CatalystTestCase):
     AAPL = 1
     MSFT = 2
     BRK_A = 3

tests/pipeline/test_quarters_estimates.py

@@ -40,7 +40,7 @@ from catalyst.pipeline.loaders.earnings_estimates import (
 from catalyst.testing.fixtures import (
     WithAdjustmentReader,
     WithTradingSessions,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.testing.predicates import assert_equal, assert_raises_regex
 from catalyst.testing.predicates import assert_frame_equal
@@ -113,7 +113,7 @@ def create_expected_df_for_factor_compute(start_date,
 
 class WithEstimates(WithTradingSessions, WithAdjustmentReader):
     """
-    ZiplineTestCase mixin providing cls.loader and cls.events as class
+    CatalystTestCase mixin providing cls.loader and cls.events as class
     level fixtures.
 
 
@@ -177,7 +177,7 @@ class WithEstimates(WithTradingSessions, WithAdjustmentReader):
 
 class WithOneDayPipeline(WithEstimates):
     """
-    ZiplineTestCase mixin providing cls.events as a class level fixture and
+    CatalystTestCase mixin providing cls.events as a class level fixture and
     defining a test for all inheritors to use.
 
     Attributes
@@ -246,7 +246,7 @@ class WithOneDayPipeline(WithEstimates):
         assert_frame_equal(results, self.expected_out)
 
 
-class PreviousWithOneDayPipeline(WithOneDayPipeline, ZiplineTestCase):
+class PreviousWithOneDayPipeline(WithOneDayPipeline, CatalystTestCase):
     """
     Tests that previous quarter loader correctly breaks if an incorrect
     number of quarters is passed.
@@ -271,7 +271,7 @@ class PreviousWithOneDayPipeline(WithOneDayPipeline, ZiplineTestCase):
         )
 
 
-class NextWithOneDayPipeline(WithOneDayPipeline, ZiplineTestCase):
+class NextWithOneDayPipeline(WithOneDayPipeline, CatalystTestCase):
     """
     Tests that next quarter loader correctly breaks if an incorrect
     number of quarters is passed.
@@ -308,7 +308,7 @@ dummy_df = pd.DataFrame({SID_FIELD_NAME: 0},
 
 class WithWrongLoaderDefinition(WithEstimates):
     """
-    ZiplineTestCase mixin providing cls.events as a class level fixture and
+    CatalystTestCase mixin providing cls.events as a class level fixture and
     defining a test for all inheritors to use.
 
     Attributes
@@ -372,7 +372,7 @@ class WithWrongLoaderDefinition(WithEstimates):
 
 
 class PreviousWithWrongNumQuarters(WithWrongLoaderDefinition,
-                                   ZiplineTestCase):
+                                   CatalystTestCase):
     """
     Tests that previous quarter loader correctly breaks if an incorrect
     number of quarters is passed.
@@ -383,7 +383,7 @@ class PreviousWithWrongNumQuarters(WithWrongLoaderDefinition,
 
 
 class NextWithWrongNumQuarters(WithWrongLoaderDefinition,
-                               ZiplineTestCase):
+                               CatalystTestCase):
     """
     Tests that next quarter loader correctly breaks if an incorrect
     number of quarters is passed.
@@ -398,7 +398,7 @@ options = ["split_adjustments_loader",
            "split_adjusted_asof"]
 
 
-class WrongSplitsLoaderDefinition(WithEstimates, ZiplineTestCase):
+class WrongSplitsLoaderDefinition(WithEstimates, CatalystTestCase):
     """
     Test class that tests that loaders break correctly when incorrectly
     instantiated.
@@ -436,7 +436,7 @@ class WrongSplitsLoaderDefinition(WithEstimates, ZiplineTestCase):
 
 class WithEstimatesTimeZero(WithEstimates):
     """
-    ZiplineTestCase mixin providing cls.events as a class level fixture and
+    CatalystTestCase mixin providing cls.events as a class level fixture and
     defining a test for all inheritors to use.
 
     Attributes
@@ -622,7 +622,7 @@ class WithEstimatesTimeZero(WithEstimates):
                              sid_estimates)
 
 
-class NextEstimate(WithEstimatesTimeZero, ZiplineTestCase):
+class NextEstimate(WithEstimatesTimeZero, CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return NextEarningsEstimatesLoader(events, columns)
@@ -662,7 +662,7 @@ class BlazeNextEstimateLoaderTestCase(NextEstimate):
         )
 
 
-class PreviousEstimate(WithEstimatesTimeZero, ZiplineTestCase):
+class PreviousEstimate(WithEstimatesTimeZero, CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return PreviousEarningsEstimatesLoader(events, columns)
@@ -703,7 +703,7 @@ class BlazePreviousEstimateLoaderTestCase(PreviousEstimate):
 
 class WithEstimateMultipleQuarters(WithEstimates):
     """
-    ZiplineTestCase mixin providing cls.events, cls.make_expected_out as
+    CatalystTestCase mixin providing cls.events, cls.make_expected_out as
     class-level fixtures and self.test_multiple_qtrs_requested as a test.
 
     Attributes
@@ -797,7 +797,7 @@ class WithEstimateMultipleQuarters(WithEstimates):
 
 
 class NextEstimateMultipleQuarters(
-    WithEstimateMultipleQuarters, ZiplineTestCase
+    WithEstimateMultipleQuarters, CatalystTestCase
 ):
     @classmethod
     def make_loader(cls, events, columns):
@@ -854,7 +854,7 @@ class BlazeNextEstimateMultipleQuarters(NextEstimateMultipleQuarters):
 
 class PreviousEstimateMultipleQuarters(
     WithEstimateMultipleQuarters,
-    ZiplineTestCase
+    CatalystTestCase
 ):
 
     @classmethod
@@ -903,7 +903,7 @@ class BlazePreviousEstimateMultipleQuarters(PreviousEstimateMultipleQuarters):
 
 class WithVaryingNumEstimates(WithEstimates):
     """
-    ZiplineTestCase mixin providing fixtures and a test to ensure that we
+    CatalystTestCase mixin providing fixtures and a test to ensure that we
     have the correct overwrites when the event date changes. We want to make
     sure that if we have a quarter with an event date that gets pushed back,
     we don't start overwriting for the next quarter early. Likewise,
@@ -973,7 +973,7 @@ class WithVaryingNumEstimates(WithEstimates):
 
 class PreviousVaryingNumEstimates(
     WithVaryingNumEstimates,
-    ZiplineTestCase
+    CatalystTestCase
 ):
     def assert_compute(self, estimate, today):
         if today == pd.Timestamp('2015-01-13', tz='utc'):
@@ -1003,7 +1003,7 @@ class BlazePreviousVaryingNumEstimates(PreviousVaryingNumEstimates):
 
 class NextVaryingNumEstimates(
     WithVaryingNumEstimates,
-    ZiplineTestCase
+    CatalystTestCase
 ):
 
     def assert_compute(self, estimate, today):
@@ -1034,7 +1034,7 @@ class BlazeNextVaryingNumEstimates(NextVaryingNumEstimates):
 
 class WithEstimateWindows(WithEstimates):
     """
-    ZiplineTestCase mixin providing fixures and a test to test running a
+    CatalystTestCase mixin providing fixures and a test to test running a
     Pipeline with an estimates loader over differently-sized windows.
 
     Attributes
@@ -1198,7 +1198,7 @@ class WithEstimateWindows(WithEstimates):
         )
 
 
-class PreviousEstimateWindows(WithEstimateWindows, ZiplineTestCase):
+class PreviousEstimateWindows(WithEstimateWindows, CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return PreviousEarningsEstimatesLoader(events, columns)
@@ -1279,7 +1279,7 @@ class BlazePreviousEstimateWindows(PreviousEstimateWindows):
         return BlazePreviousEstimatesLoader(bz.data(events), columns)
 
 
-class NextEstimateWindows(WithEstimateWindows, ZiplineTestCase):
+class NextEstimateWindows(WithEstimateWindows, CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return NextEarningsEstimatesLoader(events, columns)
@@ -1394,7 +1394,7 @@ class BlazeNextEstimateWindows(NextEstimateWindows):
 
 class WithSplitAdjustedWindows(WithEstimateWindows):
     """
-    ZiplineTestCase mixin providing fixures and a test to test running a
+    CatalystTestCase mixin providing fixures and a test to test running a
     Pipeline with an estimates loader over differently-sized windows and with
     split adjustments.
     """
@@ -1572,7 +1572,7 @@ class WithSplitAdjustedWindows(WithEstimateWindows):
 
 
 class PreviousWithSplitAdjustedWindows(WithSplitAdjustedWindows,
-                                       ZiplineTestCase):
+                                       CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return PreviousSplitAdjustedEarningsEstimatesLoader(
@@ -1726,7 +1726,7 @@ class BlazePreviousWithSplitAdjustedWindows(PreviousWithSplitAdjustedWindows):
         )
 
 
-class NextWithSplitAdjustedWindows(WithSplitAdjustedWindows, ZiplineTestCase):
+class NextWithSplitAdjustedWindows(WithSplitAdjustedWindows, CatalystTestCase):
 
     @classmethod
     def make_loader(cls, events, columns):
@@ -1951,7 +1951,7 @@ class BlazeNextWithSplitAdjustedWindows(NextWithSplitAdjustedWindows):
 
 class WithSplitAdjustedMultipleEstimateColumns(WithEstimates):
     """
-    ZiplineTestCase mixin for having multiple estimate columns that are
+    CatalystTestCase mixin for having multiple estimate columns that are
     split-adjusted to make sure that adjustments are applied correctly.
 
     Attributes
@@ -2136,7 +2136,7 @@ class WithSplitAdjustedMultipleEstimateColumns(WithEstimates):
 
 
 class PreviousWithSplitAdjustedMultipleEstimateColumns(
-    WithSplitAdjustedMultipleEstimateColumns, ZiplineTestCase
+    WithSplitAdjustedMultipleEstimateColumns, CatalystTestCase
 ):
     @classmethod
     def make_loader(cls, events, columns):
@@ -2218,7 +2218,7 @@ class BlazePreviousWithMultipleEstimateColumns(
 
 
 class NextWithSplitAdjustedMultipleEstimateColumns(
-    WithSplitAdjustedMultipleEstimateColumns, ZiplineTestCase
+    WithSplitAdjustedMultipleEstimateColumns, CatalystTestCase
 ):
     @classmethod
     def make_loader(cls, events, columns):
@@ -2295,7 +2295,7 @@ class BlazeNextWithMultipleEstimateColumns(
 
 class WithAdjustmentBoundaries(WithEstimates):
     """
-    ZiplineTestCase mixin providing class-level attributes, methods,
+    CatalystTestCase mixin providing class-level attributes, methods,
     and a test to make sure that when the split-adjusted-asof-date is not
     strictly within the date index, we can still apply adjustments correctly.
 
@@ -2470,7 +2470,7 @@ class WithAdjustmentBoundaries(WithEstimates):
 
 
 class PreviousWithAdjustmentBoundaries(WithAdjustmentBoundaries,
-                                       ZiplineTestCase):
+                                       CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return partial(PreviousSplitAdjustedEarningsEstimatesLoader,
@@ -2612,7 +2612,7 @@ class BlazePreviousWithAdjustmentBoundaries(PreviousWithAdjustmentBoundaries):
 
 
 class NextWithAdjustmentBoundaries(WithAdjustmentBoundaries,
-                                   ZiplineTestCase):
+                                   CatalystTestCase):
     @classmethod
     def make_loader(cls, events, columns):
         return partial(NextSplitAdjustedEarningsEstimatesLoader,
@@ -2720,7 +2720,7 @@ class BlazeNextWithAdjustmentBoundaries(NextWithAdjustmentBoundaries):
                        split_adjusted_column_names=['estimate'])
 
 
-class QuarterShiftTestCase(ZiplineTestCase):
+class QuarterShiftTestCase(CatalystTestCase):
     """
     This tests, in isolation, quarter calculation logic for shifting quarters
     backwards/forwards from a starting point.

tests/pipeline/test_slice.py

@@ -31,12 +31,12 @@ from catalyst.testing import (
 )
 from catalyst.testing.fixtures import (
     WithSeededRandomPipelineEngine,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.numpy_utils import datetime64ns_dtype
 
 
-class SliceTestCase(WithSeededRandomPipelineEngine, ZiplineTestCase):
+class SliceTestCase(WithSeededRandomPipelineEngine, CatalystTestCase):
     sids = ASSET_FINDER_EQUITY_SIDS = Int64Index([1, 2, 3])
     START_DATE = Timestamp('2015-01-31', tz='UTC')
     END_DATE = Timestamp('2015-03-01', tz='UTC')

tests/pipeline/test_statistical.py

@@ -42,7 +42,7 @@ from catalyst.testing import (
 from catalyst.testing.fixtures import (
     WithSeededRandomPipelineEngine,
     WithTradingEnvironment,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.numpy_utils import (
     bool_dtype,
@@ -51,7 +51,7 @@ from catalyst.utils.numpy_utils import (
 )
 
 
-class StatisticalBuiltInsTestCase(WithTradingEnvironment, ZiplineTestCase):
+class StatisticalBuiltInsTestCase(WithTradingEnvironment, CatalystTestCase):
     sids = ASSET_FINDER_EQUITY_SIDS = Int64Index([1, 2, 3])
     START_DATE = Timestamp('2015-01-31', tz='UTC')
     END_DATE = Timestamp('2015-03-01', tz='UTC')
@@ -388,7 +388,7 @@ class StatisticalBuiltInsTestCase(WithTradingEnvironment, ZiplineTestCase):
 
 
 class StatisticalMethodsTestCase(WithSeededRandomPipelineEngine,
-                                 ZiplineTestCase):
+                                 CatalystTestCase):
     sids = ASSET_FINDER_EQUITY_SIDS = Int64Index([1, 2, 3])
     START_DATE = Timestamp('2015-01-31', tz='UTC')
     END_DATE = Timestamp('2015-03-01', tz='UTC')

tests/pipeline/test_technical.py

@@ -21,7 +21,7 @@ from catalyst.pipeline.factors.equity import (
     AnnualizedVolatility,
 )
 from catalyst.testing import parameter_space
-from catalyst.testing.fixtures import ZiplineTestCase
+from catalyst.testing.fixtures import CatalystTestCase
 from catalyst.testing.predicates import assert_equal
 from .base import BasePipelineTestCase
 
@@ -115,7 +115,7 @@ class BollingerBandsTestCase(BasePipelineTestCase):
         self.assertIs(upper, bbands.upper)
 
 
-class AroonTestCase(ZiplineTestCase):
+class AroonTestCase(CatalystTestCase):
     window_length = 10
     nassets = 5
     dtype = [('down', 'f8'), ('up', 'f8')]
@@ -148,7 +148,7 @@ class AroonTestCase(ZiplineTestCase):
         assert_equal(out, expected_out)
 
 
-class TestFastStochasticOscillator(ZiplineTestCase):
+class TestFastStochasticOscillator(CatalystTestCase):
     """
     Test the Fast Stochastic Oscillator
     """
@@ -218,7 +218,7 @@ class TestFastStochasticOscillator(ZiplineTestCase):
         assert_equal(out, expected_out_k, array_decimal=6)
 
 
-class IchimokuKinkoHyoTestCase(ZiplineTestCase):
+class IchimokuKinkoHyoTestCase(CatalystTestCase):
     def test_ichimoku_kinko_hyo(self):
         window_length = 52
         today = pd.Timestamp('2014', tz='utc')
@@ -334,7 +334,7 @@ class IchimokuKinkoHyoTestCase(ZiplineTestCase):
         )
 
 
-class TestRateOfChangePercentage(ZiplineTestCase):
+class TestRateOfChangePercentage(CatalystTestCase):
     @parameterized.expand([
         ('constant', [2.] * 10, 0.0),
         ('step', [2.] + [1.] * 9, -50.0),
@@ -358,7 +358,7 @@ class TestRateOfChangePercentage(ZiplineTestCase):
         assert_equal(out, np.full((len(assets),), expected))
 
 
-class TestLinearWeightedMovingAverage(ZiplineTestCase):
+class TestLinearWeightedMovingAverage(CatalystTestCase):
     def test_wma1(self):
         wma1 = LinearWeightedMovingAverage(
             inputs=(USEquityPricing.close,),
@@ -390,7 +390,7 @@ class TestLinearWeightedMovingAverage(ZiplineTestCase):
         assert_equal(out, np.array([30.,  31.,  32.,  33.,  34.]))
 
 
-class TestTrueRange(ZiplineTestCase):
+class TestTrueRange(CatalystTestCase):
 
     def test_tr_basic(self):
         tr = TrueRange()
@@ -407,7 +407,7 @@ class TestTrueRange(ZiplineTestCase):
         assert_equal(out, np.full((3,), 2.))
 
 
-class MovingAverageConvergenceDivergenceTestCase(ZiplineTestCase):
+class MovingAverageConvergenceDivergenceTestCase(CatalystTestCase):
 
     def expected_ewma(self, data_df, window):
         # Comment copied from `test_engine.py`:
@@ -532,7 +532,7 @@ class MovingAverageConvergenceDivergenceTestCase(ZiplineTestCase):
         )
 
 
-class AnnualizedVolatilityTestCase(ZiplineTestCase):
+class AnnualizedVolatilityTestCase(CatalystTestCase):
     """
     Test Annualized Volatility
     """

tests/pipeline/test_term.py

@@ -34,7 +34,7 @@ from catalyst.pipeline.factors import RecarrayField
 from catalyst.pipeline.sentinels import NotSpecified
 from catalyst.pipeline.term import AssetExists, Slice
 from catalyst.testing import parameter_space
-from catalyst.testing.fixtures import WithTradingSessions, ZiplineTestCase
+from catalyst.testing.fixtures import WithTradingSessions, CatalystTestCase
 from catalyst.testing.predicates import (
     assert_equal,
     assert_raises,
@@ -155,7 +155,7 @@ def to_dict(l):
     return dict(zip(map(str, range(len(l))), l))
 
 
-class DependencyResolutionTestCase(WithTradingSessions, ZiplineTestCase):
+class DependencyResolutionTestCase(WithTradingSessions, CatalystTestCase):
 
     TRADING_CALENDAR_STRS = ('NYSE',)
     START_DATE = pd.Timestamp('2014-01-02', tz='UTC')

tests/pipeline/test_us_equity_pricing_loader.py

@@ -55,7 +55,7 @@ from catalyst.testing import (
 )
 from catalyst.testing.fixtures import (
     WithAdjustmentReader,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 # Test calendar ranges over the month of June 2015
@@ -258,7 +258,7 @@ DIVIDENDS_EXPECTED = DataFrame(
 
 
 class USEquityPricingLoaderTestCase(WithAdjustmentReader,
-                                    ZiplineTestCase):
+                                    CatalystTestCase):
     START_DATE = TEST_CALENDAR_START
     END_DATE = TEST_CALENDAR_STOP
     asset_ids = 1, 2, 3

tests/risk/test_risk_cumulative.py

@@ -18,7 +18,7 @@ import pandas as pd
 import catalyst.finance.risk as risk
 from catalyst.utils import factory
 
-from catalyst.testing.fixtures import WithTradingEnvironment, ZiplineTestCase
+from catalyst.testing.fixtures import WithTradingEnvironment, CatalystTestCase
 
 from catalyst.finance.trading import SimulationParameters
 
@@ -30,7 +30,7 @@ BENCHMARK = [BENCHMARK_BASE] * 251
 DECIMAL_PLACES = 8
 
 
-class TestRisk(WithTradingEnvironment, ZiplineTestCase):
+class TestRisk(WithTradingEnvironment, CatalystTestCase):
 
     def init_instance_fixtures(self):
         super(TestRisk, self).init_instance_fixtures()

tests/risk/test_risk_period.py

@@ -22,7 +22,7 @@ import catalyst.finance.risk as risk
 from catalyst.utils import factory
 
 from catalyst.finance.trading import SimulationParameters
-from catalyst.testing.fixtures import WithTradingEnvironment, ZiplineTestCase
+from catalyst.testing.fixtures import WithTradingEnvironment, CatalystTestCase
 
 from catalyst.finance.risk.period import RiskMetricsPeriod
 
@@ -34,7 +34,7 @@ BENCHMARK = [BENCHMARK_BASE] * 251
 DECIMAL_PLACES = 8
 
 
-class TestRisk(WithTradingEnvironment, ZiplineTestCase):
+class TestRisk(WithTradingEnvironment, CatalystTestCase):
 
     def init_instance_fixtures(self):
         super(TestRisk, self).init_instance_fixtures()
@@ -232,6 +232,28 @@ class TestRisk(WithTradingEnvironment, ZiplineTestCase):
         # The sortino ratio is calculated by a empyrical function so testing
         # of period sortino ratios will be limited to determine if the value is
         # numerical. This tests for its existence and format.
+
+        # This test needs a different result set that, with some
+        # negative results, otherwise fails in a legitimate way.
+
+        RETURNS = (np.random.rand(251) * 0.1) - 0.05
+
+        self.algo_returns = factory.create_returns_from_list(
+            RETURNS,
+            self.sim_params
+        )
+
+        self.metrics = risk.RiskReport(
+            self.algo_returns,
+            self.sim_params,
+            benchmark_returns=self.benchmark_returns,
+            trading_calendar=self.trading_calendar,
+            treasury_curves=self.env.treasury_curves,
+        )
+
+        for x in self.metrics.month_periods:
+            print (type(x.sortino))
+
         np.testing.assert_equal(
             all(isinstance(x.sortino, float)
                 for x in self.metrics.month_periods),

tests/test_algorithm.py

@@ -109,7 +109,7 @@ from catalyst.testing.fixtures import (
     WithSimParams,
     WithTradingEnvironment,
     WithTmpDir,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.test_algorithms import (
     access_account_in_init,
@@ -190,7 +190,7 @@ import catalyst.utils.factory as factory
 _multiprocess_can_split_ = False
 
 
-class TestRecordAlgorithm(WithSimParams, WithDataPortal, ZiplineTestCase):
+class TestRecordAlgorithm(WithSimParams, WithDataPortal, CatalystTestCase):
     ASSET_FINDER_EQUITY_SIDS = 133,
 
     def test_record_incr(self):
@@ -210,7 +210,7 @@ class TestRecordAlgorithm(WithSimParams, WithDataPortal, ZiplineTestCase):
 class TestMiscellaneousAPI(WithLogger,
                            WithSimParams,
                            WithDataPortal,
-                           ZiplineTestCase):
+                           CatalystTestCase):
 
     START_DATE = pd.Timestamp('2006-01-03', tz='UTC')
     END_DATE = pd.Timestamp('2006-01-04', tz='UTC')
@@ -819,7 +819,7 @@ def log_nyse_close(context, data):
 class TestTransformAlgorithm(WithLogger,
                              WithDataPortal,
                              WithSimParams,
-                             ZiplineTestCase):
+                             CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-01-06', tz='utc')
 
@@ -1092,7 +1092,7 @@ def before_trading_start(context, data):
 class TestPositions(WithLogger,
                     WithDataPortal,
                     WithSimParams,
-                    ZiplineTestCase):
+                    CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-01-06', tz='utc')
     SIM_PARAMS_CAPITAL_BASE = 1000
@@ -1225,7 +1225,7 @@ class TestPositions(WithLogger,
 
 class TestBeforeTradingStart(WithDataPortal,
                              WithSimParams,
-                             ZiplineTestCase):
+                             CatalystTestCase):
     START_DATE = pd.Timestamp('2016-01-06', tz='utc')
     END_DATE = pd.Timestamp('2016-01-07', tz='utc')
     SIM_PARAMS_CAPITAL_BASE = 10000
@@ -1578,7 +1578,7 @@ class TestBeforeTradingStart(WithDataPortal,
 class TestAlgoScript(WithLogger,
                      WithDataPortal,
                      WithSimParams,
-                     ZiplineTestCase):
+                     CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-12-31', tz='utc')
     DATA_PORTAL_USE_MINUTE_DATA = False
@@ -2331,7 +2331,7 @@ def handle_data(context, data):
 class TestCapitalChanges(WithLogger,
                          WithDataPortal,
                          WithSimParams,
-                         ZiplineTestCase):
+                         CatalystTestCase):
 
     sids = 0, 1
 
@@ -2339,16 +2339,16 @@ class TestCapitalChanges(WithLogger,
     def make_equity_info(cls):
         data = make_simple_equity_info(
             cls.sids,
-            pd.Timestamp('2006-01-03', tz='UTC'),
-            pd.Timestamp('2006-01-09', tz='UTC'),
+            pd.Timestamp('2016-01-03', tz='UTC'),
+            pd.Timestamp('2016-01-09', tz='UTC'),
         )
         return data
 
     @classmethod
     def make_equity_minute_bar_data(cls):
         minutes = cls.trading_calendar.minutes_in_range(
-            pd.Timestamp('2006-01-03', tz='UTC'),
-            pd.Timestamp('2006-01-09', tz='UTC')
+            pd.Timestamp('2016-01-03', tz='UTC'),
+            pd.Timestamp('2016-01-09', tz='UTC')
         )
         return trades_by_sid_to_dfs(
             {
@@ -2366,8 +2366,8 @@ class TestCapitalChanges(WithLogger,
     @classmethod
     def make_equity_daily_bar_data(cls):
         days = cls.trading_calendar.sessions_in_range(
-            pd.Timestamp('2006-01-03', tz='UTC'),
-            pd.Timestamp('2006-01-09', tz='UTC')
+            pd.Timestamp('2016-01-03', tz='UTC'),
+            pd.Timestamp('2016-01-09', tz='UTC')
         )
         return trades_by_sid_to_dfs(
             {
@@ -2387,12 +2387,12 @@ class TestCapitalChanges(WithLogger,
     ])
     def test_capital_changes_daily_mode(self, change_type, value):
         sim_params = factory.create_simulation_parameters(
-            start=pd.Timestamp('2006-01-03', tz='UTC'),
-            end=pd.Timestamp('2006-01-09', tz='UTC')
+            start=pd.Timestamp('2016-01-03', tz='UTC'),
+            end=pd.Timestamp('2016-01-09', tz='UTC')
         )
 
         capital_changes = {
-            pd.Timestamp('2006-01-06', tz='UTC'):
+            pd.Timestamp('2016-01-06', tz='UTC'):
                 {'type': change_type, 'value': value}
         }
 
@@ -2429,7 +2429,7 @@ def order_stuff(context, data):
         self.assertEqual(len(capital_change_packets), 1)
         self.assertEqual(
             capital_change_packets[0],
-            {'date': pd.Timestamp('2006-01-06', tz='UTC'),
+            {'date': pd.Timestamp('2016-01-06', tz='UTC'),
              'type': 'cash',
              'target': 153000.0 if change_type == 'target' else None,
              'delta': 50000.0})
@@ -2532,23 +2532,23 @@ def order_stuff(context, data):
 
         self.assertEqual(
             algo.capital_change_deltas,
-            {pd.Timestamp('2006-01-06', tz='UTC'): 50000.0}
+            {pd.Timestamp('2016-01-06', tz='UTC'): 50000.0}
         )
 
     @parameterized.expand([
-        ('interday_target', [('2006-01-04', 2388.0)]),
-        ('interday_delta', [('2006-01-04', 1000.0)]),
-        ('intraday_target', [('2006-01-04 17:00', 2186.0),
-                             ('2006-01-04 18:00', 2806.0)]),
-        ('intraday_delta', [('2006-01-04 17:00', 500.0),
-                            ('2006-01-04 18:00', 500.0)]),
+        ('interday_target', [('2016-01-04', 2388.0)]),
+        ('interday_delta', [('2016-01-04', 1000.0)]),
+        ('intraday_target', [('2016-01-04 17:00', 2186.0),
+                             ('2016-01-04 18:00', 2806.0)]),
+        ('intraday_delta', [('2016-01-04 17:00', 500.0),
+                            ('2016-01-04 18:00', 500.0)]),
     ])
     def test_capital_changes_minute_mode_daily_emission(self, change, values):
         change_loc, change_type = change.split('_')
 
         sim_params = factory.create_simulation_parameters(
-            start=pd.Timestamp('2006-01-03', tz='UTC'),
-            end=pd.Timestamp('2006-01-05', tz='UTC'),
+            start=pd.Timestamp('2016-01-03', tz='UTC'),
+            end=pd.Timestamp('2016-01-05', tz='UTC'),
             data_frequency='minute',
             capital_base=1000.0
         )
@@ -2692,29 +2692,29 @@ def order_stuff(context, data):
         if change_loc == 'interday':
             self.assertEqual(
                 algo.capital_change_deltas,
-                {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0}
+                {pd.Timestamp('2016-01-04', tz='UTC'): 1000.0}
             )
         else:
             self.assertEqual(
                 algo.capital_change_deltas,
-                {pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0,
-                 pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0}
+                {pd.Timestamp('2016-01-04 17:00', tz='UTC'): 500.0,
+                 pd.Timestamp('2016-01-04 18:00', tz='UTC'): 500.0}
             )
 
     @parameterized.expand([
-        ('interday_target', [('2006-01-04', 2388.0)]),
-        ('interday_delta', [('2006-01-04', 1000.0)]),
-        ('intraday_target', [('2006-01-04 17:00', 2186.0),
-                             ('2006-01-04 18:00', 2806.0)]),
-        ('intraday_delta', [('2006-01-04 17:00', 500.0),
-                            ('2006-01-04 18:00', 500.0)]),
+        ('interday_target', [('2016-01-04', 2388.0)]),
+        ('interday_delta', [('2016-01-04', 1000.0)]),
+        ('intraday_target', [('2016-01-04 17:00', 2186.0),
+                             ('2016-01-04 18:00', 2806.0)]),
+        ('intraday_delta', [('2016-01-04 17:00', 500.0),
+                            ('2016-01-04 18:00', 500.0)]),
     ])
     def test_capital_changes_minute_mode_minute_emission(self, change, values):
         change_loc, change_type = change.split('_')
 
         sim_params = factory.create_simulation_parameters(
-            start=pd.Timestamp('2006-01-03', tz='UTC'),
-            end=pd.Timestamp('2006-01-05', tz='UTC'),
+            start=pd.Timestamp('2016-01-03', tz='UTC'),
+            end=pd.Timestamp('2016-01-05', tz='UTC'),
             data_frequency='minute',
             emission_rate='minute',
             capital_base=1000.0
@@ -2933,20 +2933,20 @@ def order_stuff(context, data):
         if change_loc == 'interday':
             self.assertEqual(
                 algo.capital_change_deltas,
-                {pd.Timestamp('2006-01-04', tz='UTC'): 1000.0}
+                {pd.Timestamp('2016-01-04', tz='UTC'): 1000.0}
             )
         else:
             self.assertEqual(
                 algo.capital_change_deltas,
-                {pd.Timestamp('2006-01-04 17:00', tz='UTC'): 500.0,
-                 pd.Timestamp('2006-01-04 18:00', tz='UTC'): 500.0}
+                {pd.Timestamp('2016-01-04 17:00', tz='UTC'): 500.0,
+                 pd.Timestamp('2016-01-04 18:00', tz='UTC'): 500.0}
             )
 
 
 class TestGetDatetime(WithLogger,
                       WithSimParams,
                       WithDataPortal,
-                      ZiplineTestCase):
+                      CatalystTestCase):
     SIM_PARAMS_DATA_FREQUENCY = 'minute'
     START_DATE = to_utc('2014-01-02 9:31')
     END_DATE = to_utc('2014-01-03 9:31')
@@ -2994,7 +2994,7 @@ class TestGetDatetime(WithLogger,
         self.assertFalse(algo.first_bar)
 
 
-class TestTradingControls(WithSimParams, WithDataPortal, ZiplineTestCase):
+class TestTradingControls(WithSimParams, WithDataPortal, CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-01-06', tz='utc')
 
@@ -3468,7 +3468,7 @@ class TestTradingControls(WithSimParams, WithDataPortal, ZiplineTestCase):
                 algo.run(data_portal)
 
 
-class TestAccountControls(WithDataPortal, WithSimParams, ZiplineTestCase):
+class TestAccountControls(WithDataPortal, WithSimParams, CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-01-06', tz='utc')
 
@@ -3616,7 +3616,7 @@ class TestAccountControls(WithDataPortal, WithSimParams, ZiplineTestCase):
 #                 format(i, actual_position, expected_positions[i]))
 
 
-class TestFutureFlip(WithDataPortal, WithSimParams, ZiplineTestCase):
+class TestFutureFlip(WithDataPortal, WithSimParams, CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-09', tz='utc')
     END_DATE = pd.Timestamp('2006-01-10', tz='utc')
     sid, = ASSET_FINDER_EQUITY_SIDS = (1,)
@@ -3677,7 +3677,7 @@ class TestFutureFlip(WithDataPortal, WithSimParams, ZiplineTestCase):
                 format(i, actual_position, expected_positions[i]))
 
 
-class TestFuturesAlgo(WithDataPortal, WithSimParams, ZiplineTestCase):
+class TestFuturesAlgo(WithDataPortal, WithSimParams, CatalystTestCase):
     START_DATE = pd.Timestamp('2016-01-06', tz='utc')
     END_DATE = pd.Timestamp('2016-01-07', tz='utc')
     FUTURE_MINUTE_BAR_START_DATE = pd.Timestamp('2016-01-05', tz='UTC')
@@ -3879,7 +3879,7 @@ class TestFuturesAlgo(WithDataPortal, WithSimParams, ZiplineTestCase):
             self.assertEqual(txn['price'], expected_price)
 
 
-class TestTradingAlgorithm(WithTradingEnvironment, ZiplineTestCase):
+class TestTradingAlgorithm(WithTradingEnvironment, CatalystTestCase):
     def test_analyze_called(self):
         self.perf_ref = None
 
@@ -3907,7 +3907,7 @@ class TestTradingAlgorithm(WithTradingEnvironment, ZiplineTestCase):
 
 class TestOrderCancelation(WithDataPortal,
                            WithSimParams,
-                           ZiplineTestCase):
+                           CatalystTestCase):
 
     START_DATE = pd.Timestamp('2016-01-05', tz='utc')
     END_DATE = pd.Timestamp('2016-01-07', tz='utc')
@@ -4100,7 +4100,7 @@ class TestOrderCancelation(WithDataPortal,
             self.assertFalse(log_catcher.has_warnings)
 
 
-class TestEquityAutoClose(WithTradingEnvironment, WithTmpDir, ZiplineTestCase):
+class TestEquityAutoClose(WithTradingEnvironment, WithTmpDir, CatalystTestCase):
     """
     Tests if delisted equities are properly removed from a portfolio holding
     positions in said equities.
@@ -4661,7 +4661,7 @@ class TestEquityAutoClose(WithTradingEnvironment, WithTmpDir, ZiplineTestCase):
         )
 
 
-class TestOrderAfterDelist(WithTradingEnvironment, ZiplineTestCase):
+class TestOrderAfterDelist(WithTradingEnvironment, CatalystTestCase):
     start = pd.Timestamp('2016-01-05', tz='utc')
     day_1 = pd.Timestamp('2016-01-06', tz='utc')
     day_4 = pd.Timestamp('2016-01-11', tz='utc')
@@ -4756,7 +4756,7 @@ class TestOrderAfterDelist(WithTradingEnvironment, ZiplineTestCase):
                 self.assertEqual(expected_message, w.message)
 
 
-class AlgoInputValidationTestCase(WithTradingEnvironment, ZiplineTestCase):
+class AlgoInputValidationTestCase(WithTradingEnvironment, CatalystTestCase):
 
     def test_reject_passing_both_api_methods_and_script(self):
         script = dedent(
@@ -4787,7 +4787,7 @@ class AlgoInputValidationTestCase(WithTradingEnvironment, ZiplineTestCase):
                 )
 
 
-class TestPanelData(WithTradingEnvironment, ZiplineTestCase):
+class TestPanelData(WithTradingEnvironment, CatalystTestCase):
 
     @parameterized.expand([
         ('daily',

tests/test_api_shim.py

@@ -17,7 +17,7 @@ from catalyst.testing.fixtures import (
     WithCreateBarData,
     WithDataPortal,
     WithSimParams,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
@@ -133,7 +133,7 @@ def handle_data(context, data):
 class TestAPIShim(WithCreateBarData,
                   WithDataPortal,
                   WithSimParams,
-                  ZiplineTestCase,
+                  CatalystTestCase,
                   ):
     START_DATE = pd.Timestamp("2016-01-05", tz='UTC')
     END_DATE = pd.Timestamp("2016-01-28", tz='UTC')

tests/test_assets.py

@@ -80,7 +80,7 @@ from catalyst.testing import (
 from catalyst.testing.predicates import assert_equal
 from catalyst.testing.fixtures import (
     WithAssetFinder,
-    ZiplineTestCase,
+    CatalystTestCase,
     WithTradingCalendars,
 )
 from catalyst.utils.range import range
@@ -345,7 +345,7 @@ class AssetTestCase(TestCase):
                 'a' < self.asset3
 
 
-class TestFuture(WithAssetFinder, ZiplineTestCase):
+class TestFuture(WithAssetFinder, CatalystTestCase):
     @classmethod
     def make_futures_info(cls):
         return pd.DataFrame.from_dict(
@@ -458,7 +458,7 @@ class TestFuture(WithAssetFinder, ZiplineTestCase):
             TestFuture.asset_finder.lookup_future_symbol('XXX99')
 
 
-class AssetFinderTestCase(WithTradingCalendars, ZiplineTestCase):
+class AssetFinderTestCase(WithTradingCalendars, CatalystTestCase):
     asset_finder_type = AssetFinder
 
     def write_assets(self, **kwargs):
@@ -1395,7 +1395,7 @@ class AssetFinderTestCase(WithTradingCalendars, ZiplineTestCase):
             )
 
 
-class TestAssetDBVersioning(ZiplineTestCase):
+class TestAssetDBVersioning(CatalystTestCase):
 
     def init_instance_fixtures(self):
         super(TestAssetDBVersioning, self).init_instance_fixtures()
@@ -1533,7 +1533,7 @@ class TestAssetDBVersioning(ZiplineTestCase):
         assert_equal(expected_data, actual_data)
 
 
-class TestVectorizedSymbolLookup(WithAssetFinder, ZiplineTestCase):
+class TestVectorizedSymbolLookup(WithAssetFinder, CatalystTestCase):
 
     @classmethod
     def make_equity_info(cls):

tests/test_bar_data.py

@@ -38,7 +38,7 @@ from catalyst.testing import (
 from catalyst.testing.fixtures import (
     WithCreateBarData,
     WithDataPortal,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.calendars.trading_calendar import days_at_time
@@ -108,7 +108,7 @@ class WithBarDataChecks(object):
 class TestMinuteBarData(WithCreateBarData,
                         WithBarDataChecks,
                         WithDataPortal,
-                        ZiplineTestCase):
+                        CatalystTestCase):
     START_DATE = pd.Timestamp('2016-01-05', tz='UTC')
     END_DATE = ASSET_FINDER_EQUITY_END_DATE = pd.Timestamp(
         '2016-01-07',
@@ -730,7 +730,7 @@ class TestMinuteBarData(WithCreateBarData,
 
 class TestMinuteBarDataFuturesCalendar(WithCreateBarData,
                                        WithBarDataChecks,
-                                       ZiplineTestCase):
+                                       CatalystTestCase):
 
     START_DATE = pd.Timestamp('2016-01-05', tz='UTC')
     END_DATE = ASSET_FINDER_EQUITY_END_DATE = pd.Timestamp(
@@ -857,7 +857,7 @@ class TestMinuteBarDataFuturesCalendar(WithCreateBarData,
 class TestDailyBarData(WithCreateBarData,
                        WithBarDataChecks,
                        WithDataPortal,
-                       ZiplineTestCase):
+                       CatalystTestCase):
     START_DATE = pd.Timestamp('2016-01-05', tz='UTC')
     END_DATE = ASSET_FINDER_EQUITY_END_DATE = pd.Timestamp(
         '2016-01-11',

tests/test_benchmark.py

@@ -32,12 +32,12 @@ from catalyst.testing.fixtures import (
     WithDataPortal,
     WithSimParams,
     WithTradingCalendars,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 
 class TestBenchmark(WithDataPortal, WithSimParams, WithTradingCalendars,
-                    ZiplineTestCase):
+                    CatalystTestCase):
     START_DATE = pd.Timestamp('2006-01-03', tz='utc')
     END_DATE = pd.Timestamp('2006-12-29', tz='utc')
 

tests/test_continuous_futures.py

@@ -41,7 +41,7 @@ from catalyst.testing.fixtures import (
     WithDataPortal,
     WithBcolzFutureMinuteBarReader,
     WithSimParams,
-    ZiplineTestCase,
+    CatalystTestCase,
 )
 
 
@@ -49,7 +49,7 @@ class ContinuousFuturesTestCase(WithCreateBarData,
                                 WithDataPortal,
                                 WithSimParams,
                                 WithBcolzFutureMinuteBarReader,
-                                ZiplineTestCase):
+                                CatalystTestCase):
 
     START_DATE = pd.Timestamp('2015-01-05', tz='UTC')
     END_DATE = pd.Timestamp('2016-10-19', tz='UTC')
@@ -1285,7 +1285,7 @@ def record_current_contract(algo, data):
 
 
 class OrderedContractsTestCase(WithAssetFinder,
-                               ZiplineTestCase):
+                               CatalystTestCase):
 
     @classmethod
     def make_root_symbols_info(self):

tests/test_data_portal.py

@@ -27,7 +27,7 @@ from catalyst.data.minute_bars import (
 )
 from catalyst.testing import parameter_space
 from catalyst.testing.fixtures import (
-    ZiplineTestCase,
+    CatalystTestCase,
     WithTradingSessions,
     WithDataPortal,
     alias,
@@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype
 
 class DataPortalTestBase(WithDataPortal,
                          WithTradingSessions,
-                         ZiplineTestCase):
+                         CatalystTestCase):
 
     ASSET_FINDER_EQUITY_SIDS = (1, 2)
     START_DATE = pd.Timestamp('2016-08-01')

tests/test_examples.py

@@ -22,7 +22,7 @@ import pandas as pd
 from catalyst import examples
 from catalyst.data.bundles import register, unregister
 from catalyst.testing import test_resource_path
-from catalyst.testing.fixtures import WithTmpDir, ZiplineTestCase
+from catalyst.testing.fixtures import WithTmpDir, CatalystTestCase
 from catalyst.testing.predicates import assert_equal
 from catalyst.utils.cache import dataframe_cache
 from catalyst.utils.paths import update_modified_time
@@ -34,7 +34,7 @@ _multiprocess_can_split_ = False
 matplotlib.use('Agg')
 
 
-class ExamplesTests(WithTmpDir, ZiplineTestCase):
+class ExamplesTests(WithTmpDir, CatalystTestCase):
     # some columns contain values with unique ids that will not be the same
 
     @classmethod