merging from the develop branch

[MIG] Migrated buy_and_hodl and buy_low_sell_high to version 0.3 to work with Poloniex exchange

README.rst

@@ -1 +1,3 @@
-All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_ can be found in the `catalyst-docs wiki <https://github.com/enigmampc/catalyst-docs/wiki>`_.
+All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_
+can be found in the 
+`documentation website <https://enigmampc.github.io/catalyst>`_.

catalyst/__main__.py

@@ -9,7 +9,8 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.exchange_utils import delete_algo_folder
+from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 
@@ -38,7 +39,7 @@ except NameError:
     '--default-extension/--no-default-extension',
     is_flag=True,
     default=True,
-    help="Don't load the default catalyst extension.py file in $ZIPLINE_HOME.",
+    help="Don't load the default catalyst extension.py file in $CATALYST_HOME.",
 )
 @click.version_option()
 def main(extension, strict_extensions, default_extension):
@@ -490,25 +491,90 @@ def live(ctx,
     default=True,
     help='Print progress information to the terminal.'
 )
+@click.option(
+    '--verbose/--no-verbose`',
+    default=False,
+    help='Show a progress indicator for every currency pair.'
+)
+@click.option(
+    '--validate/--no-validate`',
+    default=False,
+    help='Report potential anomalies found in data bundles.'
+)
 def ingest_exchange(exchange_name, data_frequency, start, end,
-                    include_symbols, exclude_symbols, show_progress):
+                    include_symbols, exclude_symbols, show_progress, verbose,
+                    validate):
     """
     Ingest data for the given exchange.
     """
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
     exchange = get_exchange(exchange_name)
     exchange_bundle = ExchangeBundle(exchange)
 
-    click.echo('ingesting exchange bundle {}'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,
         exclude_symbols=exclude_symbols,
         start=start,
         end=end,
-        show_progress=show_progress
+        show_progress=show_progress,
+        show_breakdown=verbose,
+        show_report=validate
     )
 
 
+@main.command(name='clean-algo')
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='The label of the algorithm to for which to clean the state.'
+)
+@click.pass_context
+def clean_algo(ctx, algo_namespace):
+    click.echo(
+        'Deleting the state folder of algo: {}...'.format(algo_namespace)
+    )
+    delete_algo_folder(algo_namespace)
+
+
+@main.command(name='clean-exchange')
+@click.option(
+    '-x',
+    '--exchange-name',
+    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the exchange bundle to ingest (supported: bitfinex,'
+         ' bittrex, poloniex).',
+)
+@click.option(
+    '-f',
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute'}),
+    default=None,
+    help='The bundle data frequency to remove. If not specified, it will '
+         'remove both daily and minute bundles.',
+)
+@click.pass_context
+def clean_exchange(ctx, exchange_name, data_frequency):
+    """Clean up bundles from 'ingest-exchange'.
+    """
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    exchange = get_exchange(exchange_name)
+    exchange_bundle = ExchangeBundle(exchange)
+
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
+    exchange_bundle.clean(
+        data_frequency=data_frequency,
+    )
+    click.echo('Done')
+
+
 @main.command()
 @click.option(
     '-b',
@@ -598,7 +664,7 @@ def ingest(ctx, bundle, exchange_name, compile_locally, assets_version,
          ' This may not be passed with -e / --before or -a / --after',
 )
 def clean(bundle, before, after, keep_last):
-    """Clean up data downloaded with the ingest command.
+    """Clean up bundles from 'ingest'.
     """
     bundles_module.clean(
         bundle,

catalyst/algorithm.py

@@ -138,8 +138,9 @@ from catalyst.gens.sim_engine import MinuteSimulationClock
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
+from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger("ZiplineLog")
+log = logbook.Logger("CatalystLog", level=LOG_LEVEL)
 
 
 class TradingAlgorithm(object):

catalyst/assets/_assets.pyx

@@ -17,6 +17,8 @@
 """
 Cythonized Asset object.
 """
+import hashlib
+
 cimport cython
 from cpython.number cimport PyNumber_Index
 from cpython.object cimport (
@@ -501,7 +503,11 @@ cdef class TradingPair(Asset):
 
         if sid == 0 or sid is None:
             try:
-                sid = abs(hash(symbol)) % (10 ** 4)
+                # sid = abs(hash(symbol)) % (10 ** 4)
+                # TODO: try to encode the symbol in the main scope
+                sid = int(
+                    hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+                ) % 10 ** 6
             except Exception as e:
                 raise SidHashError(symbol=symbol)
 
@@ -553,6 +559,20 @@ cdef class TradingPair(Asset):
             end_minute=self.end_minute
         )
 
+    def is_exchange_open(self, dt_minute):
+        """
+        Parameters
+        ----------
+        dt_minute: pd.Timestamp (UTC, tz-aware)
+            The minute to check.
+
+        Returns
+        -------
+        boolean: whether the asset's exchange is open at the given minute.
+        """
+        #TODO: consider implementing to spot holds
+        return True
+
     cpdef __reduce__(self):
         """
         Function used by pickle to determine how to serialize/deserialize this

catalyst/assets/assets.py

@@ -76,7 +76,9 @@ from catalyst.utils.numpy_utils import as_column
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.sqlite_utils import group_into_chunks, coerce_string_to_eng
 
-log = Logger('assets.py')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('assets.py', level=LOG_LEVEL)
 
 # A set of fields that need to be converted to strings before building an
 # Asset to avoid unicode fields

catalyst/constants.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+import logbook
+
+LOG_LEVEL = logbook.INFO
+
+DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+
+AUTO_INGEST = False

catalyst/curate/poloniex.py

@@ -6,9 +6,8 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 
 DT_START        = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END          = int(time.time())
-CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
-CSV_OUT_FOLDER  = '/Volumes/enigma/data/poloniex/'
+DT_END          = pd.to_datetime('today').value // 10 ** 9
+CSV_OUT_FOLDER  = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
 CONN_RETRIES    = 2
 
 logbook.StderrHandler().push_application()
@@ -27,13 +26,15 @@ class PoloniexCurator(object):
             try:
                 os.makedirs(CSV_OUT_FOLDER)
             except Exception as e:
-                log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
+                log.error('Failed to create data folder: {}'.format(
+                            CSV_OUT_FOLDER))
                 log.exception(e)
 
-    '''
-        Retrieves and returns all currency pairs from the exchange 
-    '''
+
     def get_currency_pairs(self):
+        '''
+        Retrieves and returns all currency pairs from the exchange 
+        '''
         url = self._api_path + 'command=returnTicker'
 
         try:
@@ -49,89 +50,136 @@ class PoloniexCurator(object):
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
 
-        log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
+        log.debug('Currency pairs retrieved successfully: {}'.format(
+                    len(self.currency_pairs)
+                    ))
+
 
 
-    '''
-        Helper function that reads tradeID and date fields from CSV readline
-    '''
     def _retrieve_tradeID_date(self, row):
+        '''
+        Helper function that reads tradeID and date fields from CSV readline
+        '''
         tId = int(row.split(',')[0])
-        d   = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9
+        d   = pd.to_datetime(row.split(',')[1], 
+                             infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
-    '''
-        Retrieves TradeHistory from exchange for a given currencyPair between start and end dates.
-        If no start date is provided, uses a system-wide one (beginning of time for cryptotrading)
-        If no end date is provided, 'now' is used
+
+    def retrieve_trade_history(self, currencyPair, start=DT_START, 
+                               end=DT_END, temp=None):
+        '''
+        Retrieves TradeHistory from exchange for a given currencyPair 
+        between start and end dates. If no start date is provided, uses 
+        a system-wide one (beginning of time for cryptotrading).
+        If no end date is provided, 'now' is used.
+
         Stores results in CSV file on disk.
-        This function is called recursively to work around the limitations imposed by the provider API.
-    '''
-    def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None):
+        
+        This function is called recursively to work around the 
+        limitations imposed by the provider API.
+        '''
         csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
 
         '''
-            Check what data we already have on disk, reading first and last lines from file.
-            Data is stored on file from NEWEST to OLDEST.
+        Check what data we already have on disk, reading first and last 
+        lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
             with open(csv_fn, 'ab+') as f: 
                 f.seek(0, os.SEEK_END)
-                if(f.tell() > 2):                           # First check file is not zero size
-                    f.seek(0)                               # Go to the beginning to read first line
-                    last_tradeID, end_file    = self._retrieve_tradeID_date(f.readline())
-                    f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                    while f.read(1) != b"\n":               # Until EOL is found...
-                        f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
+                if(f.tell() > 2):                   # Check file size is not 0
+                    f.seek(0)                       # Go to start to read 
+                    last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
+                    f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
+                    while f.read(1) != b"\n":       # Until EOL is found...
+                        f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
                     first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
 
-                    if( first_tradeID == 1 and end_file + 3600 > DT_END ):
+                    if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
+                        or (currencyPair == 'BTC_HUC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_RIC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_XCP' and first_tradeID ==     2)
+                        or (currencyPair == 'BTC_NAV' and first_tradeID ==  4569)
+                        or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
                         return
 
         except Exception as e:
-            log.error('Error opening file: %s' % csv_fn)
+            log.error('Error opening file: {}'.format(csv_fn))
             log.exception(e)
 
         '''
-        Poloniex API limits querying TradeHistory to intervals smaller than 1 month,
-        so we make sure that start date is never more than 1 month apart from end date
+        Poloniex API limits querying TradeHistory to intervals smaller 
+        than 1 month, so we make sure that start date is never more than
+        1 month apart from end date
         '''
-        if( end - start > 2419200 ):       # 60 s/min * 60 min/hr * 24 hr/day * 28 days     
+        if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days     
             newstart = end - 2419200
         else:
             newstart = start
 
-        log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t '
-                    + time.ctime(newstart) + ' - '+ time.ctime(end))
+        log.debug('{}: Retrieving from {} to {}\t {} - {}'.format(
+                    currencyPair, str(newstart), str(end),
+                    time.ctime(newstart), time.ctime(end)))
 
-        url = self._api_path + 'command=returnTradeHistory&currencyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end)
+        url = '{path}command=returnTradeHistory&currencyPair={pair}' \
+              '&start={start}&end={end}'.format(
+                    path = self._api_path,
+                    pair = currencyPair,
+                    start = str(newstart),
+                    end = str(end)
+                )
+        print url
 
-        try:
-            response = requests.get(url)
-        except Exception as e:
-            log.error('Failed to retrieve trade history data for %s' % currencyPair)
-            log.exception(e)
+        attempts = 0
+        success = 0
+        while attempts < CONN_RETRIES:
+            try:
+                response = requests.get(url)
+            except Exception as e:
+                log.error('Failed to retrieve trade history data for {}'.format(
+                            currencyPair
+                            ))
+                log.exception(e)
+                attempts += 1
+            else:
+                try:
+                    if isinstance(response.json(), dict) and response.json()['error']:
+                        log.error('Failed to to retrieve trade history data '
+                                  'for {}: {}'.format(
+                                    currencyPair,
+                                    response.json()['error']
+                                    ))
+                        attempts += 1
+                except Exception as e:
+                    log.exception(e)
+                    attempts += 1
+                else:
+                    success = 1
+                    break
+
+        if not success:
             return None
-        else:
-            if isinstance(response.json(), dict) and response.json()['error']:
-                log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error']))
-                exit(1)
+
 
         '''
-            If we get to transactionId == 1, and we already have that on disk, 
-            we got to the end of TradeHistory for this coin.
+        If we get to transactionId == 1, and we already have that on 
+        disk, we got to the end of TradeHistory for this coin.
         '''
-        if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID): 
+        if('first_tradeID' in locals() 
+                and response.json()[-1]['tradeID'] == first_tradeID): 
             return
 
         '''
             There are primarily two scenarios:
-                a) There is newer data available that we need to add at the beginning
-                   of the file. We'll retrieve all what we need until we get to what 
-                   we already have, writing it to a temporary file; and we will write
-                   that at the beginning of our existing file.
-                b) We are going back in time, appending at the end of our existing
-                   TradeHistory until the first transaction for this currencyPair
+                a) There is newer data available that we need to add at 
+                   the beginning of the file. We'll retrieve all what we 
+                   need until we get to what we already have, writing it 
+                   to a temporary file; and we will write that at the 
+                   beginning of our existing file.
+                b) We are going back in time, appending at the end of 
+                   our existing TradeHistory until the first transaction 
+                   for this currencyPair
         '''
         try: 
             if( 'end_file' in locals() and end_file + 3600 < end):
@@ -151,8 +199,10 @@ class PoloniexCurator(object):
                         item['globalTradeID']        
                     ])
                 if( response.json()[-1]['tradeID'] > last_tradeID ):
-                    end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
-                    self.retrieve_trade_history(currencyPair, start, end, temp=temp) 
+                    end = pd.to_datetime( response.json()[-1]['date'], 
+                                infer_datetime_format=True).value // 10 ** 9
+                    self.retrieve_trade_history(currencyPair, start, 
+                                                end, temp=temp) 
                 else:
                     with open(csv_fn,'rb+') as f:
                         shutil.copyfileobj(f,temp)
@@ -165,7 +215,8 @@ class PoloniexCurator(object):
                 with open(csv_fn, 'ab') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in response.json():
-                        if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ):
+                        if( 'first_tradeID' in locals() 
+                                and item['tradeID'] >= first_tradeID ):
                             continue
                         csvwriter.writerow([
                             item['tradeID'],
@@ -176,52 +227,71 @@ class PoloniexCurator(object):
                             item['total'],
                             item['globalTradeID']
                         ])
-                end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
+                end = pd.to_datetime(response.json()[-1]['date'], 
+                            infer_datetime_format=True).value // 10 ** 9
 
         except Exception as e:
-            log.error('Error opening %s' % csv_fn)
+            log.error('Error opening {}'.format(csv_fn))
             log.exception(e)
 
         '''
-            If we got here, we aren't done yet. Call recursively with 'end' times
-            that go sequentially back in time.
+            If we got here, we aren't done yet. Call recursively with 
+            'end' times that go sequentially back in time.
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
 
-    '''
+
+    def generate_ohlcv(self, df):
+        '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
         by resampling with 1-minute period
-    '''
-    def generate_ohlcv(self, df):
-        df.set_index('date', inplace=True)                      # Index by date
-        vol = df['total'].to_frame('volume')                    # Will deal with vol separately, as ohlc() messes it up
-        df.drop('total', axis=1, inplace=True)                  # Drop volume data from dataframe
-        ohlc = df.resample('T').ohlc()                          # Resample OHLC in 1min bins
-        ohlc.columns = ohlc.columns.map(lambda t: t[1])         # Raname columns by dropping 'rate'
-        closes = ohlc['close'].fillna(method='pad')             # Pad forward missing 'close'
-        ohlc = ohlc.apply(lambda x: x.fillna(closes))           # Fill N/A with last close
-        vol = vol.resample('T').sum().fillna(0)                 # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)                   # Concatenate OHLC + Volume
+        '''
+        df.set_index('date', inplace=True)               # Index by date
+        vol = df['total'].to_frame('volume')             # set Vol aside
+        df.drop('total', axis=1, inplace=True)           # Drop volume data 
+        ohlc = df.resample('T').ohlc()                   # Resample OHLC 1min
+        ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Raname columns by dropping 'rate'
+        closes = ohlc['close'].fillna(method='pad')      # Pad fwd missing 'close'
+        ohlc = ohlc.apply(lambda x: x.fillna(closes))    # Fill N/A with last close
+        vol = vol.resample('T').sum().fillna(0)          # Add volumes by bin
+        ohlcv = pd.concat([ohlc,vol], axis=1)            # Concatenate OHLC + Vol
         return ohlcv
 
 
-    '''
+
+    def write_ohlcv_file(self, currencyPair): 
+        '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
-    '''
-    def write_ohlcv_file(self, currencyPair):        
+        '''       
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
         csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if( os.path.isfile(csv_1min) ):
-            log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
+        if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
+            log.debug(currencyPair+': 1min data file already up to date. '
+                      'Delete the file if you want to rebuild it.')
         else:
-            df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
-                    dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
-            df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
+            df = pd.read_csv(csv_trades, 
+                                names=['tradeID',
+                                       'date',
+                                       'type',
+                                       'rate',
+                                       'amount',
+                                       'total',
+                                       'globalTradeID'], 
+                                dtype = {'tradeID': int, 
+                                         'date': str, 
+                                         'type': str, 
+                                         'rate': float, 
+                                         'amount': float, 
+                                         'total': float, 
+                                         'globalTradeID': int } 
+                                )
+            df.drop(['tradeID','type','amount','globalTradeID'], 
+                    axis=1, inplace=True)
             df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
             ohlcv = self.generate_ohlcv(df)
             try: 
-                with open(csv_1min, 'ab') as csvfile:
+                with open(csv_1min, 'w') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in ohlcv.itertuples():
                         if item.Index == 0:
@@ -235,25 +305,34 @@ class PoloniexCurator(object):
                             item.volume,
                         ])
             except Exception as e:
-                log.error('Error opening %s' % csv_fn)
+                log.error('Error opening {}'.format(csv_fn))
                 log.exception(e)
-            log.debug(currencyPair+': Generated 1min OHLCV data.')
+            log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
+
 
 
-    '''
-    Returns a data frame for a given currencyPair from data on disk
-    '''
     def onemin_to_dataframe(self, currencyPair, start, end):
+        '''
+        Returns a data frame for a given currencyPair from data on disk
+        '''
         csv_fn     = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df         = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
+        df         = pd.read_csv(csv_fn, names=['date', 
+                                                'open', 
+                                                'high', 
+                                                'low', 
+                                                'close', 
+                                                'volume']
+                                )
         df['date'] = pd.to_datetime(df['date'],unit='s')
         df.set_index('date', inplace=True)
         return df[start : end]
 
-    '''
-    Generates a symbols.json file with corresponding start_date for each currencyPair
-    '''
+
     def generate_symbols_json(self, filename=None):
+        '''
+        Generates a symbols.json file with corresponding start_date 
+        for each currencyPair
+        '''
         symbol_map = {}
 
         if(filename is None):
@@ -262,14 +341,16 @@ class PoloniexCurator(object):
         with open(filename, 'w') as symbols:
             for currencyPair in self.currency_pairs:
                 start = None
-                csv_fn     = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
+                csv_fn     = '{}crypto_trades-{}.csv'.format(
+                                CSV_OUT_FOLDER, currencyPair)
                 with open(csv_fn, 'r') as f: 
                     f.seek(0, os.SEEK_END)
-                    if(f.tell() > 2):                           # First check file is not zero size
-                        f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
-                        while f.read(1) != b"\n":               # Until EOL is found...
-                            f.seek(-2, os.SEEK_CUR)             # ...jump back the read byte plus one more.
-                        start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True)
+                    if(f.tell() > 2):               # Check file size is not 0
+                        f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
+                        while f.read(1) != b"\n":   # Until EOL is found...
+                            f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
+                        start = pd.to_datetime( f.readline().split(',')[1], 
+                                                infer_datetime_format=True)
 
                 if(start is None):
                     start = time.gmtime()
@@ -279,7 +360,8 @@ class PoloniexCurator(object):
                     symbol = symbol,
                     start_date = start.strftime("%Y-%m-%d")
                 )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':'))    
+            json.dump(symbol_map, symbols, sort_keys=True, indent=2, 
+                        separators=(',',':'))    
 
 
 if __name__ == '__main__':
@@ -289,6 +371,6 @@ if __name__ == '__main__':
     
     for currencyPair in pc.currency_pairs:
         pc.retrieve_trade_history(currencyPair)
+        log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
 
-    

catalyst/data/_equities.pyx

@@ -215,7 +215,7 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in ['open', 'high', 'low', 'close']:
+        if column_name in ['open', 'high', 'low', 'close', 'volume']:
             where_nan = (outbuf == 0)
             outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN

catalyst/data/bundles/base.py

@@ -30,8 +30,10 @@ from catalyst.utils.cli import (
 )
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
+
 logbook.StderrHandler().push_application()
-log = logbook.Logger(__name__)
+log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 

catalyst/data/bundles/quandl.py

@@ -40,7 +40,9 @@ from catalyst.utils.cli import maybe_show_progress
 
 from . import core as bundles
 
-log = Logger(__name__)
+from catalyst.constants import LOG_LEVEL
+
+log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
 class QuandlBundle(BaseEquityPricingBundle):

catalyst/data/data_portal.py

@@ -68,7 +68,9 @@ from catalyst.errors import (
     HistoryWindowStartsBeforeData,
 )
 
-log = Logger('DataPortal')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('DataPortal', level=LOG_LEVEL)
 
 BASE_FIELDS = frozenset([
     "open",

catalyst/data/loader.py

@@ -32,7 +32,9 @@ from ..utils.paths import (
     data_root,
 )
 
-logger = logbook.Logger('Loader')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
 INDEX_MAPPING = {
@@ -95,7 +97,8 @@ def has_data_for_dates(series_or_df, first_date, last_date):
 
 def load_crypto_market_data(trading_day=None, trading_days=None,
                             bm_symbol=None, bundle=None, bundle_data=None,
-                            environ=None, exchange=None):
+                            environ=None, exchange=None, start_dt=None,
+                            end_dt=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
 
@@ -104,8 +107,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
-    first_date = get_calendar('OPEN').first_trading_session
-    now = pd.Timestamp.utcnow()
+    # if start_dt is None:
+    start_dt = get_calendar('OPEN').first_trading_session
+
+    if end_dt is None:
+        end_dt = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
     # **two** full trading days prior to the most recently completed trading
@@ -131,7 +137,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 1]
+    last_date = trading_days[trading_days.get_loc(end_dt, method='ffill') - 1]
 
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
@@ -146,14 +152,14 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     br = exchange.get_history_window(
         assets=[benchmark_asset],
         end_dt=last_date,
-        bar_count=pd.Timedelta(last_date - first_date).days,
+        bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
         data_frequency='daily')
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
-    br.loc[first_date]=0
-    br=br.sort_index()
+    br.loc[start_dt] = 0
+    br = br.sort_index()
 
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
@@ -162,10 +168,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         bm_symbol,
         first_date_treasury,
         last_date,
-        now,
+        end_dt,
         environ,
     )
-    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
+    benchmark_returns = br[br.index.slice_indexer(start_dt, last_date)]
     treasury_curves = tc[
         tc.index.slice_indexer(first_date_treasury, last_date)]
     return benchmark_returns, treasury_curves

catalyst/data/minute_bars.py

@@ -44,8 +44,9 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
 
-logger = logbook.Logger('MinuteBars')
+logger = logbook.Logger('MinuteBars', level=LOG_LEVEL)
 
 US_EQUITIES_MINUTES_PER_DAY = 390
 FUTURES_MINUTES_PER_DAY = 1440
@@ -1125,7 +1126,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             else:
                 return np.nan
 
-        #if field != 'volume':
+        # if field != 'volume':
         value *= self._ohlc_ratio_inverse_for_sid(sid)
         return value
 
@@ -1206,7 +1207,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             minute_dt.value / NANOS_IN_MINUTE,
             self._minutes_per_day,
             False,
-            )
+        )
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
         """
@@ -1262,10 +1263,10 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 where = values != 0
                 # first slice down to len(where) because we might not have
                 # written data for all the minutes requested
-                #if field != 'volume':
+                # if field != 'volume':
                 out[:len(where), i][where] = (
                     values[where] * self._ohlc_ratio_inverse_for_sid(sid))
-                #else:
+                # else:
                 #    out[:len(where), i][where] = values[where]
 
             results.append(out)
@@ -1353,9 +1354,10 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
     path : str
         The path of the HDF5 file from which to source data.
     """
+
     def __init__(self, path):
         self._panel = pd.read_hdf(path)
 
     def read(self, dts, sids):
         panel = self._panel[sids, dts, :]
-        return panel.iteritems()
+        return panel.iteritems()

catalyst/data/us_equity_pricing.py

@@ -83,7 +83,9 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-logger = logbook.Logger('UsEquityPricing')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('UsEquityPricing', level=LOG_LEVEL)
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
 OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])

catalyst/examples/buy_and_hodl.py

@@ -24,7 +24,7 @@ from catalyst.api import (
 )
 
 def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
+    context.ASSET_NAME = 'BTC_USDT'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
@@ -49,14 +49,14 @@ def handle_data(context, data):
     orders = get_open_orders(context.asset) or []
     for order in orders:
         cancel_order(order)
-    
+
     # Stop buying after passing the reserve threshold
     cash = context.portfolio.cash
     if cash <= reserve_value:
         context.is_buying = False
 
     # Retrieve current asset price from pricing data
-    price = data[context.asset].price
+    price = data.current(context.asset, 'price')
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
@@ -70,7 +70,7 @@ def handle_data(context, data):
 
     record(
         price=price,
-        volume=data[context.asset].volume,
+        volume=data.current(context.asset, 'volume'),
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,

catalyst/examples/buy_btc_simple.py

@@ -0,0 +1,29 @@
+'''
+	This is a very simple example referenced in the beginner's tutorial:
+	https://enigmampc.github.io/catalyst/beginner-tutorial.html
+
+	Run this example, by executing the following from your terminal:
+	catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+	If you want to run this code using another exchange, make sure that 
+	the asset is available on that exchange. For example, if you were to run 
+	it for exchange Poloniex, you would need to edit the following line:
+
+		context.asset = symbol('btc_usdt')     # note 'usdt' instead of 'usd'
+
+	and specify exchange poloniex as follows:
+
+	catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
+
+	To see which assets are available on each exchange, visit:
+	https://www.enigma.co/catalyst/status
+'''
+
+from catalyst.api import order, record, symbol
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc = data.current(context.asset, 'price'))

catalyst/examples/buy_low_sell_high.py

@@ -27,7 +27,7 @@ log = Logger(algo_namespace)
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USD'
+    context.ASSET_NAME = 'XRP_USDT'
     context.asset = symbol(context.ASSET_NAME)
 
     context.TARGET_POSITIONS = 5000

catalyst/examples/buy_low_sell_high_live.py

@@ -1,6 +1,7 @@
 import talib
 from logbook import Logger
 
+import pandas as pd
 from catalyst.api import (
     order,
     order_target_percent,
@@ -17,10 +18,10 @@ log = Logger('buy low sell high')
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_BTC'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 300
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
     context.SLIPPAGE_ALLOWED = 0.02
 
@@ -33,31 +34,31 @@ def initialize(context):
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1d'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
-    # elif rsi <= 70:
-    #    buy_increment = 5
+        buy_increment = 0.5
+    elif rsi <= 70:
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -146,11 +147,22 @@ def analyze(context, stats):
 
 
 run_algorithm(
+    capital_base=100000,
     initialize=initialize,
     handle_data=handle_data,
     analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc'
+    exchange_name='poloniex',
+    start=pd.to_datetime('2017-5-01', utc=True),
+    end=pd.to_datetime('2017-10-16', utc=True),
+    base_currency='usdt',
+    data_frequency='daily'
 )
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc'
+# )

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -1,175 +0,0 @@
-import talib
-from logbook import Logger
-import pandas as pd
-
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_low_sell_high_neo'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.asset = symbol('neo_btc', 'bitfinex')
-
-    context.TARGET_POSITIONS = 50000
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'close')
-    log.info('got price {price}'.format(price=price))
-
-    if price is None:
-        log.warn('no pricing data')
-        return
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=1,
-        frequency='1m'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.1
-    else:
-        buy_increment = None
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(price=price)
-
-    orders = get_open_orders(context.asset)
-    if len(orders) > 0:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        limit_price = price * (1 + context.SLIPPAGE_ALLOWED)
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=limit_price
-        )
-        pass
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-
-    pass
-
-
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='bitfinex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc',
-#     live_graph=False
-# )
-
-# Backtest
-run_algorithm(
-    capital_base=250,
-    start=pd.to_datetime('2017-10-01', utc=True),
-    end=pd.to_datetime('2017-10-15', utc=True),
-    data_frequency='minute',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    algo_namespace=algo_namespace,
-    base_currency='btc'
-)

catalyst/examples/mean_reversion.py

@@ -0,0 +1,283 @@
+# For this example, we're going to write a simple momentum script.  When the 
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+from datetime import timedelta
+
+import pandas as pd
+import talib
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+from logbook import Logger
+from talib.common import MA_Type
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, \
+    get_open_orders
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.exchange.stats_utils import extract_transactions, trend_direction
+
+algo_namespace = 'momentum'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Ether in USD Tether.
+    context.eth_btc = symbol('etc_usdt')
+    context.base_price = None
+    context.current_day = None
+    context.trigger = None
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.eth_btc variable.  For this example, we're 
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.eth_btc,
+        fields='close',
+        bar_count=50,
+        frequency='15T'
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+    upper, middle, lower = talib.BBANDS(
+        prices.values,
+        timeperiod=20,
+        nbdevup=2,
+        nbdevdn=2,
+        matype=MA_Type.EMA
+    )
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.eth_btc, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        volume=current['volume'],
+        upper_band=upper[-1],
+        lower_band=lower[-1],
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.eth_btc)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.eth_btc):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.   
+    pos_amount = context.portfolio.positions[context.eth_btc].amount
+
+    # In this example, we're using a trigger instead of buying directly after
+    # a signal. Since this is mean reversion, our signals go against the
+    # momentum. Using a trigger allow us to spot the opportunity but trade
+    # only when a trade reversal begins.
+    if context.trigger is not None:
+        # The tread_direction() method determines the trend based on the last
+        # two bars of the series.
+        direction = trend_direction(rsi)
+        if context.trigger[1] == 'buy' and direction == 'up':
+            log.info(
+                '{}: buying - price: {}, rsi: {}, bband: {}'.format(
+                    data.current_dt, price, rsi[-1], lower[-1]
+                )
+            )
+            order_target_percent(context.eth_btc, 1)
+            context.traded_today = True
+            context.trigger = None
+
+        elif context.trigger[1] == 'sell' and direction == 'down':
+            log.info(
+                '{}: selling - price: {}, rsi: {}, bband: {}'.format(
+                    data.current_dt, price, rsi[-1], upper[-1]
+                )
+            )
+            order_target_percent(context.eth_btc, 0)
+            context.traded_today = True
+            context.trigger = None
+
+        # If we found a signal but no trade reversal within two hours, we
+        # reset the trigger.
+        elif context.trigger[0] + timedelta(hours=2) < data.current_dt:
+            context.trigger = None
+
+    else:
+        # Determining the entry and exit signals based on RSI and SMA
+        if rsi[-1] <= 30 and pos_amount == 0:
+            context.trigger = (data.current_dt, 'buy')
+
+        elif rsi[-1] >= 80 and pos_amount > 0:
+            context.trigger = (data.current_dt, 'sell')
+
+
+def analyze(context=None, perf=None):
+    import matplotlib.pyplot as plt
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+    perf.loc[:, 'upper_band'].plot(ax=ax2, label='Upper')
+    perf.loc[:, 'lower_band'].plot(ax=ax2, label='Lower')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.eth_btc.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.axhline(70, color='darkgoldenrod')
+    ax6.axhline(30, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'backtest'
+
+    if MODE == 'backtest':
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            start=pd.to_datetime('2017-7-1', utc=True),
+            # end=pd.to_datetime('2017-9-30', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )
+
+    elif MODE == 'live':
+        run_algorithm(
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            live_graph=True
+        )

catalyst/examples/mean_reversion_simple.py

@@ -0,0 +1,248 @@
+# For this example, we're going to write a simple momentum script.  When the 
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+from datetime import timedelta
+
+import pandas as pd
+import talib
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+from logbook import Logger
+from talib.common import MA_Type
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, \
+    get_open_orders
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.exchange.stats_utils import extract_transactions, trend_direction
+
+algo_namespace = 'momentum'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Ether in USD Tether.
+    context.eth_btc = symbol('etc_usdt')
+    context.base_price = None
+    context.current_day = None
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.eth_btc variable.  For this example, we're 
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.eth_btc,
+        fields='close',
+        bar_count=50,
+        frequency='15T'
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.eth_btc, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        volume=current['volume'],
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.eth_btc)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.eth_btc):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.   
+    pos_amount = context.portfolio.positions[context.eth_btc].amount
+
+    if rsi[-1] <= 30 and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        order_target_percent(context.eth_btc, 1)
+        context.traded_today = True
+
+    elif rsi[-1] >= 80 and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        order_target_percent(context.eth_btc, 0)
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    import matplotlib.pyplot as plt
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.eth_btc.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.axhline(70, color='darkgoldenrod')
+    ax6.axhline(30, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'backtest'
+
+    if MODE == 'backtest':
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py -x poloniex -s 2017-7-1 -e 2017-7-31 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            start=pd.to_datetime('2017-7-1', utc=True),
+            end=pd.to_datetime('2017-7-31', utc=True),
+        )
+
+    elif MODE == 'live':
+        run_algorithm(
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            live_graph=True
+        )

catalyst/examples/rsi_profit_target.py

@@ -0,0 +1,276 @@
+from datetime import timedelta
+
+import pandas as pd
+import numpy as np
+import talib
+from logbook import Logger
+
+from catalyst.api import (
+    order,
+    symbol,
+    record,
+    get_open_orders,
+)
+from catalyst.exchange.stats_utils import crossover, crossunder
+from catalyst.utils.run_algo import run_algorithm
+
+algo_namespace = 'rsi'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('initializing algo')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
+
+    context.MAX_HOLDINGS = 0.2
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERSOLD_BBANDS = 45
+    context.RSI_OVERBOUGHT_BBANDS = 55
+    context.SLIPPAGE_ALLOWED = 0.03
+
+    context.TARGET = 0.15
+    context.STOP_LOSS = 0.1
+    context.STOP = 0.03
+    context.position = None
+
+    context.last_bar = None
+
+    context.errors = []
+    pass
+
+
+def _handle_buy_sell_decision(context, data, signal, price):
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        log.info('skipping bar until all open orders execute')
+        return
+
+    positions = context.portfolio.positions
+    if context.position is None and context.asset in positions:
+        position = positions[context.asset]
+        context.position = dict(
+            cost_basis=position['cost_basis'],
+            amount=position['amount'],
+            stop=None
+        )
+
+    action = None
+    if context.position is not None:
+        cost_basis = context.position['cost_basis']
+        amount = context.position['amount']
+        log.info(
+            'found {amount} positions with cost basis {cost_basis}'.format(
+                amount=amount,
+                cost_basis=cost_basis
+            )
+        )
+        stop = context.position['stop']
+
+        target = cost_basis * (1 + context.TARGET)
+        if price >= target:
+            context.position['cost_basis'] = price
+            context.position['stop'] = context.STOP
+
+        stop_target = context.STOP_LOSS if stop is None else context.STOP
+        if price < cost_basis * (1 - stop_target):
+            log.info('executing stop loss')
+            order(
+                asset=context.asset,
+                amount=-amount,
+                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            action = 0
+            context.position = None
+
+    else:
+        if signal == 'long':
+            log.info('opening position')
+            buy_amount = context.MAX_HOLDINGS / price
+            order(
+                asset=context.asset,
+                amount=buy_amount,
+                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
+            )
+            context.position = dict(
+                cost_basis=price,
+                amount=buy_amount,
+                stop=None
+            )
+            action = 0
+
+
+def _handle_data_rsi_only(context, data):
+    price = data.current(context.asset, 'close')
+    log.info('got price {price}'.format(price=price))
+
+    if price is np.nan:
+        log.warn('no pricing data')
+        return
+
+    if context.base_price is None:
+        context.base_price = price
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=17,
+            frequency='30T'
+        )
+    except Exception as e:
+        log.warn('historical data not available: '.format(e))
+        return
+
+    rsi = talib.RSI(prices.values, timeperiod=16)[-1]
+    log.info('got rsi {}'.format(rsi))
+
+    signal = None
+    if rsi < context.RSI_OVERSOLD:
+        signal = 'long'
+
+    # Making sure that the price is still current
+    price = data.current(context.asset, 'close')
+    cash = context.portfolio.cash
+    log.info(
+        'base currency available: {cash}, cap: {cap}'.format(
+            cash=cash,
+            cap=context.MAX_HOLDINGS
+        )
+    )
+    volume = data.current(context.asset, 'volume')
+    price_change = (price - context.base_price) / context.base_price
+    record(
+        price=price,
+        price_change=price_change,
+        rsi=rsi,
+        volume=volume,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+    )
+
+    _handle_buy_sell_decision(context, data, signal, price)
+
+
+def handle_data(context, data):
+    dt = data.current_dt
+
+    if context.last_bar is None or (
+                context.last_bar + timedelta(minutes=15)) <= dt:
+        context.last_bar = dt
+    else:
+        return
+
+    log.info('BAR {}'.format(dt))
+    try:
+        _handle_data_rsi_only(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    results.loc[:, 'price'].plot(ax=ax2)
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    trans = results.loc[[t != [] for t in results.transactions], :]
+    buys = trans.loc[[t[0]['amount'] > 0 for t in trans.transactions], :]
+    sells = trans.loc[[t[0]['amount'] < 0 for t in trans.transactions], :]
+    # buys = results.loc[results['action'] == 1, :]
+    # sells = results.loc[results['action'] == 0, :]
+
+    ax2.plot(
+        buys.index,
+        results.loc[buys.index, 'price'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        results.loc[sells.index, 'price'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results.loc[:, ['alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Alpha / Beta ')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results.loc[:, ['starting_cash', 'cash']].plot(ax=ax4)
+    ax4.set_ylabel('Base Currency ({})'.format(base_currency))
+
+    results['algorithm'] = results.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results.loc[:, 'rsi'].plot(ax=ax6)
+    ax6.set_ylabel('RSI')
+
+    ax6.plot(
+        buys.index,
+        results.loc[buys.index, 'rsi'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax6.plot(
+        sells.index,
+        results.loc[sells.index, 'rsi'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+run_algorithm(
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=analyze,
+    exchange_name='bittrex',
+    live=True,
+    algo_namespace=algo_namespace,
+    base_currency='btc',
+    live_graph=False
+)
+
+# Backtest
+# run_algorithm(
+#     capital_base=0.5,
+#     data_frequency='minute',
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     algo_namespace=algo_namespace,
+#     base_currency='btc',
+#     start=pd.to_datetime('2017-9-1', utc=True),
+#     end=pd.to_datetime('2017-10-1', utc=True),
+# )

catalyst/examples/simple_loop.py

@@ -1,5 +1,5 @@
-import pandas as pd
 import talib
+import pandas as pd
 
 from catalyst import run_algorithm
 from catalyst.api import symbol
@@ -7,7 +7,7 @@ from catalyst.api import symbol
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('xrp_btc')
+    context.asset = symbol('swift_btc')
 
 
 def handle_data(context, data):
@@ -16,36 +16,37 @@ def handle_data(context, data):
     price = data.current(context.asset, 'close')
     print('got price {price}'.format(price=price))
 
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=15,
-        frequency='1d'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    print('got rsi: {}'.format(rsi))
-    pass
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=15,
+            frequency='1D'
+        )
+        rsi = talib.RSI(prices.values, timeperiod=14)[-1]
+        print('got rsi: {}'.format(rsi))
+    except Exception as e:
+        print(e)
 
 
+run_algorithm(
+    capital_base=250,
+    start=pd.to_datetime('2015-4-1', utc=True),
+    end=pd.to_datetime('2017-11-1', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='bittrex',
+    algo_namespace='simple_loop',
+    base_currency='btc'
+)
 # run_algorithm(
-#     capital_base=250,
-#     start=pd.to_datetime('2015-08-01', utc=True),
-#     end=pd.to_datetime('2017-9-30', utc=True),
-#     data_frequency='daily',
 #     initialize=initialize,
 #     handle_data=handle_data,
 #     analyze=None,
 #     exchange_name='poloniex',
+#     live=True,
 #     algo_namespace='simple_loop',
-#     base_currency='eth'
-# )
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=None,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace='simple_loop',
-    base_currency='eth',
-    live_graph=False
-)
+#     base_currency='eth',
+#     live_graph=False

catalyst/exchange/asset_finder_exchange.py

@@ -1,6 +1,8 @@
 from logbook import Logger
 
-log = Logger('AssetFinderExchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('AssetFinderExchange', level=LOG_LEVEL)
 
 
 class AssetFinderExchange(object):
@@ -41,9 +43,9 @@ class AssetFinderExchange(object):
         """
         for sid in sids:
             if sid in self._asset_cache:
-                log.info('got asset from cache: {}'.format(sid))
+                log.debug('got asset from cache: {}'.format(sid))
             else:
-                log.info('fetching asset: {}'.format(sid))
+                log.debug('fetching asset: {}'.format(sid))
         return list()
 
     def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,10 +1,10 @@
 import base64
+import datetime
 import hashlib
 import hmac
 import json
 import re
 import time
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import (
     InvalidOrderStyle, OrderCancelError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.protocol import Account
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -33,7 +33,9 @@ requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-log = Logger('Bitfinex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 
@@ -56,7 +58,7 @@ class Bitfinex(Exchange):
 
         # Max is 90 but playing it safe
         # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -238,7 +240,7 @@ class Bitfinex(Exchange):
         # TODO: fetch account data and keep in cache
         return None
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count=None,
                     start_dt=None, end_dt=None):
         """
         Retrieve OHLVC candles from Bitfinex
@@ -253,33 +255,40 @@ class Bitfinex(Exchange):
         '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
          '1M'
         """
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
 
-        freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
+        allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
+                               '360T', '720T', '1D', '7D', '14D', '30D']
+        if freq not in allowed_frequencies:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+        freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
         if freq_match:
             number = int(freq_match.group(1))
             unit = freq_match.group(2)
 
-            if unit == 'd':
-                converted_unit = 'D'
+            if unit == 'T':
+                if number in [60, 180, 360, 720]:
+                    number = number / 60
+                    converted_unit = 'h'
+                else:
+                    converted_unit = 'm'
             else:
                 converted_unit = unit
 
             frequency = '{}{}'.format(number, converted_unit)
-            allowed_frequencies = ['1m', '5m', '15m', '30m', '1h', '3h', '6h',
-                                   '12h', '1D', '7D', '14D', '1M']
 
-            if frequency not in allowed_frequencies:
-                raise InvalidHistoryFrequencyError(
-                    frequency=data_frequency
-                )
-        elif data_frequency == 'minute':
-            frequency = '1m'
-        elif data_frequency == 'daily':
-            frequency = '1D'
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
@@ -665,10 +674,11 @@ class Bitfinex(Exchange):
         return time.strftime('%Y-%m-%d',
                              time.gmtime(int(response.json()[-1][0] / 1000)))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         exchange_symbol = asset.exchange_symbol
         try:
             self.ask_request()
+            # TODO: implement limit
             response = self._request(
                 'book/{}'.format(exchange_symbol), None)
             data = response.json()

catalyst/exchange/bittrex/bittrex.py

@@ -1,29 +1,33 @@
 import json
+import time
 
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
 
-log = Logger('Bittrex')
+# TODO: consider using this: https://github.com/mondeja/bittrex_v2
+
+log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 
 
 class Bittrex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Bittrex_api(key=key, secret=secret)
         self.name = 'bittrex'
         self.color = 'blue'
         self.base_currency = base_currency
@@ -64,10 +68,10 @@ class Bittrex(Exchange):
         return exchange_symbol.lower()
 
     def get_balances(self):
+        balances = self.api.getbalances()
         try:
             log.debug('retrieving wallet balances')
             self.ask_request()
-            balances = self.api.getbalances()
 
         except Exception as e:
             raise ExchangeRequestError(error=e)
@@ -206,44 +210,59 @@ class Bittrex(Exchange):
                 error=status['message']
             )
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
-                    start_date=None):
+    def get_candles(self, freq, assets, bar_count=None,
+                    start_dt=None, end_dt=None):
         """
         Supported Intervals
         -------------------
         day, oneMin, fiveMin, thirtyMin, hour
 
-        :param data_frequency:
+        :param freq:
         :param assets:
         :param bar_count:
+        :param start_dt
+        :param end_dt
         :return:
         """
-        log.info('retrieving candles')
 
-        if data_frequency == 'minute' or data_frequency == '1m':
+        # TODO: this has no effect at the moment
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
+
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        if freq == '1T':
             frequency = 'oneMin'
-        elif data_frequency == '5m':
+        elif freq == '5T':
             frequency = 'fiveMin'
-        elif data_frequency == '30m':
+        elif freq == '30T':
             frequency = 'thirtyMin'
-        elif data_frequency == '1h':
+        elif freq == '60T':
             frequency = 'hour'
-        elif data_frequency == 'daily' or data_frequency == '1D':
+        elif freq == '1D':
             frequency = 'day'
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
         for asset in asset_list:
+            end = int(time.mktime(end_dt.timetuple()))
             url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_=1499127220008'.format(
+                  '&tickInterval={frequency}&_={end}'.format(
                 url=URL2,
                 symbol=self.get_symbol(asset),
-                frequency=frequency
+                frequency=frequency,
+                end=end
             )
 
             try:
@@ -271,9 +290,11 @@ class Bittrex(Exchange):
                 return ohlc
 
             ordered_candles = list(reversed(candles))
+            ohlc_map = dict()
             if bar_count is None:
                 ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
             else:
+                # TODO: optimize
                 ohlc_bars = []
                 for candle in ordered_candles[:bar_count]:
                     ohlc = ohlc_from_candle(candle)
@@ -358,7 +379,7 @@ class Bittrex(Exchange):
             json.dump(symbol_map, f, sort_keys=True, indent=2,
                       separators=(',', ':'))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         if order_type == 'all':
             order_type = 'both'
         elif order_type == 'bid':
@@ -369,7 +390,11 @@ class Bittrex(Exchange):
             raise ValueError('invalid type')
 
         exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(market=exchange_symbol, type=order_type)
+        data = self.api.getorderbook(
+            market=exchange_symbol,
+            type=order_type,
+            depth=100
+        )
 
         result = dict()
         for exchange_type in data:

catalyst/exchange/bittrex/bittrex_api.py

@@ -3,11 +3,12 @@ import json
 import time
 import hmac
 import hashlib
-
-from six.moves import urllib
+import ssl
 
 # Workaround for backwards compatibility
 # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
+from six.moves import urllib
+
 urlopen = urllib.request.urlopen
 
 
@@ -39,13 +40,17 @@ class Bittrex_api(object):
         if method not in self.public:
             url += '&apikey=' + self.key
             url += '&nonce=' + str(int(time.time()))
-            signature = hmac.new(self.secret, url, hashlib.sha512).hexdigest()
+
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 url.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
             headers = {'apisign': signature}
         else:
             headers = {}
 
         req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(req).read())
+        response = json.loads(urlopen(
+            req, context=ssl._create_unverified_context()).read())
 
         if response["result"]:
             return response["result"]

catalyst/exchange/bundle_utils.py

@@ -1,31 +1,48 @@
 import calendar
-import tarfile
-
-import requests
-from datetime import timedelta, datetime, date
 import os
-import pandas as pd
-import numpy as np
+import tarfile
+from datetime import timedelta, datetime, date
 
+import numpy as np
+import pandas as pd
 import pytz
 
-from catalyst.data.bundles import from_bundle_ingest_dirname
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_errors import ApiCandlesError, \
-    PricingDataBeforeTradingError, NoDataAvailableOnExchange
 from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
-from catalyst.utils.deprecate import deprecated
-from catalyst.utils.paths import data_path
 
 EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
 API_URL = 'http://data.enigma.co/api/v1'
 
 
 def get_date_from_ms(ms):
+    """
+    The date from the number of miliseconds from the epoch.
+
+    Parameters
+    ----------
+    ms: int
+
+    Returns
+    -------
+    datetime
+
+    """
     return datetime.fromtimestamp(ms / 1000.0)
 
 
 def get_seconds_from_date(date):
+    """
+    The number of seconds from the epoch.
+
+    Parameters
+    ----------
+    date: datetime
+
+    Returns
+    -------
+    int
+
+    """
     epoch = datetime.utcfromtimestamp(0)
     epoch = epoch.replace(tzinfo=pytz.UTC)
 
@@ -36,16 +53,19 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
     """
     Download and extract a bcolz bundle.
 
-    :param exchange_name:
-    :param symbol:
-    :param data_frequency:
-    :param period:
-    :return:
+    Parameters
+    ----------
+    exchange_name: str
+    symbol: str
+    data_frequency: str
+    period: str
 
-    Note:
+    Returns
+    -------
+    str
         Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
-    """
 
+    """
     root = get_exchange_bundles_folder(exchange_name)
     name = '{exchange}-{frequency}-{symbol}-{period}'.format(
         exchange=exchange_name,
@@ -70,113 +90,189 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
 
 
 def get_delta(periods, data_frequency):
+    """
+    Get a time delta based on the specified data frequency.
+
+    Parameters
+    ----------
+    periods: int
+    data_frequency: str
+
+    Returns
+    -------
+    timedelta
+
+    """
     return timedelta(minutes=periods) \
         if data_frequency == 'minute' else timedelta(days=periods)
 
 
-def get_periods_range(start_dt, end_dt, data_frequency):
-    freq = 'T' if data_frequency == 'minute' else 'D'
+def get_periods_range(start_dt, end_dt, freq):
+    """
+    Get a date range for the specified parameters.
+
+    Parameters
+    ----------
+    start_dt: datetime
+    end_dt: datetime
+    freq: str
+
+    Returns
+    -------
+    DateTimeIndex
+
+    """
+    if freq == 'minute':
+        freq = 'T'
+
+    elif freq == 'daily':
+        freq = 'D'
 
     return pd.date_range(start_dt, end_dt, freq=freq)
 
 
-def get_periods(start_dt, end_dt, data_frequency):
-    delta = end_dt - start_dt
+def get_periods(start_dt, end_dt, freq):
+    """
+    The number of periods in the specified range.
 
-    if data_frequency == 'minute':
-        delta_periods = delta.total_seconds() / 60
+    Parameters
+    ----------
+    start_dt: datetime
+    end_dt: datetime
+    freq: str
 
-    elif data_frequency == 'daily':
-        delta_periods = delta.total_seconds() / 60 / 60 / 24
+    Returns
+    -------
+    int
 
-    else:
-        raise ValueError('frequency not supported')
-
-    return int(delta_periods)
+    """
+    return len(get_periods_range(start_dt, end_dt, freq))
 
 
-def get_start_dt(end_dt, bar_count, data_frequency):
+def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
+    """
+    The start date based on specified end date and data frequency.
+
+    Parameters
+    ----------
+    end_dt: datetime
+    bar_count: int
+    data_frequency: str
+
+    Returns
+    -------
+    datetime
+
+    """
     periods = bar_count
     if periods > 1:
         delta = get_delta(periods, data_frequency)
         start_dt = end_dt - delta
+
+        if not include_first:
+            start_dt += get_delta(1, data_frequency)
     else:
         start_dt = end_dt
 
     return start_dt
 
 
-def get_adj_dates(start, end, assets, data_frequency):
+def get_period_label(dt, data_frequency):
     """
-    Contains a date range to the trading availability of the specified pairs.
+    The period label for the specified date and frequency.
+
+    Parameters
+    ----------
+    dt: datetime
+    data_frequency: str
+
+    Returns
+    -------
+    str
 
-    :param start:
-    :param end:
-    :param assets:
-    :param data_frequency:
-    :return:
     """
-    earliest_trade = None
-    last_entry = None
-    for asset in assets:
-        if earliest_trade is None or earliest_trade > asset.start_date:
-            earliest_trade = asset.start_date
-
-        end_asset = asset.end_minute if data_frequency == 'minute' else \
-            asset.end_daily
-        if end_asset is not None and \
-                (last_entry is None or end_asset > last_entry):
-            last_entry = end_asset
-
-    if start is None or earliest_trade > start:
-        start = earliest_trade
-
-    if end is None or (last_entry is not None and end > last_entry):
-        end = last_entry
-
-    if end is None or start >= end:
-        raise NoDataAvailableOnExchange(
-            exchange=asset.exchange.title(),
-            symbol=[asset.symbol.encode('utf-8')],
-            data_frequency=data_frequency,
-        )
-
-    return start, end
+    return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
+        else '{}'.format(dt.year)
 
 
-def get_month_start_end(dt):
+def get_month_start_end(dt, first_day=None, last_day=None):
     """
-    Returns the first and last day of the month for the specified date.
+    The first and last day of the month for the specified date.
+
+    Parameters
+    ----------
+    dt: datetime
+    first_day: datetime
+    last_day: datetime
+
+    Returns
+    -------
+    datetime, datetime
 
-    :param dt:
-    :return:
     """
     month_range = calendar.monthrange(dt.year, dt.month)
-    month_start = pd.to_datetime(datetime(
-        dt.year, dt.month, 1, 0, 0, 0, 0
-    ), utc=True)
 
-    month_end = pd.to_datetime(datetime(
-        dt.year, dt.month, month_range[1], 23, 59, 0, 0
-    ), utc=True)
+    if first_day:
+        month_start = first_day
+    else:
+        month_start = pd.to_datetime(datetime(
+            dt.year, dt.month, 1, 0, 0, 0, 0
+        ), utc=True)
+
+    if last_day:
+        month_end = last_day
+    else:
+        month_end = pd.to_datetime(datetime(
+            dt.year, dt.month, month_range[1], 23, 59, 0, 0
+        ), utc=True)
+
+        if month_end > pd.Timestamp.utcnow():
+            month_end = pd.Timestamp.utcnow().floor('1D')
 
     return month_start, month_end
 
 
-def get_year_start_end(dt):
+def get_year_start_end(dt, first_day=None, last_day=None):
     """
-    Returns the first and last day of the year for the specified date.
+    The first and last day of the year for the specified date.
+
+    Parameters
+    ----------
+
+    dt: datetime
+    first_day: datetime
+    last_day: datetime
+
+    Returns
+    -------
+    datetime, datetime
 
-    :param dt:
-    :return:
     """
-    year_start = pd.to_datetime(date(dt.year, 1, 1), utc=True)
-    year_end = pd.to_datetime(date(dt.year, 12, 31), utc=True)
+    year_start = first_day if first_day \
+        else pd.to_datetime(date(dt.year, 1, 1), utc=True)
+    year_end = last_day if last_day \
+        else pd.to_datetime(date(dt.year, 12, 31), utc=True)
+
+    if year_end > pd.Timestamp.utcnow():
+        year_end = pd.Timestamp.utcnow().floor('1D')
 
     return year_start, year_end
 
 
 def get_df_from_arrays(arrays, periods):
+    """
+    A DataFrame from the specified OHCLV arrays.
+
+    Parameters
+    ----------
+    arrays: Object
+    periods: DateTimeIndex
+
+    Returns
+    -------
+    DataFrame
+
+    """
     ohlcv = dict()
     for index, field in enumerate(
             ['open', 'high', 'low', 'close', 'volume']):
@@ -189,202 +285,35 @@ def get_df_from_arrays(arrays, periods):
     return df
 
 
-def get_df_from_candles(candles, bar_count, end_dt, data_frequency,
-                        previous_candle=None):
-    """
-    Create candles for each period of the specified range, forward-filling
-    missing candles with the previous value.
-
-    :param candles:
-    :param bar_count:
-    :param end_dt:
-    :param data_frequency:
-    :param previous_candle:
-
-    :return:
-    """
-    all_dates = []
-    all_candles = []
-
-    start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-    date = start_dt
-
-    # TODO: this works well with a small number of candles, consider using numpy as needed
-    while date <= end_dt:
-        candle = next((
-            candle for candle in candles if candle['last_traded'] == date
-        ), previous_candle)
-
-        if candle is None:
-            candle = candles[0]
-
-        all_dates.append(date)
-        all_candles.append(candle)
-
-        previous_candle = candle
-
-        date += get_delta(1, data_frequency)
-
-    return all_dates, all_candles
-
-
-def get_trailing_candles_dt(asset, start_dt, end_dt, data_frequency):
-    missing_start = None
-
-    if asset.end_minute is not None and start_dt < asset.end_minute:
-        if asset.end_minute < end_dt:
-            delta = get_delta(1, data_frequency)
-
-            missing_start = asset.end_minute + delta
-
-    else:
-        missing_start = start_dt
-
-    return missing_start
-
-
 def range_in_bundle(asset, start_dt, end_dt, reader):
     """
     Evaluate whether price data of an asset is included has been ingested in
     the exchange bundle for the given date range.
 
-    :param asset:
-    :param start_dt:
-    :param end_dt:
-    :param reader:
-    :return:
+    Parameters
+    ----------
+    asset: TradingPair
+    start_dt: datetime
+    end_dt: datetime
+    reader: BcolzBarMinuteReader
+
+    Returns
+    -------
+    bool
+
     """
     has_data = True
-    if has_data and reader is not None:
+    dates = [start_dt, end_dt]
+
+    while dates and has_data:
         try:
-            start_close = \
-                reader.get_value(asset.sid, start_dt, 'close')
+            dt = dates.pop(0)
+            close = reader.get_value(asset.sid, dt, 'close')
 
-            if np.isnan(start_close):
+            if np.isnan(close):
                 has_data = False
 
-            else:
-                end_close = reader.get_value(asset.sid, end_dt, 'close')
-
-                if np.isnan(end_close):
-                    has_data = False
-
         except Exception as e:
             has_data = False
 
-    else:
-        has_data = False
-
     return has_data
-
-
-def find_most_recent_time(bundle_name):
-    """
-    Find most recent "time folder" for a given bundle.
-
-    :param bundle_name:
-        The name of the targeted bundle.
-
-    :return folder:
-        The name of the time folder.
-    """
-    try:
-        bundle_folders = os.listdir(
-            data_path([bundle_name]),
-        )
-    except OSError:
-        return None
-
-    most_recent_bundle = dict()
-    for folder in bundle_folders:
-        date = from_bundle_ingest_dirname(folder)
-        if not most_recent_bundle or date > \
-                most_recent_bundle[most_recent_bundle.keys()[0]]:
-            most_recent_bundle = dict()
-            most_recent_bundle[folder] = date
-
-    if most_recent_bundle:
-        return most_recent_bundle.keys()[0]
-    else:
-        return None
-
-
-@deprecated
-def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
-    """
-    History API provides OHLCV data for any of the supported exchanges up to yesterday.
-
-    :param exchange_name: string
-        Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex).
-    :param data_frequency: string
-        Required: The bar frequency (minute or daily)
-    :param symbol: string
-        Required: The trading pair symbol, using Catalyst naming convention
-    :param start: datetime
-        Optional: The start date.
-    :param end: datetime
-        Optional: The end date.
-
-    :return ohlcv: list[dict[string, float]]
-        Each row contains the following dictionary for the resulting bars:
-        'ts'     : int, the timestamp in seconds
-        'open'   : float
-        'high'   : float
-        'low'    : float
-        'close'  : float
-        'volume' : float
-
-    Notes
-    =====
-    Using seconds for the start and end dates for ease of use in the
-    function query parameters.
-
-    Sometimes, one minute goes by without completing a trade of the given
-    trading pair on the given exchange. To minimize the payload size, we
-    don't return identical sequential bars. Post-processing code will
-    forward fill missing bars outside of this function.
-    """
-
-    start_seconds = get_seconds_from_date(start) if start else None
-    end_seconds = get_seconds_from_date(end) if end else None
-
-    if exchange_name not in EXCHANGE_NAMES:
-        raise ValueError(
-            'get_history function only supports the following exchanges: {}'.format(
-                list(EXCHANGE_NAMES)))
-
-    if data_frequency != 'daily' and data_frequency != 'minute':
-        raise ValueError(
-            'get_history currently only supports daily and minute data.'
-        )
-
-    url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format(
-        api_url=API_URL,
-        exchange=exchange_name,
-        symbol=symbol,
-        data_frequency=data_frequency,
-    )
-
-    if start_seconds:
-        url += '&start={}'.format(start_seconds)
-
-    if end_seconds:
-        url += '&end={}'.format(end_seconds)
-
-    try:
-        response = requests.get(url)
-    except Exception as e:
-        raise ValueError(e)
-
-    data = response.json()
-
-    if 'error' in data:
-        raise ApiCandlesError(error=data['error'])
-
-    for candle in data:
-        last_traded = pd.Timestamp.utcfromtimestamp(candle['ts'])
-        last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-        candle['last_traded'] = last_traded
-
-    return data

catalyst/exchange/exchange.py

@@ -1,5 +1,4 @@
 import abc
-import re
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
@@ -9,22 +8,25 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_adj_dates
+    get_delta, get_periods, get_periods_range
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
-    InvalidHistoryFrequencyError, MismatchingFrequencyError, \
-    BundleNotFoundError, NoDataAvailableOnExchange
+    PricingDataNotLoadedError, \
+    NoDataAvailableOnExchange
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
-from catalyst.exchange.exchange_utils import get_exchange_symbols
+from catalyst.exchange.exchange_utils import get_exchange_symbols, \
+    get_frequency, resample_history_df
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
+from catalyst.utils.deprecate import deprecated
 
-log = Logger('Exchange')
+log = Logger('Exchange', level=LOG_LEVEL)
 
 
 class Exchange:
@@ -50,9 +52,11 @@ class Exchange:
     @property
     def portfolio(self):
         """
-        Return the Portfolio
+        The exchange portfolio
 
-        :return:
+        Returns
+        -------
+        ExchangePortfolio
         """
         if self._portfolio is None:
             self._portfolio = ExchangePortfolio(
@@ -70,6 +74,22 @@ class Exchange:
     def time_skew(self):
         pass
 
+    def is_open(self, dt):
+        """
+        Is the exchange open
+
+        Parameters
+        ----------
+        dt: Timestamp
+
+        Returns
+        -------
+        bool
+
+        """
+        # TODO: implement for each exchange.
+        return True
+
     def ask_request(self):
         """
         Asks permission to issue a request to the exchange.
@@ -78,7 +98,9 @@ class Exchange:
         The application will pause if the maximum requests per minute
         permitted by the exchange is exceeded.
 
-        :return boolean:
+        Returns
+        -------
+        bool
 
         """
         now = pd.Timestamp.utcnow()
@@ -87,7 +109,7 @@ class Exchange:
             self.request_cpt[now] = 0
             return True
 
-        cpt_date = self.request_cpt.keys()[0]
+        cpt_date = list(self.request_cpt.keys())[0]
         cpt = self.request_cpt[cpt_date]
 
         if now > cpt_date + timedelta(minutes=1):
@@ -110,10 +132,16 @@ class Exchange:
 
     def get_symbol(self, asset):
         """
-        Get the exchange specific symbol of the given asset.
+        The the exchange specific symbol of the specified market.
+
+        Parameters
+        ----------
+        asset: TradingPair
+
+        Returns
+        -------
+        str
 
-        :param asset: Asset
-        :return: symbol: str
         """
         symbol = None
 
@@ -131,17 +159,34 @@ class Exchange:
         """
         Get a list of symbols corresponding to each given asset.
 
-        :param assets: Asset[]
-        :return:
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[str]
+
         """
         symbols = []
-
         for asset in assets:
             symbols.append(self.get_symbol(asset))
 
         return symbols
 
     def get_assets(self, symbols=None):
+        """
+        The list of markets for the specified symbols.
+
+        Parameters
+        ----------
+        symbols: list[str]
+
+        Returns
+        -------
+        list[TradingPair]
+
+        """
         assets = []
 
         if symbols is not None:
@@ -156,9 +201,16 @@ class Exchange:
 
     def get_asset(self, symbol):
         """
-        Find an Asset on the current exchange based on its Catalyst symbol
-        :param symbol: the [target]_[base] currency pair symbol
-        :return: Asset
+        The market for the specified symbol.
+
+        Parameters
+        ----------
+        symbol: str
+
+        Returns
+        -------
+        TradingPair
+
         """
         asset = None
 
@@ -167,8 +219,10 @@ class Exchange:
                 asset = self.assets[key]
 
         if not asset:
-            supported_symbols = [pair.symbol.encode('utf-8') for pair in
-                                 self.assets.values()]
+            supported_symbols = [
+                pair.symbol for pair in list(self.assets.values())
+            ]
+
             raise SymbolNotFoundOnExchange(
                 symbol=symbol,
                 exchange=self.name.title(),
@@ -187,7 +241,6 @@ class Exchange:
         currency pair symbol. The universal symbol is contained in the
         'symbol' attribute of each asset.
 
-
         Notes
         -----
         The sid of each asset is calculated based on a numeric hash of the
@@ -196,8 +249,8 @@ class Exchange:
 
         This method can be overridden if an exchange offers equivalent data
         via its api.
-        """
 
+        """
         symbol_map = self.fetch_symbol_map()
         for exchange_symbol in symbol_map:
             asset = symbol_map[exchange_symbol]
@@ -258,8 +311,10 @@ class Exchange:
         For each executed order found, create a transaction and apply to the
         Portfolio.
 
-        :return:
-        transactions: Transaction[]
+        Returns
+        -------
+        list[Transaction]
+
         """
         transactions = list()
         if self.portfolio.open_orders:
@@ -342,17 +397,24 @@ class Exchange:
         """
         Similar to 'get_spot_value' but for a single asset
 
-        Note
-        ----
+        Notes
+        -----
         We're writing each minute bar to disk using zipline's machinery.
         This is especially useful when running multiple algorithms
         concurrently. By using local data when possible, we try to reaching
         request limits on exchanges.
 
-        :param asset:
-        :param field:
-        :param data_frequency:
-        :return value: The spot value of the given asset / field
+        Parameters
+        ----------
+        asset: TradingPair
+        field: str
+        data_frequency: str
+
+        Returns
+        -------
+        float
+            The spot value of the given asset / field
+
         """
         log.debug(
             'fetching spot value {field} for symbol {symbol}'.format(
@@ -361,7 +423,8 @@ class Exchange:
             )
         )
 
-        ohlc = self.get_candles(data_frequency, asset)
+        freq = '1T' if data_frequency == 'minute' else '1D'
+        ohlc = self.get_candles(freq, asset)
         if field not in ohlc:
             raise KeyError('Invalid column: %s' % field)
 
@@ -370,75 +433,50 @@ class Exchange:
 
         return value
 
-    def get_series_from_bundle(self, assets, start_dt, end_dt, data_frequency,
-                               field):
-        """
-
-        :return:
-        """
-        reader = self.bundle.get_reader(data_frequency)
-
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=self.name.title(),
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        try:
-            arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=start_dt,
-                end_dt=end_dt
-            )
-
-            periods = self.bundle.get_calendar_periods_range(
-                start_dt, end_dt, data_frequency
-            )
-
-            for asset_index, asset in enumerate(assets):
-                asset_values = arrays[asset_index]
-
-                value_series = pd.Series(asset_values[0], index=periods)
-                series[asset] = value_series
-
-        except Exception as e:
-            log.debug('unable to retrieve from bundle: {}'.format(e))
-
-        return series
-
     def get_series_from_candles(self, candles, start_dt, end_dt,
-                                field, previous_value=None):
+                                data_frequency, field, previous_value=None):
         """
         Get a series of field data for the specified candles.
 
-        :param candles:
-        :param start_dt:
-        :param end_dt:
-        :param field:
-        :param previous_value:
-        :return:
-        """
+        Parameters
+        ----------
+        candles: list[dict[str, float]]
+        start_dt: datetime
+        end_dt: datetime
+        data_frequency: str
+        field: str
+        previous_value: float
 
+        Returns
+        -------
+        Series
+
+        """
         dates = [candle['last_traded'] for candle in candles]
         values = [candle[field] for candle in candles]
-
-        periods = pd.date_range(start_dt, end_dt)
         series = pd.Series(values, index=dates)
 
-        series.reindex(periods, method='ffill', fill_value=previous_value)
+        periods = get_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+        # TODO: ensure that this working as expected, if not use fillna
+        series = series.reindex(
+            periods,
+            method='ffill',
+            fill_value=previous_value,
+        )
 
         return series
 
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency=None,
-                           ffill=True):
+    @deprecated
+    def get_history_window_direct(self,
+                                  assets,
+                                  end_dt,
+                                  bar_count,
+                                  frequency,
+                                  field,
+                                  data_frequency=None,
+                                  ffill=True):
 
         """
         Public API method that returns a dataframe containing the requested
@@ -446,10 +484,11 @@ class Exchange:
 
         Parameters
         ----------
-        assets : list of catalyst.data.Asset objects
+        assets : list[TradingPair]
             The assets whose data is desired.
 
-        end_dt: not applicable to cryptocurrencies
+        end_dt: datetime
+            The date of the last bar
 
         bar_count: int
             The number of bars desired.
@@ -471,70 +510,89 @@ class Exchange:
 
         Returns
         -------
-        A dataframe containing the requested data.
+        DataFrame
+            A dataframe containing the requested data.
+
         """
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
 
-        freq_match = re.match(r'([0-9].*)(m|M|d|D)', frequency, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1))
-            unit = freq_match.group(2)
+        # The get_history method supports multiple asset
+        candles = self.get_candles(
+            data_frequency=frequency,
+            assets=assets,
+            bar_count=bar_count,
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
+        candle_series = self.get_series_from_candles(
+            candles=candles,
+            start_dt=start_dt,
+            end_dt=end_dt,
+            data_frequency=frequency,
+            field=field,
+        )
 
-        else:
-            raise InvalidHistoryFrequencyError(frequency)
+        df = pd.DataFrame(candle_series)
+        return df
 
-        if unit.lower() == 'd':
-            if data_frequency == 'minute':
-                data_frequency = 'daily'
+    def get_history_window(self,
+                           assets,
+                           end_dt,
+                           bar_count,
+                           frequency,
+                           field,
+                           data_frequency=None,
+                           ffill=True):
 
-        elif unit.lower() == 'm':
-            # if data_frequency != 'minute':
-            # raise MismatchingFrequencyError(
-            #     frequency=frequency,
-            #     data_frequency=data_frequency
-            # )
-            if data_frequency == 'daily':
-                data_frequency = 'minute'
+        """
+        Public API method that returns a dataframe containing the requested
+        history window.  Data is fully adjusted.
 
-        else:
-            raise InvalidHistoryFrequencyError(frequency)
+        Parameters
+        ----------
+        assets : list[TradingPair]
+            The assets whose data is desired.
 
+        end_dt: datetime
+            The date of the last bar.
+
+        bar_count: int
+            The number of bars desired.
+
+        frequency: string
+            "1d" or "1m"
+
+        field: string
+            The desired field of the asset.
+
+        data_frequency: string
+            The frequency of the data to query; i.e. whether the data is
+            'daily' or 'minute' bars.
+
+        # TODO: fill how?
+        ffill: boolean
+            Forward-fill missing values. Only has effect if field
+            is 'price'.
+
+        Returns
+        -------
+        DataFrame
+            A dataframe containing the requested data.
+
+        """
+        freq, candle_size, unit, data_frequency = get_frequency(
+            frequency, data_frequency
+        )
         adj_bar_count = candle_size * bar_count
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         try:
-            adj_start_dt, adj_end_dt = get_adj_dates(
-                start_dt, end_dt, assets, data_frequency
-            )
-            in_bundle = True
-
-        except NoDataAvailableOnExchange:
-            in_bundle = False
-
-        if in_bundle:
-            missing_assets = self.bundle.filter_existing_assets(
+            series = self.bundle.get_history_window_series_and_load(
                 assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
+                end_dt=end_dt,
+                bar_count=adj_bar_count,
+                field=field,
                 data_frequency=data_frequency
             )
-
-            if missing_assets:
-                self.bundle.ingest_assets(
-                    assets=assets,
-                    start_dt=adj_start_dt,
-                    end_dt=adj_end_dt,
-                    data_frequency=data_frequency
-                )
-
-            series = self.get_series_from_bundle(
-                assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
-                data_frequency=data_frequency,
-                field=field
-            )
-
-        else:
+        except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
             series = dict()
 
         for asset in assets:
@@ -542,29 +600,35 @@ class Exchange:
                 # Adding bars too recent to be contained in the consolidated
                 # exchanges bundles. We go directly against the exchange
                 # to retrieve the candles.
-
+                start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
                         if asset in series else start_dt
 
-                trailing_bar_count = \
-                    get_periods(trailing_dt, end_dt, data_frequency)
-
                 # The get_history method supports multiple asset
+                # Use the original frequency to let each api optimize
+                # the size of result sets
+                trailing_bar_count = get_periods(
+                    trailing_dt, end_dt, freq
+                )
                 candles = self.get_candles(
-                    data_frequency=data_frequency,
+                    freq=freq,
                     assets=asset,
                     bar_count=trailing_bar_count,
+                    start_dt=start_dt,
                     end_dt=end_dt
                 )
 
                 last_value = series[asset].iloc(0) if asset in series \
                     else np.nan
 
+                # Create a series with the common data_frequency, ffill
+                # missing values
                 candle_series = self.get_series_from_candles(
                     candles=candles,
                     start_dt=trailing_dt,
                     end_dt=end_dt,
+                    data_frequency=data_frequency,
                     field=field,
                     previous_value=last_value
                 )
@@ -575,23 +639,9 @@ class Exchange:
                 else:
                     series[asset] = candle_series
 
-        df = pd.DataFrame(series)
-
-        if candle_size > 1:
-            if field == 'open':
-                agg = 'first'
-            elif field == 'high':
-                agg = 'max'
-            elif field == 'low':
-                agg = 'min'
-            elif field == 'close':
-                agg = 'last'
-            elif field == 'volume':
-                agg = 'sum'
-            else:
-                raise ValueError('Invalid field.')
-
-            df = df.resample('{}T'.format(candle_size)).agg(agg)
+        df = resample_history_df(pd.DataFrame(series), freq, field)
+        # TODO: consider this more carefully
+        df.dropna(inplace=True)
 
         return df
 
@@ -600,7 +650,6 @@ class Exchange:
         Update the portfolio cash and position balances based on the
         latest ticker prices.
 
-        :return:
         """
         log.debug('synchronizing portfolio with exchange {}'.format(self.name))
         balances = self.get_balances()
@@ -622,7 +671,7 @@ class Exchange:
             portfolio.starting_cash = portfolio.cash
 
         if portfolio.positions:
-            assets = portfolio.positions.keys()
+            assets = list(portfolio.positions.keys())
             tickers = self.tickers(assets)
 
             portfolio.positions_value = 0.0
@@ -644,16 +693,20 @@ class Exchange:
 
         Parameters
         ----------
-        asset : Asset
+        asset : TradingPair
             The asset that this order is for.
+
         amount : int
             The amount of shares to order. If ``amount`` is positive, this is
             the number of shares to buy or cover. If ``amount`` is negative,
             this is the number of shares to sell or short.
+
         limit_price : float, optional
             The limit price for the order.
+
         stop_price : float, optional
             The stop price for the order.
+
         style : ExecutionStyle, optional
             The execution style for the order.
 
@@ -678,6 +731,7 @@ class Exchange:
         :class:`catalyst.finance.execution.ExecutionStyle`
         :func:`catalyst.api.order_value`
         :func:`catalyst.api.order_percent`
+
         """
         if amount == 0:
             log.warn('skipping order amount of 0')
@@ -727,8 +781,12 @@ class Exchange:
     @abstractmethod
     def get_balances(self):
         """
-        Retrieve wallet balances for the exchange
-        :return balances: A dict of currency => available balance
+        Retrieve wallet balances for the exchange.
+
+        Returns
+        -------
+        dict[TradingPair, float]
+
         """
         pass
 
@@ -737,17 +795,25 @@ class Exchange:
         """
         Place an order on the exchange.
 
-        :param asset : Asset
-            The asset that this order is for.
-        :param amount : int
+        Parameters
+        ----------
+        asset: TradingPair
+            The target market.
+
+        amount: float
             The amount of shares to order. If ``amount`` is positive, this is
             the number of shares to buy or cover. If ``amount`` is negative,
             this is the number of shares to sell or short.
-        :param style : ExecutionStyle
-            The execution style for the order.
-        :param is_buy: boolean
+
+        is_buy: bool
             Is it a buy order?
-        :return:
+
+        style: ExecutionStyle
+
+        Returns
+        -------
+        Order
+
         """
         pass
 
@@ -802,23 +868,32 @@ class Exchange:
         pass
 
     @abstractmethod
-    def get_candles(self, data_frequency, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count=None,
                     start_dt=None, end_dt=None):
         """
         Retrieve OHLCV candles for the given assets
 
-        :param data_frequency:
-            The candle frequency: minute or daily
-        :param assets: list[TradingPair]
+        Parameters
+        ----------
+        freq: str
+            The frequency alias per convention:
+            http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+
+        assets: list[TradingPair]
             The targeted assets.
-        :param bar_count:
+
+        bar_count: int
             The number of bar desired. (default 1)
-        :param end_dt: datetime, optional
+
+        end_dt: datetime, optional
             The last bar date.
-        :param start_dt: datetime, optional
+
+        start_dt: datetime, optional
             The first bar date.
 
-        :return dict[TradingPair, dict[str, Object]]: OHLCV data
+        Returns
+        -------
+        dict[TradingPair, dict[str, Object]]
             A dictionary of OHLCV candles. Each TradingPair instance is
             mapped to a list of dictionaries with this structure:
                 open: float
@@ -838,8 +913,14 @@ class Exchange:
         """
         Retrieve current tick data for the given assets
 
-        :param assets:
-        :return:
+        Parameters
+        ----------
+        assets: list[TradingPair]
+
+        Returns
+        -------
+        list[dict[str, float]
+
         """
         pass
 
@@ -847,19 +928,23 @@ class Exchange:
     def get_account(self):
         """
         Retrieve the account parameters.
-        :return:
         """
         pass
 
     @abc.abstractmethod
-    def get_orderbook(self, asset, order_type):
+    def get_orderbook(self, asset, order_type, limit):
         """
         Retrieve the the orderbook for the given trading pair.
 
-        :param asset: TradingPair
-        :param order_type: str
+        Parameters
+        ----------
+        asset: TradingPair
+        order_type: str
             The type of orders: bid, ask or all
+        limit: int
 
-        :return:
+        Returns
+        -------
+        list[dict[str, float]
         """
         pass

catalyst/exchange/exchange_algorithm.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import pickle
 import signal
 import sys
@@ -26,8 +25,7 @@ from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
-from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
-    BcolzMinuteBarReader
+from catalyst.constants import LOG_LEVEL
 from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
@@ -37,8 +35,8 @@ from catalyst.exchange.exchange_errors import (
     OrphanOrderError)
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
-    save_algo_object, get_algo_object, get_algo_folder, get_algo_df, \
+from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
+    get_algo_folder, get_algo_df, \
     save_algo_df
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
@@ -51,10 +49,10 @@ from catalyst.utils.api_support import (
     disallowed_in_before_trading_start)
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
     expect_types
-from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
+from catalyst.utils.preprocess import preprocess
 
-log = logbook.Logger('exchange_algorithm')
+log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
 
 class ExchangeAlgorithmExecutor(AlgorithmSimulator):
@@ -112,7 +110,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             else self.sim_params.end_session
 
         if exchange_name is None:
-            exchange = self.exchanges.values()[0]
+            exchange = list(self.exchanges.values())[0]
         else:
             exchange = self.exchanges[exchange_name]
 
@@ -126,7 +124,13 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
         """
         Creates a dictionary representing the state of the tracker.
 
+        Parameters
+        ----------
+        start_dt: datetime
+        end_dt: datetime
 
+        Notes
+        -----
         I rewrote this in an attempt to better control the stats.
         I don't want things to happen magically through complex logic
         pertaining to backtesting.
@@ -175,17 +179,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
-        stats['transactions'] = dict()
+        stats['transactions'] = []
         for date in period.processed_transactions:
             if start_dt <= date < end_dt:
-                stats['transactions'][date] = \
-                    period.processed_transactions[date]
+                transactions = period.processed_transactions[date]
+                for t in transactions:
+                    stats['transactions'].append(t.to_dict())
 
-        stats['orders'] = dict()
+        stats['orders'] = []
         for date in period.orders_by_modified:
             if start_dt <= date < end_dt:
-                stats['orders'][date] = \
-                    period.orders_by_modified[date]
+                orders = period.orders_by_modified[date]
+                for order in orders:
+                    stats['orders'].append(orders[order].to_dict())
 
         return stats
 
@@ -194,6 +200,7 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
+        self.frame_stats = list()
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
@@ -238,6 +245,19 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
         else:
             return MarketOrder()
 
+    def handle_data(self, data):
+        super(ExchangeTradingAlgorithmBacktest, self).handle_data(data)
+
+        minute_stats = self.prepare_period_stats(
+            data.current_dt, data.current_dt + timedelta(minutes=1))
+        self.frame_stats.append(minute_stats)
+
+    def analyze(self, perf):
+        stats = pd.DataFrame(self.frame_stats)
+        stats.set_index('period_close', inplace=True, drop=False)
+
+        super(ExchangeTradingAlgorithmBacktest, self).analyze(stats)
+
 
 class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
@@ -266,35 +286,24 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.stats_minutes = 5
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
-        # TODO: fix precision before re-enabling
-        # self._create_minute_writer()
 
         signal.signal(signal.SIGINT, self.signal_handler)
 
         log.info('initialized trading algorithm in live mode')
 
-    def _create_minute_writer(self):
-        root = get_exchange_minute_writer_root(self.exchange.name)
-        filename = os.path.join(root, 'metadata.json')
-
-        if os.path.isfile(filename):
-            writer = BcolzMinuteBarWriter.open(
-                root, self.sim_params.end_session)
-        else:
-            # TODO: need to be able to write more precise numbers
-            writer = BcolzMinuteBarWriter(
-                rootdir=root,
-                calendar=self.trading_calendar,
-                minutes_per_day=1440,
-                start_session=self.sim_params.start_session,
-                end_session=self.sim_params.end_session,
-                write_metadata=True
-            )
-
-        self.exchange.minute_writer = writer
-        self.exchange.minute_reader = BcolzMinuteBarReader(root)
-
     def signal_handler(self, signal, frame):
+        """
+        Handles the keyboard interruption signal.
+
+        Parameters
+        ----------
+        signal
+        frame
+
+        Returns
+        -------
+
+        """
         self.is_running = False
 
         if self._analyze is None:
@@ -383,7 +392,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         """
         We skip the entire performance tracker business and update the
         portfolio directly.
-        :return:
+
+        Returns
+        -------
+        ExchangePortfolio
+
         """
         # TODO: build cumulative portfolio
         return self.perf_tracker.get_portfolio(False)
@@ -449,6 +462,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                 )
 
     def add_pnl_stats(self, period_stats):
+        """
+        Save p&l stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         starting = period_stats['starting_cash']
         current = period_stats['portfolio_value']
         appreciation = (current / starting) - 1
@@ -465,6 +489,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
 
     def add_custom_signals_stats(self, period_stats):
+        """
+        Save custom signals stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         log.debug('adding custom signals stats: {}'.format(self.recorded_vars))
         df = pd.DataFrame(
             data=[self.recorded_vars],
@@ -476,6 +511,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      self.custom_signals_stats)
 
     def add_exposure_stats(self, period_stats):
+        """
+        Save exposure stats.
+
+        Parameters
+        ----------
+        period_stats
+
+        Returns
+        -------
+
+        """
         data = dict(
             long_exposure=period_stats['long_exposure'],
             base_currency=period_stats['ending_cash']
@@ -492,6 +538,14 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                      self.exposure_stats)
 
     def handle_data(self, data):
+        """
+        Wrapper around the handle_data method of each algo.
+
+        Parameters
+        ----------
+        data
+
+        """
         if not self.is_running:
             return
 
@@ -523,7 +577,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.add_pnl_stats(minute_stats)
             if self.recorded_vars:
                 self.add_custom_signals_stats(minute_stats)
-                recorded_cols = self.recorded_vars.keys()
+                recorded_cols = list(self.recorded_vars.keys())
             else:
                 recorded_cols = None
 
@@ -555,6 +609,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
+        # TODO: pickle does not seem to work in python 3
         try:
             save_algo_object(
                 algo_name=self.algo_namespace,
@@ -617,15 +672,16 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         The cumulative portfolio does not contain open orders but exchange
         portfolios do.
 
-        :param asset: TradingPair
-        :param amount: float
-        :param limit_price: float
-        :param stop_price: float
-        :param style: Style
-        :return order: Order
+        Parameters
+        ----------
+        asset: TradingPair
+        amount: float
+        limit_price: float
+        stop_price: float
+        style: Style
+        order: Order
             The catalyst order object or None
         """
-
         amount, style = self._calculate_order(asset, amount,
                                               limit_price, stop_price,
                                               style)
@@ -687,15 +743,53 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
                         'get_open_orders. Use `asset` instead.')
     @api_method
     def get_open_orders(self, asset=None):
+        """Retrieve all of the current open orders.
+
+        Parameters
+        ----------
+        asset : Asset
+            If passed and not None, return only the open orders for the given
+            asset instead of all open orders.
+
+        Returns
+        -------
+        open_orders : dict[list[Order]] or list[Order]
+            If no asset is passed this will return a dict mapping Assets
+            to a list containing all the open orders for the asset.
+            If an asset is passed then this will return a list of the open
+            orders for this asset.
+        """
         return self._get_open_orders(asset)
 
     @api_method
     def get_order(self, order_id, exchange_name):
+        """Lookup an order based on the order id returned from one of the
+        order functions.
+
+        Parameters
+        ----------
+        order_id : str
+            The unique identifier for the order.
+
+        Returns
+        -------
+        order : Order
+            The order object.
+        execution_price: float
+            The execution price per share of the order
+        """
         exchange = self.exchanges[exchange_name]
         return exchange.get_order(order_id)
 
     @api_method
     def cancel_order(self, order_param, exchange_name):
+        """Cancel an open order.
+
+        Parameters
+        ----------
+        order_param : str or Order
+            The order_id or order object to cancel.
+        """
         exchange = self.exchanges[exchange_name]
 
         order_id = order_param

catalyst/exchange/exchange_bcolz.py

@@ -3,7 +3,6 @@ import numpy as np
 from catalyst import get_calendar
 from catalyst.data.minute_bars import BcolzMinuteBarReader, \
     BcolzMinuteBarWriter
-from catalyst.exchange.bundle_utils import get_periods, get_periods_range
 
 
 class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
@@ -17,7 +16,7 @@ class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
             end_session = end_session.floor('1d')
 
         minutes_per_day = 1440 if self._data_frequency == 'minute' else 1
-        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 1000000)
+        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000)
         calendar = get_calendar('OPEN')
 
         super(BcolzExchangeBarWriter, self) \
@@ -40,17 +39,25 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
         return self._data_frequency
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
+        """
+        Parameters
+        ----------
+        fields : list of str
+           'open', 'high', 'low', 'close', or 'volume'
+        start_dt: Timestamp
+           Beginning of the window range.
+        end_dt: Timestamp
+           End of the window range.
+        sids : list of int
+           The asset identifiers in the window.
 
-        # if self._data_frequency == 'minute':
-        #     return super(BcolzExchangeBarReader, self) \
-        #         .load_raw_arrays(fields, start_dt, end_dt, sids)
-        #
-        # else:
-        #     return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
-
-        return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
-
-    def _load_daily_raw_arrays(self, fields, start_dt, end_dt, sids):
+        Returns
+        -------
+        list of np.ndarray
+            A list with an entry per field of ndarrays with shape
+            (minutes in range, sids) with a dtype of float64, containing the
+            values for the respective field over start and end dt range.
+        """
         start_idx = self._find_position_of_minute(start_dt)
         end_idx = self._find_position_of_minute(end_dt)
 
@@ -80,8 +87,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
                 if mask is None:
                     mask = a != 0
 
+                inverse_ratio = self._ohlc_ratio_inverse_for_sid(sid)
                 out[:len(mask), i][mask] = (
-                    a[mask] * self._ohlc_ratio_inverse_for_sid(sid)
+                    a[mask] * inverse_ratio
                 )
 
             if field in fields:

catalyst/exchange/exchange_blotter.py

@@ -1,19 +1,20 @@
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
-from catalyst.finance.transaction import Transaction
+from catalyst.finance.transaction import create_transaction
 
-log = Logger('exchange_blotter')
+log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos
 # Orders won't often close at Equity levels.
-# TODO: consider adjusting dynamically based on trading pair
-DEFAULT_SLIPPAGE_SPREAD = 0.02
-DEFAULT_MAKER_FEE = 0.001
-DEFAULT_TAKER_FEE = 0.002
+# TODO: should work with set_commission and set_slippage
+DEFAULT_SLIPPAGE_SPREAD = 0.0001
+DEFAULT_MAKER_FEE = 0.0015
+DEFAULT_TAKER_FEE = 0.0025
 
 
 class TradingPairFeeSchedule(CommissionModel):
@@ -96,12 +97,8 @@ class TradingPairFixedSlippage(SlippageModel):
 
             execution_price, execution_volume = self.process_order(data, order)
 
-            transaction = Transaction(
-                asset=order.asset,
-                amount=abs(execution_volume),
-                dt=dt,
-                price=execution_price,
-                order_id=order.id
+            transaction = create_transaction(
+                order, dt, execution_price, execution_volume
             )
 
             self._volume_for_bar += abs(transaction.amount)

catalyst/exchange/exchange_data_portal.py

@@ -1,35 +1,21 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import abc
-from datetime import timedelta
 from time import sleep
 
+import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
-from catalyst.errors import HistoryWindowStartsBeforeData
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
-    PricingDataBeforeTradingError,
-    PricingDataNotLoadedError, InvalidHistoryFrequencyError,
-    BundleNotFoundError)
+    PricingDataNotLoadedError)
+from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
 
-log = Logger('DataPortalExchange')
+log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
@@ -82,7 +68,7 @@ class DataPortalExchangeBase(DataPortal):
                 return pd.concat(df_list)
 
             else:
-                exchange = self.exchanges[exchange_assets.keys()[0]]
+                exchange = self.exchanges[list(exchange_assets.keys())[0]]
                 return self.get_exchange_history_window(
                     exchange,
                     assets,
@@ -153,6 +139,10 @@ class DataPortalExchangeBase(DataPortal):
                 exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
                     exchange, [assets], field, dt, data_frequency)
+
+                if not spot_values:
+                    return np.nan
+
                 return spot_values[0]
 
             else:
@@ -163,8 +153,8 @@ class DataPortalExchangeBase(DataPortal):
 
                     exchange_assets[asset.exchange].append(asset)
 
-                if len(exchange_assets.keys()) == 1:
-                    exchange = self.exchanges[exchange_assets.keys()[0]]
+                if len(list(exchange_assets.keys())) == 1:
+                    exchange = self.exchanges[list(exchange_assets.keys())[0]]
                     return self.get_exchange_spot_value(
                         exchange, assets, field, dt, data_frequency)
 
@@ -235,6 +225,25 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets: list[TradingPair]
+        end_dt: datetime
+        bar_count: int
+        frequency: str
+        field: str
+        data_frequency: str
+        ffill: bool
+
+        Returns
+        -------
+        DataFrame
+
+        """
         df = exchange.get_history_window(
             assets,
             end_dt,
@@ -247,6 +256,22 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
 
     def get_exchange_spot_value(self, exchange, assets, field, dt,
                                 data_frequency):
+        """
+        A spot value for the exchange.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets: list[TradingPair]
+        field: str
+        dt: datetime
+        data_frequency: str
+
+        Returns
+        -------
+        float
+
+        """
         exchange_spot_values = exchange.get_spot_value(
             assets, field, dt, data_frequency)
 
@@ -282,109 +307,99 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange bundle.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets: list[TradingPair]
+        end_dt: datetime
+        bar_count: int
+        frequency: str
+        field: str
+        data_frequency: str
+        ffill: bool
+
+        Returns
+        -------
+        DataFrame
+
+        """
+        bundle = self.exchange_bundles[exchange.name]  # type: ExchangeBundle
+
+        freq, candle_size, unit, adj_data_frequency = get_frequency(
+            frequency, data_frequency
+        )
+        adj_bar_count = candle_size * bar_count
+
+        if data_frequency == 'minute' and adj_data_frequency == 'daily':
+            end_dt = end_dt.floor('1D')
+
+        series = bundle.get_history_window_series_and_load(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=adj_bar_count,
+            field=field,
+            data_frequency=adj_data_frequency,
+            algo_end_dt=self._last_available_session,
+        )
+
+        df = resample_history_df(pd.DataFrame(series), freq, field)
+        return df
+
+    def get_exchange_spot_value(self,
+                                exchange,
+                                assets,
+                                field,
+                                dt,
+                                data_frequency
+                                ):
+        """
+        A spot value for the exchange bundle. Try to ingest data if not in
+        the bundle.
+
+        Parameters
+        ----------
+        exchange: Exchange
+        assets: list[TradingPair]
+        field: str
+        dt: datetime
+        data_frequency: str
+
+        Returns
+        -------
+        float
+
+        """
         bundle = self.exchange_bundles[exchange.name]
-
-        if data_frequency == 'minute':
-            dts = self.trading_calendar.minutes_window(
-                end_dt, -bar_count
-            )
-
-            self.ensure_after_first_day(dts[0], assets)
-
-        elif data_frequency == 'daily':
-            session = self.trading_calendar.minute_to_session_label(end_dt)
-            dts = self._get_days_for_window(session, bar_count)
-
-            if len(dts) == 0:
-                symbols = [asset.symbol for asset in assets]
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    symbols=symbols,
-                    exchange=exchange.name,
-                    first_trading_day= \
-                        min([asset.start_date for asset in assets]),
-                    data_frequency=data_frequency,
-                    symbol_list=','.join(symbols)
-                )
-
-            self.ensure_after_first_day(dts[0], assets)
-
+        if data_frequency == 'daily':
+            dt = dt.floor('1D')
         else:
-            raise InvalidHistoryFrequencyError(frequency=data_frequency)
+            dt = dt.floor('1 min')
 
-        reader = bundle.get_reader(data_frequency)
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=exchange.name.title(),
-                data_frequency=data_frequency
-            )
-
-        try:
-            values = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=dts[0],
-                end_dt=dts[-1]
-            )[0]
-
-        except Exception:
-            first_trading_day = self._get_first_trading_day(assets)
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-
-            symbol_list = ','.join(symbols)
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=first_trading_day,
-                exchange=exchange.name.title(),
-                symbols=symbols,
-                symbol_list=symbol_list,
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        for index, asset in enumerate(assets):
-            asset_values = values[:, index]
-
-            value_series = pd.Series(asset_values, index=dts)
-            series[asset] = value_series
-
-        return pd.DataFrame(series)
-
-    def ensure_after_first_day(self, dt, assets):
-        first_trading_day = self._get_first_trading_day(assets)
-        if dt < first_trading_day:
-            raise PricingDataBeforeTradingError(
-                first_trading_day=first_trading_day,
-                exchange=assets[0].exchange.title(),
-                symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                dt=dt,
-            )
-
-    def get_exchange_spot_value(self, exchange, assets, field, dt,
-                                data_frequency):
-        bundle = self.exchange_bundles[exchange.name]
-        reader = bundle.get_reader(data_frequency)
-
-        self.ensure_after_first_day(dt, assets)
-
-        values = []
-        for asset in assets:
+        if AUTO_INGEST:
             try:
-                value = reader.get_value(
-                    sid=asset.sid,
-                    dt=dt,
-                    field=field
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency
                 )
-                values.append(value)
-            except Exception:
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    first_trading_day=self._get_first_trading_day(assets),
-                    exchange=exchange.name.title(),
-                    symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                    symbol_list=''.join(
-                        [asset.symbol.encode('utf-8') for asset in assets]),
-                    data_frequency=data_frequency
+            except PricingDataNotLoadedError:
+                log.info(
+                    'pricing data for {symbol} not found on {dt}'
+                    ', updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        dt=dt
+                    )
                 )
-
-            return values
+                bundle.ingest_assets(
+                    assets=assets,
+                    start_dt=self._first_trading_day,
+                    end_dt=self._last_available_session,
+                    data_frequency=data_frequency,
+                    show_progress=True
+                )
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency, True
+                )
+        else:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -1,14 +1,17 @@
-import sys, traceback
+import sys
+import traceback
+
 from catalyst.errors import ZiplineError
 
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange,
+                   ExchangeAuthEmpty]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
-        print "Error traceback: {1} (line {2})\n" \
-              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue)
+        print("Error traceback: {1} (line {2})\n"
+              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
     else:
         sys.__excepthook__(exctype, excvalue, exctraceback)
 
@@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
+class ExchangeAuthEmpty(ZiplineError):
+    msg = (
+        'Please enter your API token key and secret for exchange {exchange} '
+        'in the following file: {filename}'
+    ).strip()
+
+
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '
@@ -76,6 +86,14 @@ class AlgoPickleNotFound(ZiplineError):
     ).strip()
 
 
+class InvalidHistoryFrequencyAlias(ZiplineError):
+    msg = (
+        'Invalid frequency alias {freq}. Valid suffixes are M (minute) '
+        'and D (day). For example, these aliases would be valid '
+        '1M, 5M, 1D.'
+    ).strip()
+
+
 class InvalidHistoryFrequencyError(ZiplineError):
     msg = (
         'Frequency {frequency} not supported by the exchange.'
@@ -193,18 +211,19 @@ class PricingDataBeforeTradingError(ZiplineError):
 
 
 class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Pricing data {field} for trading pairs {symbols} trading on '
-           'exchange {exchange} since {first_trading_day} is unavailable. '
-           'The bundle data is either out-of-date or has not been loaded yet. '
-           'Please ingest data using the command '
-           '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. '
-           'See catalyst documentation for details.').strip()
+    msg = ('Missing data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]'
+           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
+           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
+           'for details.').strip()
 
 
 class ApiCandlesError(ZiplineError):
     msg = ('Unable to fetch candles from the remote API: {error}.').strip()
 
+
 class NoDataAvailableOnExchange(ZiplineError):
-    msg = ('Requested data for trading pair {symbol} is not available on exchange {exchange} '
-           'in `{data_frequency}` frequency at this time. '
-           'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+    msg = (
+        'Requested data for trading pair {symbol} is not available on exchange {exchange} '
+        'in `{data_frequency}` frequency at this time. '
+        'Check `http://enigma.co/catalyst/status` for market coverage.').strip()

catalyst/exchange/exchange_execution.py

@@ -4,9 +4,16 @@ from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
 class ExchangeLimitOrder(LimitOrder):
     def get_limit_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.limit_price
 
@@ -14,9 +21,16 @@ class ExchangeLimitOrder(LimitOrder):
 class ExchangeStopOrder(StopOrder):
     def get_stop_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.stop_price
 
@@ -24,16 +38,30 @@ class ExchangeStopOrder(StopOrder):
 class ExchangeStopLimitOrder(StopLimitOrder):
     def get_limit_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.limit_price
 
     def get_stop_price(self, is_buy):
         """
-        We may be trading Satoshis with 8 decimals, we cannot round numbers
-        :param is_buy:
-        :return:
+        We may be trading Satoshis with 8 decimals, we cannot round numbers.
+
+        Parameters
+        ----------
+        is_buy: bool
+
+        Returns
+        -------
+        float
+
         """
         return self.stop_price

catalyst/exchange/exchange_portfolio.py

@@ -1,9 +1,11 @@
 import numpy as np
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
+from catalyst.utils.deprecate import deprecated
 
-log = Logger('ExchangePortfolio')
+log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
 
 class ExchangePortfolio(Portfolio):
@@ -28,10 +30,15 @@ class ExchangePortfolio(Portfolio):
         self.positions_value = 0.0
         self.open_orders = dict()
 
-    def calculate_pnl(self):
-        log.debug('calculating pnl')
-
     def create_order(self, order):
+        """
+        Create an open order and store in memory.
+
+        Parameters
+        ----------
+        order: Order
+
+        """
         log.debug('creating order {}'.format(order.id))
         self.open_orders[order.id] = order
 
@@ -46,6 +53,18 @@ class ExchangePortfolio(Portfolio):
         log.debug('open order added to portfolio')
 
     def execute_order(self, order, transaction):
+        """
+        Update the open orders and positions to apply an executed order.
+
+        Unlike with backtesting, we do not need to add slippage and fees.
+        The executed price includes transaction fees.
+
+        Parameters
+        ----------
+        order: Order
+        transaction: Transaction
+
+        """
         log.debug('executing order {}'.format(order.id))
         del self.open_orders[order.id]
 
@@ -70,7 +89,9 @@ class ExchangePortfolio(Portfolio):
 
         log.debug('updated portfolio with executed order')
 
+    @deprecated
     def execute_transaction(self, transaction):
+        # TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
         log.debug('executing transaction {}'.format(transaction.order_id))
 
         order_position = self.positions[transaction.asset] \
@@ -95,6 +116,14 @@ class ExchangePortfolio(Portfolio):
         log.debug('updated portfolio with executed order')
 
     def remove_order(self, order):
+        """
+        Removing an open order.
+
+        Parameters
+        ----------
+        order: Order
+
+        """
         log.info('removing cancelled order {}'.format(order.id))
         del self.open_orders[order.id]
 

catalyst/exchange/exchange_utils.py

@@ -1,20 +1,37 @@
 import json
 import os
 import pickle
-import urllib
+import re
+import shutil
 from datetime import date, datetime
 
 import pandas as pd
+from catalyst.assets._assets import TradingPair
+from six.moves.urllib import request
 
-from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
-    ExchangeSymbolsNotFound
-from catalyst.utils.paths import data_root, ensure_directory, last_modified_time
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
+    InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
+from catalyst.utils.paths import data_root, ensure_directory, \
+    last_modified_time
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
 
 
 def get_exchange_folder(exchange_name, environ=None):
+    """
+    The root path of an exchange folder.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     if not environ:
         environ = os.environ
 
@@ -26,22 +43,63 @@ def get_exchange_folder(exchange_name, environ=None):
 
 
 def get_exchange_symbols_filename(exchange_name, environ=None):
+    """
+    The absolute path of the exchange's symbol.json file.
+
+    Parameters
+    ----------
+    exchange_name:
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
     return os.path.join(exchange_folder, 'symbols.json')
 
 
 def download_exchange_symbols(exchange_name, environ=None):
+    """
+    Downloads the exchange's symbols.json from the repository.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     filename = get_exchange_symbols_filename(exchange_name)
     url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = urllib.urlretrieve(url=url, filename=filename)
+    response = request.urlretrieve(url=url, filename=filename)
     return response
 
 
 def get_exchange_symbols(exchange_name, environ=None):
+    """
+    The de-serialized content of the exchange's symbols.json.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
     filename = get_exchange_symbols_filename(exchange_name)
 
     if not os.path.isfile(filename) or \
-            pd.Timedelta(pd.Timestamp('now', tz='UTC') - last_modified_time(filename)).days > 1:
+                    pd.Timedelta(pd.Timestamp('now',
+                                              tz='UTC') - last_modified_time(
+                        filename)).days > 1:
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
@@ -55,7 +113,37 @@ def get_exchange_symbols(exchange_name, environ=None):
         )
 
 
+def get_symbols_string(assets):
+    """
+    A concatenated string of symbols from a list of assets.
+
+    Parameters
+    ----------
+    assets: list[TradingPair]
+
+    Returns
+    -------
+    str
+
+    """
+    array = [assets] if isinstance(assets, TradingPair) else assets
+    return ', '.join([asset.symbol for asset in array])
+
+
 def get_exchange_auth(exchange_name, environ=None):
+    """
+    The de-serialized contend of the exchange's auth.json file.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    Object
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
     filename = os.path.join(exchange_folder, 'auth.json')
 
@@ -64,13 +152,45 @@ def get_exchange_auth(exchange_name, environ=None):
             data = json.load(data_file)
             return data
     else:
-        raise ExchangeAuthNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
+        data = dict(name=exchange_name, key='', secret='')
+        with open(filename, 'w') as f:
+            json.dump(data, f, sort_keys=False, indent=2,
+                      separators=(',', ':'))
+            return data
+
+
+def delete_algo_folder(algo_name, environ=None):
+    """
+    Delete the folder containing the algo state.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
+    shutil.rmtree(folder)
 
 
 def get_algo_folder(algo_name, environ=None):
+    """
+    The algorithm root folder of the algorithm.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     if not environ:
         environ = os.environ
 
@@ -82,6 +202,21 @@ def get_algo_folder(algo_name, environ=None):
 
 
 def get_algo_object(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized object of the algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    environ:
+    rel_path: str
+
+    Returns
+    -------
+    Object
+
+    """
     if algo_name is None:
         return None
 
@@ -103,6 +238,18 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
 
 
 def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+    """
+    Serialize and save an object by algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    obj: Object
+    environ:
+    rel_path: str
+
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -115,16 +262,22 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
-def append_algo_object(algo_name, key, obj, environ=None):
-    algo_folder = get_algo_folder(algo_name, environ)
-    filename = os.path.join(algo_folder, key + '.p')
-
-    mode = 'a+b' if os.path.isfile(filename) else 'wb'
-    with open(filename, mode) as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized DataFrame of an algo name and key.
+
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    environ:
+    rel_path: str
+
+    Returns
+    -------
+    DataFrame
+
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -143,19 +296,43 @@ def get_algo_df(algo_name, key, environ=None, rel_path=None):
 
 
 def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
-    folder = get_algo_folder(algo_name, environ)
+    """
+    Serialize to csv and save a DataFrame by algo name and key.
 
+    Parameters
+    ----------
+    algo_name: str
+    key: str
+    df: pd.DataFrame
+    environ:
+    rel_path: str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)
 
     filename = os.path.join(folder, key + '.csv')
 
-    with open(filename, 'wb') as handle:
-        df.to_csv(handle)
+    with open(filename, 'wt') as handle:
+        df.to_csv(handle, encoding='UTF_8')
 
 
 def get_exchange_minute_writer_root(exchange_name, environ=None):
+    """
+    The minute writer folder for the exchange.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    BcolzExchangeBarWriter
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     minute_data_folder = os.path.join(exchange_folder, 'minute_data')
@@ -163,7 +340,21 @@ def get_exchange_minute_writer_root(exchange_name, environ=None):
 
     return minute_data_folder
 
+
 def get_exchange_bundles_folder(exchange_name, environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    Parameters
+    ----------
+    exchange_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     temp_bundles = os.path.join(exchange_folder, 'temp_bundles')
@@ -173,8 +364,140 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
 
 
 def perf_serial(obj):
-    """JSON serializer for objects not serializable by default json code"""
+    """
+    JSON serializer for objects not serializable by default json code
 
+    Parameters
+    ----------
+    obj: Object
+
+    Returns
+    -------
+    str
+
+    """
     if isinstance(obj, (datetime, date)):
         return obj.isoformat()
+
     raise TypeError("Type %s not serializable" % type(obj))
+
+
+def get_common_assets(exchanges):
+    """
+    The assets available in all specified exchanges.
+
+    Parameters
+    ----------
+    exchanges: list[Exchange]
+
+    Returns
+    -------
+    list[TradingPair]
+
+    """
+    symbols = []
+    for exchange_name in exchanges:
+        s = [asset.symbol for asset in exchanges[exchange_name].get_assets()]
+        symbols.append(s)
+
+    inter_symbols = set.intersection(*map(set, symbols))
+
+    assets = []
+    for symbol in inter_symbols:
+        for exchange_name in exchanges:
+            asset = exchanges[exchange_name].get_asset(symbol)
+            assets.append(asset)
+
+    return assets
+
+
+def get_frequency(freq, data_frequency):
+    """
+    Get the frequency parameters.
+
+    Notes
+    -----
+    We're trying to use Pandas convention for frequency aliases.
+
+    Parameters
+    ----------
+    freq: str
+    data_frequency: str
+
+    Returns
+    -------
+    str, int, str, str
+
+    """
+    if freq == 'minute':
+        unit = 'T'
+        candle_size = 1
+
+    elif freq == 'daily':
+        unit = 'D'
+        candle_size = 1
+
+    else:
+        freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
+        if freq_match:
+            candle_size = int(freq_match.group(1)) if freq_match.group(1) \
+                else 1
+            unit = freq_match.group(2)
+
+        else:
+            raise InvalidHistoryFrequencyError(frequency=freq)
+
+    if unit.lower() == 'd':
+        alias = '{}D'.format(candle_size)
+
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm' or unit == 'T':
+        alias = '{}T'.format(candle_size)
+
+        if data_frequency == 'daily':
+            data_frequency = 'minute'
+
+    # elif unit.lower() == 'h':
+    #     candle_size = candle_size * 60
+    #
+    #     alias = '{}T'.format(candle_size)
+    #     if data_frequency == 'daily':
+    #         data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyAlias(freq=freq)
+
+    return alias, candle_size, unit, data_frequency
+
+
+def resample_history_df(df, freq, field):
+    """
+    Resample the OHCLV DataFrame using the specified frequency.
+
+    Parameters
+    ----------
+    df: DataFrame
+    freq: str
+    field: str
+
+    Returns
+    -------
+    DataFrame
+
+    """
+    if field == 'open':
+        agg = 'first'
+    elif field == 'high':
+        agg = 'max'
+    elif field == 'low':
+        agg = 'min'
+    elif field == 'close':
+        agg = 'last'
+    elif field == 'volume':
+        agg = 'sum'
+    else:
+        raise ValueError('Invalid field.')
+
+    return df.resample(freq).agg(agg)

catalyst/exchange/factory.py

@@ -5,28 +5,39 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 from catalyst.exchange.poloniex.poloniex import Poloniex
 
 
-def get_exchange(exchange_name):
+def get_exchange(exchange_name, base_currency=None):
     exchange_auth = get_exchange_auth(exchange_name)
     if exchange_name == 'bitfinex':
         return Bitfinex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,  # TODO: make optional at the exchange
+            base_currency=base_currency,
             portfolio=None
         )
+
     elif exchange_name == 'bittrex':
         return Bittrex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,
+            base_currency=base_currency,
             portfolio=None
         )
+
     elif exchange_name == 'poloniex':
         return Poloniex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,
+            base_currency=base_currency,
             portfolio=None
         )
+
     else:
         raise ExchangeNotFoundError(exchange_name=exchange_name)
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges

catalyst/exchange/live_graph_clock.py

@@ -1,17 +1,3 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from datetime import timedelta
-
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
@@ -19,11 +5,11 @@ from catalyst.gens.sim_engine import (
 )
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
-
-log = Logger('LiveGraphClock')
+log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
 class LiveGraphClock(object):
@@ -34,8 +20,8 @@ class LiveGraphClock(object):
 
     This mixes the clock with a live graph.
 
-    Note
-    ----
+    Notes
+    -----
     This seemingly awkward approach allows us to run the program using a single
     thread. This is important because Matplotlib does not play nice with
     multi-threaded environments. Zipline probably does not either.
@@ -54,7 +40,7 @@ class LiveGraphClock(object):
 
     def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
 
-        global mdates, plt                  #TODO: Could be cleaner
+        global mdates, plt  # TODO: Could be cleaner
         import matplotlib.dates as mdates
         from matplotlib import pyplot as plt
         from matplotlib import style
@@ -96,11 +82,12 @@ class LiveGraphClock(object):
         """
         Trying to assign reasonable parameters to the time axis.
 
-        TODO: room for improvement
+        Parameters
+        ----------
+        ax:
 
-        :param ax:
-        :return:
         """
+        # TODO: room for improvement
         ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
         ax.xaxis.set_major_formatter(self.fmt)
 
@@ -114,9 +101,21 @@ class LiveGraphClock(object):
         ax.grid(True)
 
     def set_legend(self, ax):
+        """
+        Set legend on the chart.
+
+        Parameters
+        ----------
+        ax
+
+        """
         ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
 
     def draw_pnl(self):
+        """
+        Draw p&l line on the chart.
+
+        """
         ax = self.ax_pnl
         df = self.context.pnl_stats
 
@@ -137,6 +136,10 @@ class LiveGraphClock(object):
         self.format_ax(ax)
 
     def draw_custom_signals(self):
+        """
+        Draw custom signals on the chart.
+
+        """
         ax = self.ax_custom_signals
         df = self.context.custom_signals_stats
 
@@ -155,6 +158,10 @@ class LiveGraphClock(object):
         self.format_ax(ax)
 
     def draw_exposure(self):
+        """
+        Draw exposure line on the chart.
+
+        """
         ax = self.ax_exposure
         context = self.context
         df = context.exposure_stats

catalyst/exchange/poloniex/poloniex.py

@@ -1,44 +1,39 @@
-import base64
-import hashlib
-import hmac
 import json
-import re
+import json
 import time
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 import pytz
-import requests
-# import six
-from six import iteritems
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+# import six
+from six import iteritems
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-
+from catalyst.constants import LOG_LEVEL
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError,
-    OrphanOrderReverseError)
+    InvalidOrderStyle, OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
+    ExchangeStopLimitOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
+    download_exchange_symbols, get_symbols_string
+from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
+from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
+from catalyst.protocol import Account
 
-log = Logger('Poloniex')
+log = Logger('Poloniex', level=LOG_LEVEL)
 
 
 class Poloniex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Poloniex_api(key=key, secret=secret)
         self.name = 'poloniex'
         self.assets = {}
         self.load_assets()
@@ -49,7 +44,7 @@ class Poloniex(Exchange):
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -124,9 +119,9 @@ class Poloniex(Exchange):
         return order, executed_price
 
     def get_balances(self):
-        log.debug('retrieving wallets balances')
+        balances = self.api.returnbalances()
         try:
-            balances = self.api.returnbalances()
+            log.debug('retrieving wallets balances')
         except Exception as e:
             log.debug(e)
             raise ExchangeRequestError(error=e)
@@ -176,12 +171,12 @@ class Poloniex(Exchange):
         # TODO: fetch account data and keep in cache
         return None
 
-    def get_candles(self, data_frequency, assets, bar_count=None,
+    def get_candles(self, freq, assets, bar_count=None,
                     start_dt=None, end_dt=None):
         """
         Retrieve OHLVC candles from Poloniex
 
-        :param data_frequency:
+        :param freq:
         :param assets:
         :param bar_count:
         :return:
@@ -191,25 +186,40 @@ class Poloniex(Exchange):
         '5m', '15m', '30m', '2h', '4h', '1D'
         """
 
-        # TODO: implement end_dt and start_dt filters
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
 
-        if (
-                        data_frequency == '5m' or data_frequency == 'minute'):  # TODO: Polo does not have '1m'
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=freq,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        if freq == '1T' and (bar_count == 1 or bar_count is None):
+            # TODO: use the order book instead
+            # We use the 5m to fetch the last bar
             frequency = 300
-        elif (data_frequency == '15m'):
+        elif freq == '5T':
+            frequency = 300
+        elif freq == '15T':
             frequency = 900
-        elif (data_frequency == '30m'):
+        elif freq == '30T':
             frequency = 1800
-        elif (data_frequency == '2h'):
+        elif freq == '120T':
             frequency = 7200
-        elif (data_frequency == '4h'):
+        elif freq == '240T':
             frequency = 14400
-        elif (data_frequency == '1D' or data_frequency == 'daily'):
+        elif freq == '1D':
             frequency = 86400
         else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
+            # Poloniex does not offer 1m data candles
+            # It is likely to error out there frequently
+            raise InvalidHistoryFrequencyError(frequency=freq)
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
@@ -217,15 +227,18 @@ class Poloniex(Exchange):
 
         for asset in asset_list:
 
+            # TODO: what's wrong with this?
+            # end = int(time.mktime(end_dt.timetuple()))
             end = int(time.time())
-            if (bar_count is None):
+            if bar_count is None:
                 start = end - 2 * frequency
             else:
                 start = end - bar_count * frequency
 
             try:
-                response = self.api.returnchartdata(self.get_symbol(asset),
-                                                    frequency, start, end)
+                response = self.api.returnchartdata(
+                    self.get_symbol(asset), frequency, start, end
+                )
             except Exception as e:
                 raise ExchangeRequestError(error=e)
 

catalyst/exchange/poloniex/poloniex_api.py

@@ -3,6 +3,7 @@ import json
 import time
 import hmac
 import hashlib
+import ssl
 
 from six.moves import urllib
 
@@ -19,19 +20,25 @@ class Poloniex_api(object):
         self.max_requests_per_second = 6
         self.request_cpt = dict()
 
-        self.public  = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                        'returnTradeHistory', 'returnChartData',
-                        'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses',
-                        'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders',
-                        'returnTradeHistory','returnOrderTrades',
+        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
+                       'returnTradeHistory', 'returnChartData',
+                       'returnCurrencies', 'returnLoanOrders']
+        self.trading = ['returnBalances', 'returnCompleteBalances',
+                        'returnDepositAddresses',
+                        'generateNewAddress', 'returnDepositsWithdrawals',
+                        'returnOpenOrders',
+                        'returnTradeHistory', 'returnOrderTrades',
                         'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo','returnAvailableAccountBalances',
+                        'withdraw', 'returnFeeInfo',
+                        'returnAvailableAccountBalances',
                         'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary','marginBuy','marginSell',
-                        'getMarginPosition', 'closeMarginPosition','createLoanOffer',
-                        'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans',
-                        'returnLendingHistory','toggleAutoRenew']
+                        'returnMarginAccountSummary', 'marginBuy',
+                        'marginSell',
+                        'getMarginPosition', 'closeMarginPosition',
+                        'createLoanOffer',
+                        'cancelLoanOffer', 'returnOpenLoanOffers',
+                        'returnActiveLoans',
+                        'returnLendingHistory', 'toggleAutoRenew']
 
     def ask_request(self):
         """
@@ -50,7 +57,7 @@ class Poloniex_api(object):
             self.request_cpt[now] = 0
             return True
 
-        cpt_date = self.request_cpt.keys()[0]
+        cpt_date = list(self.request_cpt.keys())[0]
         cpt = self.request_cpt[cpt_date]
 
         if now > cpt_date + 1:
@@ -59,9 +66,8 @@ class Poloniex_api(object):
             return True
 
         if cpt >= self.max_requests_per_second:
-            
-            log.debug('max requests 6 reached, sleeping for 1 seconds')
-            sleep(1)
+
+            time.sleep(1)
 
             now = time.time()
             self.request_cpt = dict()
@@ -73,22 +79,36 @@ class Poloniex_api(object):
     def query(self, method, req={}):
 
         if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req)
+            url = 'https://poloniex.com/public?command=' + method + '&' + \
+                  urllib.parse.urlencode(req)
             headers = {}
             post_data = None
         elif method in self.trading:
             url = 'https://poloniex.com/tradingApi'
             req['command'] = method
-            req['nonce']   = int(time.time()*1000)
-            post_data      = urllib.parse.urlencode(req)
-            signature      = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
-            headers        = { 'Sign': signature, 'Key': self.key}
+            req['nonce'] = int(time.time() * 1000)
+            post_data = urllib.parse.urlencode(req)
+
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 post_data.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
+            headers = {'Sign': signature, 'Key': self.key}
+
+            post_data = post_data.encode('utf-8')
         else:
-            raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints')
+            raise ValueError(
+                'Method "' + method + '" not found in neither the Public API '
+                                      'or Trading API endpoints'
+            )
 
         self.ask_request()
-        req = urllib.request.Request(url, data=post_data, headers=headers)
-        return json.loads(urlopen(req).read())
+        req = urllib.request.Request(
+            url,
+            data=post_data,
+            headers=headers,
+        )
+        return json.loads(
+            urlopen(req, context=ssl._create_unverified_context()).read())
 
     def returnticker(self):
         return self.query('returnTicker', {})
@@ -100,15 +120,17 @@ class Poloniex_api(object):
         return self.query('returnOrderBook', {'currencyPair': market})
 
     def returntradehistory(self, market, start=None, end=None):
-        if(start is not None and end is not None):
-            return self.query('returntradehistory', 
-                              {'currencyPair': market, 'start': start, 'end': end })
+        if (start is not None and end is not None):
+            return self.query('returntradehistory',
+                              {'currencyPair': market, 'start': start,
+                               'end': end})
         else:
-            return self.query('returntradehistory', {'currencyPair': market })
+            return self.query('returntradehistory', {'currencyPair': market})
 
     def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData', {'currencyPair': market, 'period': period,
-                          'start': start, 'end': end})
+        return self.query('returnChartData',
+                          {'currencyPair': market, 'period': period,
+                           'start': start, 'end': end})
 
     def returncurrencies(self):
         return self.query('returnCurrencies', {})
@@ -120,7 +142,7 @@ class Poloniex_api(object):
         return self.query('returnBalances')
 
     def returncompletebalances(self, account):
-        if(account):
+        if (account):
             return self.query('returnCompleteBalances', {'account': account})
         else:
             return self.query('returnCompleteBalances')
@@ -132,43 +154,54 @@ class Poloniex_api(object):
         return self.query('generateNewAddress', {'currency': currency})
 
     def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals', {'start': start, 'end': end})
+        return self.query('returnDepositsWithdrawals',
+                          {'start': start, 'end': end})
 
     def returnopenorders(self, market):
         return self.query('returnOpenOrders', {'currencyPair': market})
 
     def returntradehistory(self, market):
-        #TODO: optional start and/or end and limit
+        # TODO: optional start and/or end and limit
         return self.query('returnTradeHistory', {'currencyPair': market})
 
     def returnordertrades(self, ordernumber):
         return self.query('returnOrderTrades', {'orderNumber': ordernumber})
 
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
-        if(fillorkill):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
+            postonly=0):
+        if (fillorkill):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'fillOrKill': fillorkill, })
-        elif(immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+        elif (immediateorcancel):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'immediateOrCancel': immediateorcancel, })
-        elif(postonly):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+        elif (postonly):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'postOnly': postonly, })
         else:
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount, })
 
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
-        if(fillorkill):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'fillOrKill': fillorkill, })
-        elif(immediateorcancel):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'immediateOrCancel': immediateorcancel, })
-        elif(postonly):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
-                                      'postOnly': postonly, })
+    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
+             postonly=0):
+        if (fillorkill):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'fillOrKill': fillorkill, })
+        elif (immediateorcancel):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'immediateOrCancel': immediateorcancel, })
+        elif (postonly):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'postOnly': postonly, })
         else:
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount, })
 
     def cancelorder(self, ordernumber):
         return self.query('cancelOrder', {'orderNumber': ordernumber})
@@ -180,4 +213,3 @@ class Poloniex_api(object):
 
     def returnfeeinfo(self):
         return self.query('returnFeeInfo')
-

catalyst/exchange/simple_clock.py

@@ -16,13 +16,13 @@ from time import sleep
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START,
-    MINUTE_END,
-    SESSION_END
+    SESSION_START
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
 class SimpleClock(object):

catalyst/exchange/stats_utils.py

@@ -1,14 +1,127 @@
+import numbers
+
+import numpy as np
 import pandas as pd
 
 
+def trend_direction(series):
+    if series[-1] is np.nan or series[-1] is np.nan:
+        return None
+
+    if series[-1] > series[-2]:
+        return 'up'
+    else:
+        return 'down'
+
+
+def crossover(source, target):
+    """
+    The `x`-series is defined as having crossed over `y`-series if the value
+    of `x` is greater than the value of `y` and the value of `x` was less than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if source[-1] is np.nan or source[-2] is np.nan \
+            or target[-1] is np.nan or target[-2] is np.nan:
+        return False
+
+    if source[-1] > target[-1] and source[-2] < target[-2]:
+        return True
+    else:
+        return False
+
+
+def crossunder(source, target):
+    """
+    The `x`-series is defined as having crossed under `y`-series if the value
+    of `x` is less than the value of `y` and the value of `x` was greater than
+    the value of `y` on the bar immediately preceding the current bar.
+
+    Parameters
+    ----------
+    source: Series
+    target: Series
+
+    Returns
+    -------
+    bool
+
+    """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] < target <= source[-2]:
+            return True
+        else:
+            return False
+    else:
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target[-1] is np.nan or target[-2] is np.nan:
+            return False
+
+        if source[-1] < target[-1] and source[-2] >= target[-2]:
+            return True
+        else:
+            return False
+
+
+def vwap(df):
+    """
+    Volume-weighted average price (VWAP) is a ratio generally used by
+    institutional investors and mutual funds to make buys and sells so as not
+    to disturb the market prices with large orders. It is the average share
+    price of a stock weighted against its trading volume within a particular
+    time frame, generally one day.
+
+    Read more: Volume Weighted Average Price - VWAP
+    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+
+    Returns
+    -------
+
+    """
+    if 'close' not in df.columns or 'volume' not in df.columns:
+        raise ValueError('price data must include `volume` and `close`')
+
+    vol_sum = np.nansum(df['volume'].values)
+
+    try:
+        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
+    except ZeroDivisionError:
+        ret = np.nan
+
+    return ret
+
+
 def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
 
-    :param stats_df:
-    :param num_rows:
-    :return:
+    Parameters
+    ----------
+    stats_df: DataFrame
+    num_rows: int
+
+    Returns
+    -------
+    str
+
     """
     stats_df.set_index('period_close', drop=True, inplace=True)
     stats_df.dropna(axis=1, how='all', inplace=True)
@@ -49,3 +162,49 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
         columns=columns,
         formatters=formatters
     )
+
+
+def df_to_string(df):
+    """
+    Create a formatted str representation of the DataFrame.
+
+    Parameters
+    ----------
+    df: DataFrame
+
+    Returns
+    -------
+    str
+
+    """
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+    return df.to_string()
+
+
+def extract_transactions(perf):
+    """
+    Compute indexes for buy and sell transactions
+
+    Parameters
+    ----------
+    perf: DataFrame
+        The algo performance DataFrame.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of transactions.
+
+    """
+    trans_list = perf.transactions.values
+    all_trans = [t for sublist in trans_list for t in sublist]
+    all_trans.sort(key=lambda t: t['dt'])
+
+    transactions = pd.DataFrame(all_trans)
+    if not transactions.empty:
+        transactions.set_index('dt', inplace=True, drop=True)
+    return transactions

catalyst/finance/blotter.py

@@ -34,7 +34,9 @@ from catalyst.finance.commission import (
 from catalyst.finance.cancel_policy import NeverCancel
 from catalyst.utils.input_validation import expect_types
 
-log = Logger('Blotter')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Blotter', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/finance/controls.py

@@ -24,7 +24,9 @@ from catalyst.errors import (
     TradingControlViolation,
 )
 
-log = logbook.Logger('TradingControl')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('TradingControl', level=LOG_LEVEL)
 
 
 class TradingControl(with_metaclass(abc.ABCMeta)):

catalyst/finance/execution.py

@@ -77,6 +77,7 @@ class LimitOrder(ExecutionStyle):
     Execution style representing an order to be executed at a price equal to or
     better than a specified limit price.
     """
+
     def __init__(self, limit_price, exchange=None):
         """
         Store the given price.
@@ -99,6 +100,7 @@ class StopOrder(ExecutionStyle):
     Execution style representing an order to be placed once the market price
     reaches a specified stop price.
     """
+
     def __init__(self, stop_price, exchange=None):
         """
         Store the given price.
@@ -121,6 +123,7 @@ class StopLimitOrder(ExecutionStyle):
     Execution style representing a limit order to be placed with a specified
     limit price once the market reaches a specified stop price.
     """
+
     def __init__(self, limit_price, stop_price, exchange=None):
         """
         Store the given prices
@@ -144,31 +147,20 @@ class StopLimitOrder(ExecutionStyle):
 def asymmetric_round_price_to_penny(price, prefer_round_down,
                                     diff=(0.0095 - .005)):
     """
-    Asymmetric rounding function for adjusting prices to two places in a way
-    that "improves" the price.  For limit prices, this means preferring to
-    round down on buys and preferring to round up on sells.  For stop prices,
-    it means the reverse.
+    Modified the original function because we do not want to round
+    prices on crypto exchange.
 
-    If prefer_round_down == True:
-        When .05 below to .95 above a penny, use that penny.
-    If prefer_round_down == False:
-        When .95 below to .05 above a penny, use that penny.
+    Parameters
+    ----------
+    price: float
+
+    Returns
+    -------
+    float
 
-    In math-speak:
-    If prefer_round_down: [<X-1>.0095, X.0195) -> round to X.01.
-    If not prefer_round_down: (<X-1>.0005, X.0105] -> round to X.01.
     """
-    # Subtracting an epsilon from diff to enforce the open-ness of the upper
-    # bound on buys and the lower bound on sells.  Using the actual system
-    # epsilon doesn't quite get there, so use a slightly less epsilon-ey value.
-    epsilon = float_info.epsilon * 10
-    diff = diff - epsilon
-
-    # relies on rounding half away from zero, unlike numpy's bankers' rounding
-    rounded = round(price - (diff if prefer_round_down else -diff), 2)
-    if zp_math.tolerant_equals(rounded, 0.0):
-        return 0.0
-    return rounded
+    # TODO: consider overriding outside of the original function
+    return price
 
 
 def check_stoplimit_prices(price, label):

catalyst/finance/performance/period.py

@@ -88,7 +88,10 @@ from six import itervalues, iteritems
 
 import catalyst.protocol as zp
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
+
 TRADE_TYPE = zp.DATASOURCE_TYPE.TRADE
 
 

catalyst/finance/performance/position.py

@@ -40,7 +40,9 @@ import logbook
 from catalyst.assets import Future, Asset
 from catalyst.utils.input_validation import expect_types
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class Position(object):

catalyst/finance/performance/position_tracker.py

@@ -32,7 +32,9 @@ from catalyst.assets import (
 )
 from . position import positiondict
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 PositionStats = namedtuple('PositionStats',

catalyst/finance/performance/tracker.py

@@ -70,7 +70,9 @@ import catalyst.finance.risk as risk
 
 from . position_tracker import PositionTracker
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class PerformanceTracker(object):

catalyst/finance/risk/cumulative.py

@@ -22,7 +22,7 @@ from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
-from . risk import (
+from .risk import (
     check_entry,
     choose_treasury
 )
@@ -37,13 +37,16 @@ from empyrical import (
     sharpe_ratio,
     sortino_ratio,
 )
+import warnings
+from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger('Risk Cumulative')
-
+log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
 
+warnings.filterwarnings('error')
+
 
 class RiskMetricsCumulative(object):
     """
@@ -189,9 +192,12 @@ class RiskMetricsCumulative(object):
             if len(self.benchmark_returns) == 1:
                 self.benchmark_returns = np.append(0.0, self.benchmark_returns)
 
-        self.benchmark_cumulative_returns[dt_loc] = cum_returns(
-            self.benchmark_returns
-        )[-1]
+        try:
+            self.benchmark_cumulative_returns[dt_loc] = cum_returns(
+                self.benchmark_returns
+            )[-1]
+        except Exception as e:
+            log.debug('cumulative returns error: {}'.format(e))
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -266,10 +272,15 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.downside_risk[dt_loc] = downside_risk(
             self.algorithm_returns
         )
-        self.sortino[dt_loc] = sortino_ratio(
-            self.algorithm_returns,
-            _downside_risk=self.downside_risk[dt_loc]
-        )
+
+        try:
+            self.sortino[dt_loc] = sortino_ratio(
+                self.algorithm_returns,
+                _downside_risk=self.downside_risk[dt_loc]
+            )
+        except Exception as e:
+            log.debug('sortino ratio error: {}'.format(e))
+
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
@@ -292,18 +303,18 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         rval = {
             'trading_days': self.num_trading_days,
             'benchmark_volatility':
-            self.benchmark_volatility[dt_loc],
+                self.benchmark_volatility[dt_loc],
             'algo_volatility':
-            self.algorithm_volatility[dt_loc],
+                self.algorithm_volatility[dt_loc],
             'treasury_period_return': self.treasury_period_return,
             # Though the two following keys say period return,
             # they would be more accurately called the cumulative return.
             # However, the keys need to stay the same, for now, for backwards
             # compatibility with existing consumers.
             'algorithm_period_return':
-            self.algorithm_cumulative_returns[dt_loc],
+                self.algorithm_cumulative_returns[dt_loc],
             'benchmark_period_return':
-            self.benchmark_cumulative_returns[dt_loc],
+                self.benchmark_cumulative_returns[dt_loc],
             'beta': self.beta[dt_loc],
             'alpha': self.alpha[dt_loc],
             'sharpe': self.sharpe[dt_loc],

catalyst/finance/risk/period.py

@@ -36,7 +36,9 @@ from empyrical import (
     sortino_ratio
 )
 
-log = logbook.Logger('Risk Period')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Period', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(risk.choose_treasury,
                                     risk.select_treasury_duration)

catalyst/finance/risk/report.py

@@ -63,7 +63,9 @@ from dateutil.relativedelta import relativedelta
 
 from . period import RiskMetricsPeriod
 
-log = logbook.Logger('Risk Report')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Report', level=LOG_LEVEL)
 
 
 class RiskReport(object):

catalyst/finance/risk/risk.py

@@ -61,7 +61,9 @@ Risk Report
 import logbook
 import numpy as np
 
-log = logbook.Logger('Risk')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk', level=LOG_LEVEL)
 
 
 TREASURY_DURATIONS = [

catalyst/finance/trading.py

@@ -26,7 +26,9 @@ from catalyst.data.loader import load_market_data
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.memoize import remember_last
 
-log = logbook.Logger('Trading')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Trading', level=LOG_LEVEL)
 
 
 DEFAULT_CAPITAL_BASE = 1e5

catalyst/gens/tradesimulation.py

@@ -27,7 +27,9 @@ from catalyst.gens.sim_engine import (
     BEFORE_TRADING_START_BAR
 )
 
-log = Logger('Trade Simulation')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Trade Simulation', level=LOG_LEVEL)
 
 
 class AlgorithmSimulator(object):

catalyst/sources/benchmark_source.py

@@ -72,7 +72,13 @@ class BenchmarkSource(object):
                             "benchmark_returns.")
 
     def get_value(self, dt):
-        return self._precalculated_series.loc[dt]
+        try:
+            series = self._precalculated_series
+            value = series.loc[dt]
+            return value
+        except Exception:
+            # TODO: workaround, find permanent fix
+            return 0
 
     def get_range(self, start_dt, end_dt):
         return self._precalculated_series.loc[start_dt:end_dt]

catalyst/sources/requests_csv.py

@@ -23,7 +23,9 @@ from catalyst.protocol import (
 )
 from catalyst.assets import Equity
 
-logger = Logger('Requests Source Logger')
+from catalyst.constants import LOG_LEVEL
+
+logger = Logger('Requests Source Logger', level=LOG_LEVEL)
 
 
 def roll_dts_to_midnight(dts, trading_day):

catalyst/support/issue_44.py

@@ -0,0 +1,109 @@
+import pandas as pd
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1
+    context.base_currency = 'btc'
+
+
+def handle_data(context, data):
+    lookback = 60 * 24 * 7  # (minutes, hours, days)
+    context.i += 1
+    if context.i < lookback:
+        return
+
+    today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
+
+    try:
+        # update universe everyday
+        new_day = 60 * 24
+        if not context.i % new_day:
+            context.universe = universe(context, today)
+
+        # get data every 30 minutes
+        minutes = 30
+        if not context.i % minutes and context.universe:
+            for coin in context.coins:
+                pair = str(coin.symbol)
+
+                # ohlcv data
+                open = data.history(coin, 'open', lookback,
+                                    '1m').ffill().bfill().resample(
+                    '30T').first()
+                high = data.history(coin, 'high', lookback,
+                                    '1m').ffill().bfill().resample('30T').max()
+                low = data.history(coin, 'low', lookback,
+                                   '1m').ffill().bfill().resample('30T').min()
+                close = data.history(coin, 'price', lookback,
+                                     '1m').ffill().bfill().resample(
+                    '30T').last()
+                volume = data.history(coin, 'volume', lookback,
+                                      '1m').ffill().bfill().resample(
+                    '30T').sum()
+
+                print(today, pair, close[-1])
+
+    except Exception as e:
+        print(e)
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+def universe(context, today):
+    json_symbols = get_exchange_symbols('poloniex')
+    poloniex_universe_df = pd.DataFrame.from_dict(
+        json_symbols).transpose().astype(str)
+    poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
+        lambda row: row.symbol.split('_')[1],
+        axis=1)
+    poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
+        lambda row: row.symbol.split('_')[0],
+        axis=1)
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df['base_currency'] == context.base_currency]
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df.symbol != 'gas_btc']
+
+    # Markets currently not working on Catalyst 0.3.1
+    # 2017-01-01
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
+    print(poloniex_universe_df.head())
+
+    date = str(today).split(' ')[0]
+
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df.start_date < date]
+    context.coins = symbols(*poloniex_universe_df.symbol)
+    print(len(poloniex_universe_df))
+    return poloniex_universe_df.symbol.tolist()
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-01-01', utc=True)
+    end_date = pd.to_datetime('2017-10-15', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=10000.0,
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='test')

catalyst/support/issue_47.py

@@ -0,0 +1,140 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.2
+
+These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
+You simply need to specify the exchange and the market that you want to focus on.
+You will all see how to create a universe and filter it base on the exchange and the market you desire.
+
+The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
+The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
+Use this as the backbone to create your own trading strategies.
+
+Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
+"""
+
+import numpy as np
+import pandas as pd
+from datetime import timedelta
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = 'poloniex'  # must match the exchange specified in run_algorithm
+    context.base_currency = 'eth'  # must match the base currency specified in run_algorithm
+
+
+def handle_data(context, data):
+    lookback = 60 * 24 * 7  # (minutes, hours, days) of how far to lookback in the data history
+    context.i += 1
+
+    # current date formatted into a string
+    today = context.blotter.current_dt
+    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = today - timedelta(days=(
+    lookback / (60 * 24)))  # subtract the amount of days specified in lookback
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[
+        0]  # get only the date as a string
+
+    # update universe everyday
+    new_day = 60 * 24
+    if not context.i % new_day:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
+            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
+            open = fill(data.history(coin, 'open', bar_count=lookback,
+                                     frequency='1m')).resample('30T').first()
+            high = fill(data.history(coin, 'high', bar_count=lookback,
+                                     frequency='1m')).resample('30T').max()
+            low = fill(data.history(coin, 'low', bar_count=lookback,
+                                    frequency='1m')).resample('30T').min()
+            close = fill(data.history(coin, 'price', bar_count=lookback,
+                                      frequency='1m')).resample('30T').last()
+            volume = fill(data.history(coin, 'volume', bar_count=lookback,
+                                       frequency='1m')).resample('30T').sum()
+
+            # close[-1] is the equivalent to current price
+            # displays the minute price for each pair every 30 minutes
+            print(
+            today, pair, open[-1], high[-1], low[-1], close[-1], volume[-1])
+
+            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
+            # ----------------------------------------------------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    json_symbols = get_exchange_symbols(
+        context.exchange)  # get all the pairs for the exchange
+    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(
+        str)  # convert into a dataframe
+    universe_df['base_currency'] = universe_df.apply(
+        lambda row: row.symbol.split('_')[1],
+        axis=1)
+    universe_df['market_currency'] = universe_df.apply(
+        lambda row: row.symbol.split('_')[0],
+        axis=1)
+    # Filter all the exchange pairs to only the ones for a give base currency
+    universe_df = universe_df[
+        universe_df['base_currency'] == context.base_currency]
+
+    # Filter all the pairs to ensure that pair existed in the current date range
+    universe_df = universe_df[universe_df.start_date < lookback_date]
+    universe_df = universe_df[universe_df.end_daily >= current_date]
+    context.coins = symbols(
+        *universe_df.symbol)  # convert all the pairs to symbols
+    print(universe_df.head(), len(universe_df))
+    return universe_df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace([np.inf, -np.inf],
+                                         np.nan).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-01-01', utc=True)
+    end_date = pd.to_datetime('2017-10-15', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=10000.0,
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='eth',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')
+
+"""
+Run in Terminal (inside catalyst environment):
+python simple_universe.py
+"""

catalyst/support/issue_55.py

@@ -0,0 +1,42 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('xcp_btc')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=1,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2015-3-2', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='poloniex',
+    algo_namespace='issue_55',
+    base_currency='btc'
+)

catalyst/support/issue_57.py

@@ -0,0 +1,46 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=60,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2016-2-11', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='bittrex',
+    algo_namespace='issue_57',
+    base_currency='btc'
+<<<<<<< HEAD
+)
+=======
+)
+>>>>>>> develop

catalyst/support/rodrigo_1.py

@@ -0,0 +1,153 @@
+import pandas as pd
+from logbook import Logger, DEBUG
+
+from catalyst import run_algorithm
+from catalyst.api import (schedule_function, order_target_percent, symbol,
+                          date_rules, get_open_orders, cancel_order, record,
+                          set_commission, set_slippage)
+
+log = Logger('rodrigo_1', level=DEBUG)
+"""
+The initialize function sets any data or variables that 
+you'll use in your algorithm. 
+It's only called once at the beginning of your algorithm.
+"""
+
+
+def initialize(context):
+    # Select asset of interest
+    context.asset = symbol('BTC_USD')
+
+    # set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5))
+    # set_slippage(TradingPairFixedSlippage(spread=0.5))
+    # Set up a rebalance method to run every day
+    schedule_function(rebalance, date_rule=date_rules.every_day())
+
+
+"""
+Rebalance function scheduled to run once per day.
+"""
+
+
+def rebalance(context, data):
+    # To make market decisions, we're calculating the token's
+    # moving average for the last 5 days.
+
+    # We get the price history for the last 5 days.
+    price_history = data.history(context.asset, fields='price', bar_count=5,
+                                 frequency='1d')
+
+    # Then we take an average of those 5 days.
+    average_price = price_history.mean()
+
+    # We also get the coin's current price.
+    price = data.current(context.asset, 'price')
+
+    # Cancel any outstanding orders
+    orders = get_open_orders(context.asset) or []
+    for order in orders:
+        cancel_order(order)
+
+    # If our coin is currently listed on a major exchange
+    if data.can_trade(context.asset):
+        # If the current price is 1% above the 5-day average price,
+        # we open a long position. If the current price is below the
+        # average price, then we want to close our position to 0 shares.
+        if price > (1.01 * average_price):
+            # Place the buy order (positive means buy, negative means sell)
+            order_target_percent(context.asset, .99)
+            log.info("Buying %s" % (context.asset.symbol))
+        elif price < average_price:
+            # Sell all of our shares by setting the target position to zero
+            order_target_percent(context.asset, 0)
+            log.info("Selling %s" % (context.asset.symbol))
+
+    # Use the record() method to track up to five custom signals.
+    # Record Apple's current price and the average price over the last
+    # five days.
+    cash = context.portfolio.cash
+    leverage = context.account.leverage
+
+    record(price=price, average_price=average_price, cash=cash,
+           leverage=leverage)
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(511)
+    results[['portfolio_value']].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value (USD)')
+
+    ax2 = plt.subplot(512, sharex=ax1)
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset))
+    (results[[
+        'price',
+    ]]).plot(ax=ax2)
+
+    trans = results.ix[[t != [] for t in results.transactions]]
+    buys = trans.ix[
+        [t[0]['amount'] > 0 for t in trans.transactions]
+    ]
+    sells = trans.ix[
+        [t[0]['amount'] < 0 for t in trans.transactions]
+    ]
+
+    ax2.plot(
+        buys.index,
+        results.price[buys.index],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        results.price[sells.index],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(513, sharex=ax1)
+    results[['leverage']].plot(ax=ax3)
+    ax3.set_ylabel('Leverage ')
+
+    ax4 = plt.subplot(514, sharex=ax1)
+    results[['cash']].plot(ax=ax4)
+    ax4.set_ylabel('Cash (USD)')
+
+    results[[
+        'algorithm',
+        'benchmark',
+    ]] = results[[
+        'algorithm_period_return',
+        'benchmark_period_return',
+    ]]
+
+    ax5 = plt.subplot(515, sharex=ax1)
+    results[[
+        'algorithm',
+        'benchmark',
+    ]].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+
+
+run_algorithm(
+    capital_base=100000,
+    start=pd.to_datetime('2017-1-1', utc=True),
+    end=pd.to_datetime('2017-10-22', utc=True),
+    data_frequency='minute',
+    initialize=initialize,
+    handle_data=None,
+    analyze=analyze,
+    exchange_name='bitfinex',
+    algo_namespace='rodrigo_1',
+    base_currency='usd'
+)

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,4 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-02-19', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/paths.py

@@ -126,7 +126,7 @@ def catalyst_root(environ=None):
 
     root = environ.get('ZIPLINE_ROOT', None)
     if root is None:
-        root = expanduser('~/.catalyst')
+        root = os.path.join(expanduser('~'),'.catalyst')
 
     return root
 

catalyst/utils/run_algo.py

@@ -31,19 +31,21 @@ import catalyst.utils.paths as pth
 
 from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
     ExchangeTradingAlgorithmBacktest
-from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
+    ExchangeRequestError, ExchangeAuthEmpty,
     ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, ExchangeNotFoundError)
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
+    get_algo_object, get_exchange_folder
 from logbook import Logger
 
-log = Logger('run_algo')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('run_algo', level=LOG_LEVEL)
 
 
 class _RunAlgoError(click.ClickException, ValueError):
@@ -164,6 +166,12 @@ def _run(handle_data,
 
         # This corresponds to the json file containing api token info
         exchange_auth = get_exchange_auth(exchange_name)
+
+        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
+            raise ExchangeAuthEmpty(
+                    exchange=exchange_name.title(), 
+                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
+
         if exchange_name == 'bitfinex':
             exchanges[exchange_name] = Bitfinex(
                 key=exchange_auth['key'],
@@ -191,7 +199,12 @@ def _run(handle_data,
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
-        load=partial(load_crypto_market_data, environ=environ),
+        load=partial(
+            load_crypto_market_data,
+            environ=environ,
+            start_dt=start,
+            end_dt=end
+        ),
         environ=environ,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
@@ -230,8 +243,11 @@ def _run(handle_data,
                 balances = exchange.get_balances()
             except ExchangeRequestError as e:
                 if attempt_index < 20:
-                    log.warn('exchange error when retrieving balances, {} '
-                             'trying again in 5 seconds'.format(e))
+                    log.warn(
+                        'could not retrieve balances on {}: {}'.format(
+                            exchange.name, e
+                        )
+                    )
                     sleep(5)
                     return fetch_capital_base(exchange, attempt_index + 1)
 
@@ -284,7 +300,8 @@ def _run(handle_data,
             exchanges=exchanges,
             asset_finder=None,
             trading_calendar=open_calendar,
-            first_trading_day=None,
+            first_trading_day=start,
+            last_available_session=end
         )
 
         sim_params = create_simulation_parameters(

docs/live-trading-wiki.md

@@ -1,105 +0,0 @@
-<h1>Live Trading</h1>
-This document explains how to get started with live trading.
-
-<h2>Supported Exchanges</h2>
-Catalyst can trade against these exchanges:
-
-* Bitfinex, id=`bitfinex`
-* Bittrex, id=`bittrex`
-
-<h3>Authentication</h3>
-Most exchanges require key/token combination for authentication. By
-convention, Catalyst uses an "auth.json" file to hold this data.
-
-This example illustrates the convention using the Bitfinex exchange.
-Here is how to generate key and secret values for bitfinex:
-https://docs.bitfinex.com/v1/docs/api-access. Most exchanges follow
-a similar process.
-
-The auth.json file:
-```json
-{
-  "name": "bitfinex",
-  "key": "my-key",
-  "secret": "my-secret"
-}
-```
-
-The file goes here:
-```
-~/.catalyst/data/exchanges/bitfinex/auth.json
-```
-
-Note that the 'bitfinex' directory corresponds to the id of the Bitfinex
-exchange as defined in the "Supported Exchanges" section above.
-Attempting to run an algorithm where the targeted exchange is missing
-its "auth.json" file will create the directory structure but result
-in an error.
-
-<h3>Currency Symbols</h3>
-Catalyst introduces a universal convention to reference
-trading pairs and individual currencies. This
-is required to ensure that the `symbol()` api predictably
-returns the correct asset regardless of the targeted exchange.
-
-Exchanges tend to use their own convention to represent currencies
-(e.g. XBT and BTC both represent Bitcoin on different exchanges).
-Trading pairs are also inconsistent. For example, Bitfinex
-puts the market currency before the base currency without a
-separator, Bittrex puts the base currency first and uses a dash
-seperator.
-
-Here is the Catalyst convention:
-
-*[Market Currency]_[Base Currency]* all lowercase.
-
-Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
-
-Here are some examples:
-```python
-# With Bitfinex
-bitcoin_usd_asset = symbol('btc_usd')
-ethereum_bitcoin_asset = symbol('eth_btc')
-
-# With Bittrex
-ethereum_bitcoin_asset = symbol('eth_btc')
-neo_ethereum_asset = symbol('neo_eth)
-```
-
-Note that the trading pairs are always referenced in the same manner.
-However, not all trading pairs are available on all exchanges. An
-error will occur if the specified trading pair is not trading
-on the exchange.
-
-<h2>Trading an Algorithm</h2>
-There is no special convention to follow when writing an
-algorithm for live trading. The same algorithm should work in
-backtest and live execution mode without modification.
-
-What differs are the arguments provided to the catalyst client or
-`run_algorithm()` interface. Here is example:
-
-```python
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace='my_algo_trading_xrp',
-    base_currency='btc'
-)
-```
-
-Here is the breakdown of the new arguments:
-* live: Boolean flag which enables live trading.
-* exchange_name: The name of the targeted exchange
-    (supported values: *bitfinex*, *bittrex*).
-* algo_namespace: A arbitrary label assigned to your algorithm for
-    data storage purposes.
-* base_currency: The base currency used to calculate the
-    statistics of your algorithm. Currently, the base currency of all
-    trading pairs of your algorithm must match this value.
-
-Here is a complete algorithm for reference:
-[Buy Low and Sell High](../catalyst/examples/buy_low_sell_high_live.py)

docs/source/conf.py

@@ -41,7 +41,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC'
+copyright = u'2017, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
@@ -94,6 +94,6 @@ intersphinx_mapping = {
     'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None),
 }
 
-doctest_global_setup = "import zipline"
+doctest_global_setup = "import catalyst"
 
 todo_include_todos = True

docs/source/development-guidelines.rst

@@ -1,21 +1,17 @@
 Development Guidelines
 ======================
-This page is intended for developers of Zipline, people who want to contribute to the Zipline codebase or documentation, or people who want to install from source and make local changes to their copy of Zipline.
+This page is intended for developers of Catalyst, people who want to contribute to the Catalyst codebase or documentation, or people who want to install from source and make local changes to their copy of Catalyst.
 
-All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We `track issues`__ on `GitHub`__ and also have a `mailing list`__ where you can ask questions.
-
-__ https://github.com/quantopian/zipline/issues
-__ https://github.com/
-__ https://groups.google.com/forum/#!forum/zipline
+All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We `track issues <https://github.com/enigmampc/catalyst/issues>`_ on `GitHub <https://github.com/enigmampc/catalyst>`_ and also have a `discord group <https://discord.gg/SJK32GY>`_ where you can ask questions.
 
 Creating a Development Environment
 ----------------------------------
 
-First, you'll need to clone Zipline by running:
+First, you'll need to clone Catalyst by running:
 
 .. code-block:: bash
 
-   $ git clone git@github.com:your-github-username/zipline.git
+   $ git clone git@github.com:enigmampc/catalyst.git
 
 Then check out to a new branch where you can make your changes:
 
@@ -23,15 +19,13 @@ Then check out to a new branch where you can make your changes:
 		
    $ git checkout -b some-short-descriptive-name
 
-If you don't already have them, you'll need some C library dependencies. You can follow the `install guide`__ to get the appropriate dependencies.
-
-__ install.html
+If you don't already have them, you'll need some C library dependencies. You can follow the `install guide <install.html>`_ to get the appropriate dependencies.
 
 The following section assumes you already have virtualenvwrapper and pip installed on your system. Suggested installation of Python library dependencies used for development:
 
 .. code-block:: bash
 
-   $ mkvirtualenv zipline
+   $ mkvirtualenv catalyst
    $ ./etc/ordered_pip.sh ./etc/requirements.txt
    $ pip install -r ./etc/requirements_dev.txt
    $ pip install -r ./etc/requirements_blaze.txt 
@@ -42,104 +36,39 @@ Finally, you can build the C extensions by running:
 
    $ python setup.py build_ext --inplace
 
-To finish, make sure `tests`__ pass.
+.. To finish, make sure `tests`__ pass.
 
-__ #style-guide-running-tests
+.. __ #style-guide-running-tests
 
-If you get an error running nosetests after setting up a fresh virtualenv, please try running
+.. If you get an error running nosetests after setting up a fresh virtualenv, please try running
 
-.. code-block:: bash
+.. code-block
 
-   # where zipline is the name of your virtualenv
-   $ deactivate zipline
-   $ workon zipline
+..   # where zipline is the name of your virtualenv
+..   $ deactivate zipline
+..   $ workon zipline
 
 
-Development with Docker
+.. Development with Docker
+.. -----------------------
+
+..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
+
+.. __ https://docs.docker.com/get-started/
+   
+Git Branching Structure
 -----------------------
 
-If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
+If you want to contribute to the codebase of Catalyst, familiarize yourself with our branching structure, a fairly standardized one for that matter, that follows what is documented in the following article: `A successful Git branching model <http://nvie.com/posts/a-successful-git-branching-model/>`_. To contribute, create your local branch and submit a Pull Request (PR) to the **develop** branch.
 
-__ https://docs.docker.com/get-started/
+.. image:: https://camo.githubusercontent.com/9bde6fb64a9542a572e0e2017cbb58d9d2c440ac/687474703a2f2f6e7669652e636f6d2f696d672f6769742d6d6f64656c4032782e706e67
 
-
-Style Guide & Running Tests
----------------------------
-
-We use `flake8`__ for checking style requirements and `nosetests`__ to run Zipline tests. Our `continuous integration`__ tools will run these commands.
-
-__ http://flake8.pycqa.org/en/latest/
-__ http://nose.readthedocs.io/en/latest/
-__ https://en.wikipedia.org/wiki/Continuous_integration
-
-Before submitting patches or pull requests, please ensure that your changes pass when running:
-
-.. code-block:: bash
-
-   $ flake8 zipline tests
-
-In order to run tests locally, you'll need `TA-lib`__, which you can install on Linux by running:
-
-__ https://mrjbq7.github.io/ta-lib/install.html
-
-.. code-block:: bash
-
-   $ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
-   $ tar -xvzf ta-lib-0.4.0-src.tar.gz
-   $ cd ta-lib/
-   $ ./configure --prefix=/usr
-   $ make
-   $ sudo make install
-
-And for ``TA-lib`` on OS X you can just run:
-
-.. code-block:: bash
-
-   $ brew install ta-lib
-
-Then run ``pip install`` TA-lib:
-
-.. code-block:: bash
-
-   $ pip install -r ./etc/requirements_talib.txt
-
-You should now be free to run tests:
-
-.. code-block:: bash
-		
-   $ nosetests
-
-
-Continuous Integration
-----------------------
-
-We use `Travis CI`__ for Linux-64 bit builds and `AppVeyor`__ for Windows-64 bit builds.
-
-.. note::
-
-   We do not currently have CI for OSX-64 bit builds. 32-bit builds may work but are not included in our integration tests.
-
-__ https://travis-ci.org/quantopian/zipline
-__ https://ci.appveyor.com/project/quantopian/zipline
-
-
-Packaging
----------
-To learn about how we build Zipline conda packages, you can read `this`__ section in our release process notes.
-
-__ release-process.html#uploading-conda-packages
-   
 Contributing to the Docs
 ------------------------
 
-If you'd like to contribute to the documentation on zipline.io, you can navigate to ``docs/source/`` where each `reStructuredText`__ (``.rst``) file is a separate section there. To add a section, create a new file called ``some-descriptive-name.rst`` and add ``some-descriptive-name`` to ``appendix.rst``. To edit a section, simply open up one of the existing files, make your changes, and save them.
-
-__ https://en.wikipedia.org/wiki/ReStructuredText
-
-We use `Sphinx`__ to generate documentation for Zipline, which you will need to install by running:
-
-__ http://www.sphinx-doc.org/en/stable/
+If you'd like to contribute to the documentation on enigmampc.github.io, you can navigate to ``docs/source/`` where each `reStructuredText <https://en.wikipedia.org/wiki/ReStructuredText>`_ file is a separate section there. To add a section, create a new file called ``some-descriptive-name.rst`` and add ``some-descriptive-name`` to ``index.rst``. To edit a section, simply open up one of the existing files, make your changes, and save them.
 
+We use `Sphinx <http://www.sphinx-doc.org/en/stable/>`_ to generate documentation for Catalyst, which you will need to install by running:
 
 .. code-block:: bash
 
@@ -149,7 +78,7 @@ To build and view the docs locally, run:
 
 .. code-block:: bash
 
-   # assuming you're in the Zipline root directory
+   # assuming you're in the Catalyst root directory
    $ cd docs
    $ make html
    $ {BROWSER} build/html/index.html
@@ -162,7 +91,7 @@ Standard prefixes to start a commit message:
 
 .. code-block:: text
 
-   BLD: change related to building Zipline
+   BLD: change related to building Catalyst
    BUG: bug fix
    DEP: deprecate something, or remove a deprecated object
    DEV: development tool or utility
@@ -172,15 +101,13 @@ Standard prefixes to start a commit message:
    REV: revert an earlier commit
    STY: style fix (whitespace, PEP8, flake8, etc)
    TST: addition or modification of tests
-   REL: related to releasing Zipline
+   REL: related to releasing Catalyst
    PERF: performance enhancements
 
 
 Some commit style guidelines:
 
-Commit lines should be no longer than `72 characters`__. The first line of the commit should include one of the above prefixes. There should be an empty line between the commit subject and the body of the commit. In general, the message should be in the imperative tense. Best practice is to include not only what the change is, but why the change was made.
-
-__ https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project
+Commit lines should be no longer than `72 characters <https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project>`_. The first line of the commit should include one of the above prefixes. There should be an empty line between the commit subject and the body of the commit. In general, the message should be in the imperative tense. Best practice is to include not only what the change is, but why the change was made.
 
 **Example:**
 
@@ -203,8 +130,6 @@ __ https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project
 Formatting Docstrings
 ---------------------
 
-When adding or editing docstrings for classes, functions, etc, we use `numpy`__ as the canonical reference.
-
-__ https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
+When adding or editing docstrings for classes, functions, etc, we use `numpy <https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_ as the canonical reference.
 
 

docs/source/index.rst

@@ -1,12 +1,23 @@
-.. include:: ../../README.rst
+.. include:: welcome.rst
+|
+|
+Table of Contents
+-----------------
 
 .. toctree::
    :maxdepth: 1
 
    install
    beginner-tutorial
-   bundles
+   jupyter
+   live-trading
+   naming-convention
+   videos
+   resources
    development-guidelines
-   appendix
-   release-process
    releases
+..   bundles
+..   development-guidelines
+..   appendix
+..   release-process
+

docs/source/install.rst

@@ -1,40 +1,65 @@
 Install
 =======
 
+To get started with Catalyst, you will need to install it in your computer. 
+Like any other piece of software, Catalyst has a number of dependencies 
+(other software on which it depends to run) that you will need to install, as 
+well. We recommend using a software named ``Conda`` that will manage all 
+these dependencies for you, and set up the environment needed to get you up 
+and running as easily as possible. See :ref:`Installing with Conda <conda>`.
+
 Installing with ``pip``
 -----------------------
 
-Installing Zipline via ``pip`` is slightly more involved than the average
+Installing Catalyst via ``pip`` is slightly more involved than the average
 Python package.
 
 There are two reasons for the additional complexity:
 
-1. Zipline ships several C extensions that require access to the CPython C API.
-   In order to build the C extensions, ``pip`` needs access to the CPython
-   header files for your Python installation.
+1. Catalyst ships several C extensions that require access to the CPython C 
+   API. In order to build the C extensions, ``pip`` needs access to the 
+   CPython header files for your Python installation.
 
-2. Zipline depends on `numpy <http://www.numpy.org/>`_, the core library for
+2. Catalyst depends on `numpy <http://www.numpy.org/>`_, the core library for
    numerical array computing in Python.  Numpy depends on having the `LAPACK
    <http://www.netlib.org/lapack>`_ linear algebra routines available.
 
-Because LAPACK and the CPython headers are non-Python dependencies, the correct
-way to install them varies from platform to platform.  If you'd rather use a
-single tool to install Python and non-Python dependencies, or if you're already
-using `Anaconda <http://continuum.io/downloads>`_ as your Python distribution,
-you can skip to the :ref:`Installing with Conda <conda>` section.
+Because LAPACK and the CPython headers are non-Python dependencies, the 
+correctway to install them varies from platform to platform.  If you'd rather 
+use a single tool to install Python and non-Python dependencies, or if you're 
+already using `Anaconda <http://continuum.io/downloads>`_ as your Python 
+distribution, you can skip to the :ref:`Installing with Conda <conda>` 
+section.
 
 Once you've installed the necessary additional dependencies (see below for
 your particular platform), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install zipline
+   $ pip install enigma-catalyst
 
-If you use Python for anything other than Zipline, we **strongly** recommend
+If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
 Python`_ provides an `excellent tutorial on virtualenv
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_.
+<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a 
+summarized version:
+
+.. code-block:: bash
+
+   $ pip install virtualenv
+   $ virtualenv catalyst-venv
+   $ source ./catalyst-venv/bin/activate
+   $ pip install enigma-catalyst
+
+Though not required by Catalyst directly, our example algorithms use 
+matplotlib to visually display the results of the trading algorithms. If you 
+wish to run any examples or use matplotlib during development, it can be 
+installed using:
+
+.. code-block:: bash
+
+    $ pip install matplotlib
 
 GNU/Linux
 ~~~~~~~~~
@@ -60,25 +85,27 @@ On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
    $ pacman -S lapack gcc gcc-fortran pkg-config
 
-There are also AUR packages available for installing `Python 3.4
-<https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-3.5, but Zipline only currently supports 3.4), and `ta-lib
-<https://aur.archlinux.org/packages/ta-lib/>`_, an optional Zipline dependency.
-Python 2 is also installable via:
+.. Commenting it out until Catalyst fully supports Python 3.X
+..
+.. There are also AUR packages available for installing `Python 3.4
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-.. code-block:: bash
+.. 
 
-   $ pacman -S python2
+..   $ pacman -S python2
 
 OSX
 ~~~
 
-The version of Python shipped with OSX by default is generally out of date, and
-has a number of quirks because it's used directly by the operating system.  For
-these reasons, many developers choose to install and use a separate Python
+The version of Python shipped with OSX by default is generally out of date, 
+and has a number of quirks because it's used directly by the operating system.
+For these reasons, many developers choose to install and use a separate Python
 installation. The `Hitchhiker's Guide to Python`_ provides an excellent guide
-to `Installing Python on OSX <http://docs.python-guide.org/en/latest/>`_, which
-explains how to install Python with the `Homebrew`_ manager.
+to `Installing Python on OSX <http://docs.python-guide.org/en/latest/>`_, 
+which explains how to install Python with the `Homebrew`_ manager.
 
 Assuming you've installed Python with Homebrew, you'll also likely need the
 following brew packages:
@@ -87,36 +114,294 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
+OSX + virtualenv + matplotlib
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A note about using matplotlib in virtual enviroments on OSX: it may be 
+necessary to run
+
+.. code-block:: bash
+
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+
+in order to override the default ``macosx`` backend for your system, which 
+may not be accessible from inside the virtual environment. This will allow 
+Catalyst to open matplotlib charts from within a virtual environment, which 
+is useful for displaying the performance of your backtests.  To learn more 
+about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+.. _windows:
+
 Windows
 ~~~~~~~
 
-For windows, the easiest and best supported way to install zipline is to use
+In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This 
+package contains the compiler and the set of system headers necessary for 
+producing binary wheels for Python 2.7 packages. If it's not already in your 
+system, download it and install it before proceeding to the next step.
+
+For windows, the easiest and best supported way to install Catalyst is to use
 :ref:`Conda <conda>`.
 
+Some problems we have encountered installing the **Visual C++ Compiler** 
+mentioned above are as follows:
+
+- **The system administrator has set policies to prevent this installation**.
+  
+  In some systems, there is a default *Windows Software Restriction* policy 
+  that prevents the installation of some software packages like this one. 
+  You'll have to change the Registry to circumvent this:
+
+  - Click ``Start``, and search for ``regedit`` and launch the 
+    ``Registry Editor``
+  - Navigate to the following folder:
+    ``HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer``
+  - If there is an entry for ``DisableMSI``, set the Value data to 0.
+  - If there is no such entry, click on the ``Edit`` menu -> ``New`` -> 
+    ``DWORD (32-bit) Value`` and enter ``DisableMSI`` as the Name (and by 
+    default you get 0 as the Value Data)
+
+|
+- **The installer has encountered an unexpected error installing this package. 
+  This may indicate a problem with this package. The error code is 2503.**
+
+  We have observed this when trying to install a package without enough 
+  administrator permissions. Even when you are logged in as an Administrator, 
+  you have to explictily install this package with administrator privileges:
+
+  - Click ``Start`` and find ``CMD`` or ``Command Prompt``
+  - Right click on it and choose ``Run as administrator``
+  - ``cd`` into the folder where you downloaded ``VCForPython27.msi``
+  - Run ``msiexec /i VCForPython27.msi``
+
+
+Amazon Linux AMI
+~~~~~~~~~~~~~~~~
+
+The packages ``pip`` and ``setuptools`` that come shipped by default are very 
+outdated. Thus, you first need to run:
+
+.. code-block:: bash
+
+   pip install --upgrade pip setuptools
+
+The default installation is also missing the C and C++ compilers, which you 
+install by:
+
+.. code-block:: bash
+
+   sudo yum install gcc gcc-c++
+
+Then you should follow the regular installation instructions outlined at the 
+beginning of this page.
+
+
+Troubleshooting ``pip`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Issue**: 
+   Package enigma-catalyst cannot be found 
+ 
+**Solution**: 
+   Make sure you have the most up-to-date version of pip installed, by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip
+
+   On Windows, the recommended command is:
+
+   .. code-block:: bash
+
+      python -m pip install --upgrade pip
+
+----
+
+**Issue**: 
+   Package enigma-catalyst cannot still be found, even after upgrading pip 
+   (see above), with an error similar to:
+
+   .. code-block:: bash
+
+      Downloading/unpacking enigma-catalyst
+      Could not find a version that satisfies the requirement enigma-catalyst 
+      (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 
+      0.2.dev1, 0.1.dev8, 0.1.dev6)
+      Cleaning up...
+      No distributions matching the version for enigma-catalyst
+
+**Solution**:
+   In some systems (this error has been reported in Ubuntu), pip is configured 
+   to only find stable versions by default. Since Catalyst is in alpha 
+   version, pip cannot find a matching version that satisfies the installation 
+   requirements. The solution is to include the `--pre` flag to include 
+   pre-release and development versions:
+
+   .. code-block:: bash
+
+      pip install --pre enigma-catalyst
+
+----
+
+**Issue**: 
+   Package enigma-catalyst fails to install because of outdated setuptools
+
+**Solution**: 
+   Upgrade to the most up-to-date setuptools package by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip setuptools
+
+----
+
+**Issue**:
+   Missing required packages  
+
+**Solution**:
+   Download `requirements.txt 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/requirements.txt>`_ 
+   (click on the *Raw* button and Right click -> Save As...) and use it to
+   install all the required dependencies by running:
+
+   .. code-block:: bash
+
+      pip install -r requirements.txt
+
+----
+
+**Issue**: 
+   Installation fails with error: 
+   ``fatal error: Python.h: No such file or directory``
+
+**Solution**: 
+   Some systems (this issue has been reported in Ubuntu) require `python-dev` 
+   for the proper build and installation of package dependencies. The solution 
+   is to install python-dev, which is independent of the virtual environment. 
+   In Ubuntu, you would need to run:
+
+   .. code-block:: bash
+
+      sudo apt-get install python-dev
+
+
 .. _conda:
 
 Installing with ``conda``
 -------------------------
 
-Another way to install Zipline is via the ``conda`` package manager, which
+Another way to install Catalyst is via the ``conda`` package manager, which
 comes as part of Continuum Analytics' `Anaconda
 <http://continuum.io/downloads>`_ distribution.
 
 The primary advantage of using Conda over ``pip`` is that conda natively
 understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Zipline and its dependencies
-without requiring the use of a second tool to acquire Zipline's non-Python
-dependencies.
+``scipy``.  This means that ``conda`` can install Catalyst and its 
+dependencies without requiring the use of a second tool to acquire Catalyst's 
+non-Python dependencies.
+
+  For Windows, you will need the *Microsoft Visual C++ Compiler for Python 
+  2.7*. Follow the instructions on the :ref:`Windows` section and come back 
+  here.
 
 For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, 
+you can install MiniConda, which is a smaller footprint (fewer packages and 
+smaller size) than its big brother Anaconda, but it still contains all the 
+main packages needed. To install MiniConda, you can follow these steps:
 
-Once conda has been set up you can install Zipline from our ``Quantopian``
-channel:
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 
+   for your Operating System.
+2. Install MiniConda. See the `Installation Instructions 
+   <https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal 
+   window, which should print the list of packages installed with Conda.
 
-.. code-block:: bash
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you 
+   saved the above ``python2.7-environment.yml`` file.
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new 
+   session to run Catalyst):
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 
+above failed for any reason, you can try setting up the environment manually 
+with the following steps:
+
+1. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy zlib
+
+2. Activate the environment:
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+3. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+Getting Help
+------------
+
+If after following the instructions above, and going through the 
+*Troubleshooting* sections, you still experience problems installing Catalyst,
+you can seek additional help through the following channels:
+
+- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
+  the #catalyst_dev channel where many other users (as well as the project 
+  developers) hang out, and can assist you with your particular issue. The 
+  more descriptive and the more information you can provide, the easiest will 
+  be for others to help you out.
+
+- Report the problem you are experiencing on our 
+  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ 
+  following the guidelines provided therein. Before you do so, take a moment 
+  to browse through all `previous reported issues
+  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ 
+  in the likely case that someone else experienced that same issue before, 
+  and you get a hint on how to solve it.
 
-    conda install -c Quantopian zipline
 
 .. _`Debian-derived`: https://www.debian.org/misc/children-distros
 .. _`RHEL-derived`: https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux_derivatives

docs/source/live-trading.rst

@@ -0,0 +1,118 @@
+Live Trading
+============
+This document explains how to get started with live trading.
+
+Supported Exchanges
+^^^^^^^^^^^^^^^^^^^
+Catalyst can trade against these exchanges:
+
+- Bitfinex, id= ``bitfinex``
+- Bittrex, id= ``bittrex``
+- Poloniex, id= ``poloniex``
+
+Authentication
+^^^^^^^^^^^^^^
+Most exchanges require token key/secret combination for authentication. By
+convention, Catalyst uses an ``auth.json`` file to hold this data.
+
+This example illustrates the convention using the *Bitfinex* exchange.
+Here is how to generate key and secret values for the Bitfinex exchange:
+https://docs.bitfinex.com/v1/docs/api-access. Most exchanges follow
+a similar process.
+
+The auth.json file:
+
+.. code-block:: json
+
+  {
+    "name": "bitfinex",
+    "key": "my-key",
+    "secret": "my-secret"
+  }
+
+
+The file goes here: ``~/.catalyst/data/exchanges/bitfinex/auth.json``
+
+Note that the `bitfinex` part in the directory above corresponds to the id of the Bitfinex
+exchange as defined in the "Supported Exchanges" section above.
+Attempting to run an algorithm where the targeted exchange is missing
+its ``auth.json`` file will create the directory structure and create an empty
+auth.json file, but will result in an error.
+
+Currency Symbols
+^^^^^^^^^^^^^^^^
+Catalyst introduces a universal convention to reference
+trading pairs and individual currencies. This
+is required to ensure that the ``symbol()`` api predictably
+returns the correct asset regardless of the targeted exchange.
+
+Exchanges tend to use their own convention to represent currencies
+(e.g. XBT and BTC both represent Bitcoin on different exchanges).
+Trading pairs are also inconsistent. For example, Bitfinex
+puts the market currency before the base currency without a
+separator, Bittrex puts the base currency first and uses a dash
+seperator.
+
+Here is the Catalyst convention:
+
+*[Market Currency]_[Base Currency]* all lowercase.
+
+Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
+
+Here are some examples:
+
+.. code-block:: json
+
+  # With Bitfinex
+  bitcoin_usd_asset = symbol('btc_usd')
+  ethereum_bitcoin_asset = symbol('eth_btc')
+
+  # With Bittrex
+  ethereum_bitcoin_asset = symbol('eth_btc')
+  neo_ethereum_asset = symbol('neo_eth)
+
+Note that the trading pairs are always referenced in the same manner.
+However, not all trading pairs are available on all exchanges. An
+error will occur if the specified trading pair is not trading
+on the exchange. To check which currency pairs are available on each 
+of the supported exchanges, see `Catalyst Market Coverage <https://www.enigma.co/catalyst/status`_.
+
+Trading an Algorithm
+^^^^^^^^^^^^^^^^^^^^
+There is no special convention to follow when writing an
+algorithm for live trading. The same algorithm should work in
+backtest and live execution mode without modification.
+
+What differs are the arguments provided to the catalyst client or
+`run_algorithm()` interface. Here is the same example in both interfaces:
+
+.. code-block:: bash
+
+  catalyst live -f my_algo_code -x bitfinex -c btc -n my_algo_name 
+
+.. code-block:: python
+
+  run_algorithm(
+      initialize=initialize,
+      handle_data=handle_data,
+      analyze=analyze,
+      exchange_name='bitfinex',
+      live=True,
+      algo_namespace='my_algo_name',
+      base_currency='btc'
+  )
+
+
+Here is the breakdown of the new arguments:
+
+- ``live``: Boolean flag which enables live trading.
+- ``exchange_name``: The name of the targeted exchange
+  (supported values: *bitfinex*, *bittrex*).
+- ``algo_namespace``: A arbitrary label assigned to your algorithm for
+  data storage purposes.
+- ``base_currency``: The base currency used to calculate the
+  statistics of your algorithm. Currently, the base currency of all
+  trading pairs of your algorithm must match this value.
+
+Here is a complete algorithm for reference:
+`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_

docs/source/naming-convention.rst

@@ -0,0 +1,66 @@
+Naming Convention
+=================
+
+Catalyst introduces a standardized naming convention for all asset pairs 
+trading on any exchange in the following form:
+
+
+    **{market_currency}_{base_currency}**
+
+Where {market_currency} is the asset to be traded using {base_currency} as 
+the reference, both written in lowercase and separated with an underscore.
+
+This standardization is needed to overcome the lack of consistency in the 
+naming of assets across different exchanges, and making it easier to the user
+to refer to the asset pairs that you want to trade.
+
+Catalyst maintains a `Market Coverage Overview <https://www.enigma.co/catalyst/status>`_ 
+where you can check the mapping between Catalyst naming pairs and that of each 
+exchange. Catalyst will always expect in all its functions that you will refer to 
+the asset pairs by using the Catalyst naming convention.
+
+If at any point, you input the wrong name for an asset pair, you will get an error 
+of that pair not found in the given exchange, and a list of pairs available on that exchange:
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usd
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	Error traceback: /Volumes/Data/Users/victoris/Desktop/Enigma/user-install/catalyst-dev/catalyst/exchange/exchange.py (line 175)
+	SymbolNotFoundOnExchange:  Symbol btc_usd not found on exchange Poloniex. 
+	Choose from: ['rep_usdt', 'gno_btc', 'xvc_btc', 'pink_btc', 'sys_btc', 
+	'emc2_btc', 'rads_btc', 'note_btc', 'maid_btc', 'bch_btc', 'gnt_btc', 
+	'bcn_btc', 'rep_btc', 'bcy_btc', 'cvc_btc', 'nxt_xmr', 'zec_usdt', 
+	'fct_btc', 'gas_btc', 'pot_btc', 'eth_usdt', 'btc_usdt', 'lbc_btc', 
+	'dcr_btc', 'etc_usdt', 'omg_eth', 'amp_btc', 'xpm_btc', 'nxt_btc', 
+	'vtc_btc', 'steem_eth', 'blk_xmr', 'pasc_btc', 'zec_xmr', 'grc_btc', 
+	'nxc_btc', 'btcd_btc', 'ltc_btc', 'dash_btc', 'naut_btc', 'zec_eth', 
+	'zec_btc', 'burst_btc', 'zrx_eth', 'bela_btc', 'steem_btc', 'etc_btc', 
+	'eth_btc', 'huc_btc', 'strat_btc', 'lsk_btc', 'exp_btc', 'clam_btc', 
+	'rep_eth', 'dash_xmr', 'cvc_eth', 'bch_usdt', 'zrx_btc', 'dash_usdt', 
+	'blk_btc', 'xrp_btc', 'nxt_usdt', 'neos_btc', 'omg_btc', 'bts_btc', 
+	'doge_btc', 'gnt_eth', 'sbd_btc', 'gno_eth', 'xcp_btc', 'ltc_usdt', 
+	'btm_btc', 'xmr_usdt', 'lsk_eth', 'omni_btc', 'nav_btc', 'fldc_btc', 
+	'ppc_btc', 'xbc_btc', 'dgb_btc', 'sc_btc', 'btcd_xmr', 'vrc_btc', 
+	'ric_btc', 'str_btc', 'maid_xmr', 'xmr_btc', 'sjcx_btc', 'via_btc', 
+	'xem_btc', 'nmc_btc', 'etc_eth', 'ltc_xmr', 'ardr_btc', 'gas_eth', 
+	'flo_btc', 'xrp_usdt', 'game_btc', 'bch_eth', 'bcn_xmr', 'str_usdt']
+
+In the example above, exchange Poloniex does not use USD, but uses instead the 
+USDT cryptocurrency asset that is issued on the Bitcoin blockchain via the Omni
+Layer Protocol. Each USDT unit is backed by a U.S Dollar held in the reserves of 
+Tether Limited. USDT can be transferred, stored, and spent, just like bitcoins 
+or any other cryptocurrency. Given its 1:1 mapping to the USD, is a viable alternative.
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usdt
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	    [====================================]  Fetching poloniex daily candles: :  100%
+

docs/source/releases.rst

@@ -2,24 +2,234 @@
 Release Notes
 =============
 
-.. include:: whatsnew/1.1.1.txt
+Version 0.3.7
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
 
-.. include:: whatsnew/1.1.0.txt
+Bug Fixes
+~~~~~~~~~
 
-.. include:: whatsnew/1.0.2.txt
+- Fixed an SSL cert issue (:issue:`64`)
+- Fixed cumulative stats warnings (:issue:`63`)
+- Disabled auto-ingestion because of unresolved caching issues (:issue:`47`)
+- Standardized live-trading stats (:issue:`61`)
 
-.. include:: whatsnew/1.0.1.txt
+Build
+~~~~~
 
-.. include:: whatsnew/1.0.0.txt
+- Added a mean-reversion sample algo
+- Added minutely stats in the analyze() function (:issue:`62`)
+- Added specificity to some error messages
 
-.. include:: whatsnew/0.9.0.txt
+Version 0.3.6
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
 
-.. include:: whatsnew/0.8.4.txt
+Bug Fixes
+~~~~~~~~~
 
-.. include:: whatsnew/0.8.3.txt
+- Fixed an issue with single bar data.history() (:issue:`55`)
 
-.. include:: whatsnew/0.8.0.txt
+Version 0.3.5
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
 
-.. include:: whatsnew/0.7.0.txt
+Bug Fixes
+~~~~~~~~~
+
+- Added workaround for: KeyError: Timestamp error (:issue:`53`)
+
+Version 0.3.4
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-2
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with auto-ingestion of minute data (:issue:`47`)
+- Fixed issue with sell orders in backtesting
+- Fixed data frequency issues with data.history() in backtesting
+- Fixed an issue with can_trade()
+- Reduced the commission and slippage values to account for lower volume 
+  transactions
+
+Build
+~~~~~
+
+- Added more unit tests
+
+Documentation
+~~~~~~~~~~~~~
+
+- Improved installation notes for Windows C++ compiler and Conda
+- Addition of 
+  `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
+- Addition of 
+  `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
+- Addition of 
+  `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
+- Addition of 
+  `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
+- Addition of `Development Guidelines 
+  <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
+- Addition of 
+  `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
+- Updated code docstrings
+
+
+Version 0.3.3
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-26
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix missing -x in ingest-exchange
+- Fix issue with daily chunks end date (data bundles)
+- Fix issue in the prepare_chunk logic (data bundles)
+
+Build
+~~~~~
+
+- Added data validation unit tests
+
+
+Version 0.3.2
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-25
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix to work with empty data bundles
+- Fix Windows path of ``$HOME/.catalyst`` folder
+- Fix ``etc/python2.7-environment.yml`` for Windows Conda install
+- Fix hash method to create sid numbers compatible across platforms
+- Fix an issue with asset date in chunks
+
+Build
+~~~~~
+
+- Python3 adjustments
+- Added method to clean bundle folders, and remove symbols.json
+- Implemented and improved unit tests
+
+
+Version 0.3.1
+^^^^^^^^^^^^^
+**Release Date**: 2017-10-22
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed OS-dependent path issue in data bundle
+- Changed handling of empty ``auth.json``, instead of throwing an error for 
+  missing file
+- Updated ``etc/python2.7-environment.yml`` to work with Catalyst version 0.3
+- Updated ``catalyst/examples/buy_and_hodl.py``  and 
+  ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
+
+
+Version 0.3
+^^^^^^^^^^^
+**Release Date**: 2017-10-20
+
+- Standardized live and backtesting syntax
+- Added a repository for historical data
+- Added supported for multiple exchanges per algorithm
+- Added a standardized dictionary of symbols for each exchange
+- Added auto-ingestion of bundle data while backtesting
+- Bug fixes
+
+
+Version 0.2.dev5
+^^^^^^^^^^^^^^^^
+**Release Date**: 2017-10-03
+
+- Fixes bug in data.history function that was formatting 'volume' data as 
+  integers, now they are returned as floats with up to 9 decimals of precision. 
+  Data bundles redone.
+
+Version 0.2.dev4 
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-20
+
+- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal 
+  places. Pricing resolution of daily data remains set to 9 decimal places.
+- The current data bundle takes 340MB compressed for download, and 460MB 
+  uncompressed on disk for Catalyst to use.
+
+Version 0.2.dev3
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-20
+
+- 1-minute resolution OHLCV data bundle for backtesting from Poloniex exchange
+- Implementation of trading of fractional crypto assets (i.e. 0.01 BTC)
+- Minimum trade size of a coin can be configured on a per-coin basis, defaults 
+  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger 
+  amounts, which will impact live trading)
+- Increased pricing resolution from 3 to 9 decimal places
+- The current data bundle takes 40MB compressed for download, and 99MB 
+  uncompressed on disk for Catalyst to use.
+
+Version 0.2.dev2 
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-07
+
+- Fix path issue
+
+Version 0.2.dev1
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-09-03
+
+- Implementation of live trading:
+
+  - Comprehensive trading functionality against exchanges Bitfinex and Bittrex.
+  - Support for all trading pairs available on each exchange.
+  - Multiple algorithms can trade simultaneously against a single exchange 
+    using the same account.
+  - Each algorithm has a persisted state (i.e. algorithm can be stopped and 
+    restarted preserving the state without data loss) that tracks all open 
+    orders, executed transactions and portfolio positions.
+
+- Minute by minute portfolio performance metrics.
+
+  - Daily summary performance statistics compatible with pyfolio, a Python 
+    library for performance and risk analysis of financial portfolios
+
+Version 0.1.dev9
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-08-28
+
+- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex 
+  exchange directly
+- Change of bundle storage provider from Dropbox to AWS
+- Fix issue with 1/1000 scaling issue of prices in bundle
+
+Version 0.1.dev8
+^^^^^^^^^^^^^^^^ 
+
+**Release Date**: 2017-08-18
+
+- Fixes issue in the creation of bundles (:issue:`27`)
+
+
+Version 0.1.dev7
+^^^^^^^^^^^^^^^^
+- Fixes issues in empty benchmark (:issue:`16`)
+- Fixes issue of normalizing timestamps before comparison (:issue:`24`)
+- Generic data bundles
+- CLI UI improvements
+
+Version 0.1.dev6
+^^^^^^^^^^^^^^^^
+
+**Release Date**: 2017-07-13
+
+- Initial public release
 
-.. include:: whatsnew/0.6.1.txt

docs/source/resources.rst

@@ -0,0 +1,26 @@
+Resources
+=========
+
+-  `Catalyst Whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ 
+
+
+Related 3rd Party APIs
+^^^^^^^^^^^^^^^^^^^^^^
+
+- `Zipline <http://www.zipline.io/appendix.html>`_ is a Pythonic Algorithmic 
+  Trading Library, and the project Catalyst forked off in the spring of 2017.
+- `Quantopian <https://www.quantopian.com/help>`_ provides a platform for 
+  freelance quantitative analysts develop, test, and use trading algorithms to 
+  buy and sell securities. They aim to create a crowd-sourced hedge fund by
+  fostering their community of freelance traders. Quantopian's backtesting and 
+  live-trading engine is powered by *Zipline*.
+- `Pandas <https://pandas.pydata.org/pandas-docs/stable/api.html>`_ is a Python
+  library providing high-performance, easy-to-use data structures and data 
+  analysis tools. Catalyst relies heavily on pandas, and many API functions 
+  return data as Pandas dataframes.
+- `Numpy <https://docs.scipy.org/doc/numpy/reference/>`_ is the fundamental 
+  package for scientific computing with Python. Some of the data computation 
+  that your algorithms will need, will be optimized leveraging Numpy.
+- `Matplotlib <https://matplotlib.org/1.5.3/api/index.html>`_ is a Python 2D 
+  plotting library that many of examples rely on to plot the performance of 
+  trading algorithms

docs/source/videos.rst

@@ -0,0 +1,26 @@
+Videos
+======
+
+
+Installation: MacOS
+-------------------
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/ZnsslmHljvw" frameborder="0" allowfullscreen></iframe>
+
+|
+|
+Installation: Windows
+---------------------
+
+Where things go smoothly:
+
+.. raw:: html
+
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/H8HqcEbZmkk" frameborder="0" allowfullscreen></iframe>
+
+|
+Where things don't:
+   
+   Coming up next!

docs/source/welcome.rst

@@ -0,0 +1,43 @@
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+|
+Catalyst is an algorithmic trading library for crypto-assets written in Python.
+It allows trading strategies to be easily expressed and backtested against 
+historical data (with daily and minute resolution), providing analytics and 
+insights regarding a particular strategy's performance. Catalyst also supports
+live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, 
+and Poloniex) with more being added over time. Catalyst empowers users to share 
+and curate data and build profitable, data-driven investment strategies. Please 
+visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or 
+refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for 
+further technical details.
+
+Catalyst builds on top of the well-established 
+`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
+minimize structural changes to the general API to maximize compatibility with 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
+for questions around Catalyst, algorithmic trading and technical support.
+
+Features
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.
+-  Addition of Bitcoin price (btc_usdt) as a benchmark for comparing 
+   performance across trading algorithms.

etc/python2.7-environment.yml

@@ -1,30 +1,22 @@
 name: catalyst
 channels:
-- statiskit
 - defaults
 dependencies:
 - certifi=2016.2.28=py27_0
-- coverage=4.4.1=py27_0
-- nose=1.3.7=py27_1
-- openssl=1.0.2l=0
-- path.py=10.3.1=py27_0
+- mkl=2017.0.3
+- numpy=1.13.1=py27_0
+- openssl=1.0.2l
 - pip=9.0.1=py27_1
-- python=2.7.13=0
-- pyyaml=3.12=py27_0
-- readline=6.2=2
-- setuptools=36.4.0=py27_0
-- six=1.10.0=py27_0
-- sqlite=3.13.0=0
-- tk=8.5.18=0
+- python=2.7.13
+- scipy=0.19.1=np113py27_0
+- setuptools=36.4.0=py27_1
+- sqlite=3.13.0
+- tk=8.5.18
 - wheel=0.29.0=py27_0
-- yaml=0.1.6=0
-- zlib=1.2.11=0
-- libdev=1.0.0=py27_0
-- python-dev=1.0.0=py27_0
-- python-scons=3.0.0=py27_0
+- zlib=1.2.11
 - pip:
-  - alembic==0.9.5
-  - backports.shutil-get-terminal-size==1.0.0
+  - alembic==0.9.6
+  - backports.functools-lru-cache==1.4
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
@@ -32,36 +24,22 @@ dependencies:
   - contextlib2==0.5.5
   - cycler==0.10.0
   - cyordereddict==1.0.0
-  - cython==0.26.1
+  - cython==0.27.1
   - decorator==4.1.2
   - empyrical==0.2.1
-  - enigma-catalyst>=0.2.dev2
-  - enum34==1.1.6
-  - functools32==3.2.3.post2
   - idna==2.6
   - intervaltree==2.1.0
-  - ipdb==0.10.3
-  - ipdbplugin==1.4.5
-  - ipython==5.5.0
-  - ipython-genutils==0.2.0
   - logbook==1.1.0
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.0.2
+  - matplotlib==2.1.0
   - multipledispatch==0.4.9
-  - networkx==1.11
+  - networkx==2.0
   - numexpr==2.6.4
-  - numpy==1.13.1
   - pandas==0.19.2
   - pandas-datareader==0.5.0
-  - pathlib2==2.3.0
   - patsy==0.4.1
-  - pexpect==4.2.1
-  - pickleshare==0.7.4
-  - prompt-toolkit==1.0.15
-  - ptyprocess==0.5.2
-  - pygments==2.2.0
   - pyparsing==2.2.0
   - python-dateutil==2.6.1
   - python-editor==1.0.3
@@ -69,16 +47,12 @@ dependencies:
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1
-  - scandir==1.5
-  - scipy==0.19.1
-  - scons==3.0.0a20170821
-  - simplegeneric==0.8.1
+  - six==1.11.0
   - sortedcontainers==1.5.7
   - sqlalchemy==1.1.14
   - statsmodels==0.8.0
   - subprocess32==3.2.7
   - tables==3.4.2
   - toolz==0.8.2
-  - traitlets==4.3.2
   - urllib3==1.22
-  - wcwidth==0.1.7
+  - enigma-catalyst>=0.3

etc/requirements.txt

@@ -1,7 +1,7 @@
 # Incompatible with earlier PIP versions
 pip>=7.1.0
 # bcolz fails to install if this is not in the build_requires.
-setuptools>18.0
+setuptools>36.0
 
 # Logging
 Logbook==0.12.5

etc/requirements_docs.txt

@@ -1,4 +1,3 @@
 Sphinx>=1.3.2
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
-enigma-catalyst		# readthedocs.org

setup.py

@@ -304,7 +304,7 @@ setup(
                   if '__pycache__' not in root},
     license='Apache 2.0',
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
+        'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Programming Language :: Python',

tests/exchange/base.py

@@ -1,4 +1,3 @@
-import unittest
 from abc import ABCMeta, abstractmethod
 
 

tests/exchange/test_bcolz.py

@@ -0,0 +1,150 @@
+import shutil
+import random
+import tempfile
+import pandas as pd
+
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
+    BcolzExchangeBarReader
+
+from catalyst.exchange.bundle_utils import get_df_from_arrays
+
+from nose.tools import assert_equals
+
+
+class TestBcolzWriter(object):
+    @classmethod
+    def setup_class(cls):
+        cls.columns = ['open', 'high', 'low', 'close', 'volume']
+
+    def setUp(self):
+        self.root_dir = tempfile.mkdtemp()  # Create a temporary directory
+
+    def tearDown(self):
+        shutil.rmtree(self.root_dir)  # Remove the directory after the test
+
+    def generate_df(self, exchange_name, freq, start, end):
+        bundle = ExchangeBundle(exchange_name)
+        index = bundle.get_calendar_periods_range(start, end, freq)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(random.random(), inplace=True)
+        return df
+
+    def test_bcolz_write_daily_past(self):
+        start = pd.to_datetime('2016-01-01')
+        end = pd.to_datetime('2016-12-31')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_daily_present(self):
+        start = pd.to_datetime('2017-01-01')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_minute_past(self):
+        start = pd.to_datetime('2015-04-01 00:00')
+        end = pd.to_datetime('2015-04-30 23:59')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        pass
+
+    def test_bcolz_write_minute_present(self):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def bcolz_exchange_daily_write_read(self, exchange_name):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        bundle = ExchangeBundle(exchange_name)
+
+        df = self.generate_df(exchange_name, freq, start, end)
+
+        print df.index[0],df.index[-1]
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=df.index[0],
+            end_session=df.index[-1],
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        reader = BcolzExchangeBarReader(rootdir=self.root_dir,
+                                        data_frequency=freq)
+
+        arrays = reader.load_raw_arrays(self.columns, start, end, [1, ])
+
+        periods = bundle.get_calendar_periods_range(
+            start, end, freq
+        )
+
+        dx = get_df_from_arrays(arrays, periods)
+
+        assert_equals(df.equals(df), True)
+        pass
+
+    def test_bcolz_bitfinex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('bitfinex')
+
+    def test_bcolz_poloniex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('poloniex')

tests/exchange/test_bitfinex.py

@@ -8,7 +8,7 @@ from catalyst.finance.execution import (LimitOrder)
 log = Logger('test_bitfinex')
 
 
-class BitfinexTestCase(BaseExchangeTestCase):
+class TestBitfinex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
         log.info('creating bitfinex object')
@@ -48,7 +48,7 @@ class BitfinexTestCase(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         ohlcv_neo = self.exchange.get_candles(
-            data_frequency='1m',
+            freq='1T',
             assets=self.exchange.get_asset('neo_btc')
         )
         pass

tests/exchange/test_bittrex.py

@@ -1,3 +1,4 @@
+import pandas as pd
 from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
@@ -7,15 +8,15 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 log = Logger('test_bittrex')
 
 
-class BittrexTestCase(BaseExchangeTestCase):
+class TestBittrex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        print ('creating bittrex object')
         auth = get_exchange_auth('bittrex')
         self.exchange = Bittrex(
             key=auth['key'],
             secret=auth['secret'],
-            base_currency='btc'
+            base_currency=None,
+            portfolio=None
         )
 
     def test_order(self):
@@ -51,16 +52,19 @@ class BittrexTestCase(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         ohlcv_neo = self.exchange.get_candles(
-            data_frequency='5m',
-            assets=self.exchange.get_asset('neo_btc')
+            freq='5T',
+            assets=self.exchange.get_asset('neo_btc'),
+            bar_count=20,
+            end_dt=pd.to_datetime('2017-10-20', utc=True)
         )
         ohlcv_neo_ubq = self.exchange.get_candles(
-            data_frequency='5m',
+            freq='1D',
             assets=[
                 self.exchange.get_asset('neo_btc'),
                 self.exchange.get_asset('ubq_btc')
             ],
-            bar_count=14
+            bar_count=14,
+            end_dt=pd.to_datetime('2017-10-20', utc=True)
         )
         pass
 

tests/exchange/test_bundle.py

@@ -1,34 +1,57 @@
-from logging import Logger
+import hashlib
+import os
+import tempfile
+from logging import getLogger
 
 import pandas as pd
 
 from catalyst import get_calendar
-from catalyst.exchange.bundle_utils import get_bcolz_chunk
+from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
+    get_start_dt, get_df_from_arrays
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.exchange_utils import get_exchange_folder
-from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.factory import get_exchange
+from catalyst.exchange.stats_utils import df_to_string
 from catalyst.utils.paths import ensure_directory
 
-log = Logger('test_exchange_bundle')
+log = getLogger('test_exchange_bundle')
 
 
-class ExchangeBundleTestCase:
-    def test_ingest_minute(self):
-        data_frequency = 'minute'
-        exchange_name = 'bitfinex'
+class TestExchangeBundle:
+    def test_spot_value(self):
+        data_frequency = 'daily'
+        exchange_name = 'poloniex'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
         assets = [
-            exchange.get_asset('neo_eth')
+            exchange.get_asset('btc_usdt')
+        ]
+        dt = pd.to_datetime('2017-10-14', utc=True)
+
+        values = exchange_bundle.get_spot_values(
+            assets=assets,
+            field='close',
+            dt=dt,
+            data_frequency=data_frequency
+        )
+        pass
+
+    def test_ingest_minute(self):
+        data_frequency = 'minute'
+        exchange_name = 'poloniex'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [
+            exchange.get_asset('eth_btc')
         ]
 
-        # start = pd.to_datetime('2017-09-01', utc=True)
-        start = pd.to_datetime('2017-9-15', utc=True)
-        end = pd.to_datetime('2017-9-30', utc=True)
+        start = pd.to_datetime('2016-03-01', utc=True)
+        end = pd.to_datetime('2017-11-1', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -73,18 +96,44 @@ class ExchangeBundleTestCase:
         )
         pass
 
-    def test_ingest_daily(self):
+    def test_ingest_exchange(self):
         # exchange_name = 'bitfinex'
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
         exchange_name = 'bitfinex'
-        data_frequency = 'daily'
-        include_symbols = 'etc_btc'
+        data_frequency = 'minute'
 
-        start = pd.to_datetime('2016-11-01', utc=True)
-        end = pd.to_datetime('2017-10-16', utc=True)
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
 
+        log.info('ingesting exchange bundle {}'.format(exchange_name))
+        exchange_bundle.ingest(
+            data_frequency=data_frequency,
+            include_symbols=None,
+            exclude_symbols=None,
+            start=None,
+            end=None,
+            show_progress=True
+        )
+
+        pass
+
+    def test_ingest_daily(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+        include_symbols = 'neo_btc'
+
+        # exchange_name = 'poloniex'
+        # data_frequency = 'daily'
+        # include_symbols = 'eth_btc'
+
+        # start = pd.to_datetime('2017-1-1', utc=True)
+        # end = pd.to_datetime('2017-10-16', utc=True)
+        # periods = get_periods_range(start, end, data_frequency)
+
+        start = None
+        end = None
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
 
@@ -104,12 +153,18 @@ class ExchangeBundleTestCase:
             assets.append(exchange.get_asset(pair_symbol))
 
         reader = exchange_bundle.get_reader(data_frequency)
+        start_dt = reader.first_trading_day
+        end_dt = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
+
         for asset in assets:
             arrays = reader.load_raw_arrays(
                 sids=[asset.sid],
                 fields=['close'],
-                start_dt=start,
-                end_dt=end
+                start_dt=start_dt,
+                end_dt=end_dt
             )
             print('found {} rows for {} ingestion\n{}'.format(
                 len(arrays[0]), asset.symbol, arrays[0])
@@ -253,7 +308,7 @@ class ExchangeBundleTestCase:
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neo_btc')
+        asset = exchange.get_asset('neos_btc')
 
         path = get_bcolz_chunk(
             exchange_name=exchange_name,
@@ -263,3 +318,210 @@ class ExchangeBundleTestCase:
         )
 
         pass
+
+    def test_hash_symbol(self):
+        symbol = 'etc_btc'
+        sid = int(
+            hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+        ) % 10 ** 6
+        pass
+
+    def test_validate_data(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-9-2 1:00', utc=True)
+        bar_count = 60
+
+        bundle_series = exchange_bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count * 5,
+            field='close',
+            data_frequency='minute',
+        )
+        candles = exchange.get_candles(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            freq='1T'
+        )
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+
+        frames = []
+        for asset in assets:
+            bundle_df = pd.DataFrame(
+                data=dict(bundle_price=bundle_series[asset]),
+                index=bundle_series[asset].index
+            )
+            exchange_series = exchange.get_series_from_candles(
+                candles=candles[asset],
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                field='close'
+            )
+            exchange_df = pd.DataFrame(
+                data=dict(exchange_price=exchange_series),
+                index=exchange_series.index
+            )
+
+            df = exchange_df.join(bundle_df, how='left')
+            df['last_traded'] = df.index
+            df['asset'] = asset.symbol
+            df.set_index(['asset', 'last_traded'], inplace=True)
+
+            frames.append(df)
+
+        df = pd.concat(frames)
+        print('\n' + df_to_string(df))
+        pass
+
+    def test_ingest_candles(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-10-20', utc=True)
+        bar_count = 100
+
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        candles = exchange.get_candles(
+            assets=assets,
+            start_dt=start_dt,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            freq='1T'
+        )
+
+        writer = bundle.get_writer(start_dt, end_dt, data_frequency)
+        for asset in assets:
+            dates = [candle['last_traded'] for candle in candles[asset]]
+
+            values = dict()
+            for field in ['open', 'high', 'low', 'close', 'volume']:
+                values[field] = [candle[field] for candle in candles[asset]]
+
+            periods = bundle.get_calendar_periods_range(
+                start_dt, end_dt, data_frequency
+            )
+            df = pd.DataFrame(values, index=dates)
+            df = df.loc[periods].fillna(method='ffill')
+
+            # TODO: why do I get an extra bar?
+            bundle.ingest_df(
+                ohlcv_df=df,
+                data_frequency=data_frequency,
+                asset=asset,
+                writer=writer,
+                empty_rows_behavior='raise',
+                duplicates_behavior='raise'
+            )
+
+        bundle_series = bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            field='close',
+            data_frequency=data_frequency,
+            reset_reader=True
+        )
+        df = pd.DataFrame(bundle_series)
+        print('\n' + df_to_string(df))
+        pass
+
+    def main_bundle_to_csv(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset('eth_btc')
+
+        start_dt = pd.to_datetime('2016-5-31', utc=True)
+        end_dt = pd.to_datetime('2016-6-1', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange=exchange,
+            data_frequency=data_frequency,
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
+
+    def bundle_to_csv(self):
+        exchange_name = 'poloniex'
+        data_frequency = 'minute'
+        period = '2017-09'
+        symbol = 'eth_btc'
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset(symbol)
+
+        path = get_bcolz_chunk(
+            exchange_name=exchange.name,
+            symbol=asset.symbol,
+            data_frequency=data_frequency,
+            period=period
+        )
+        self._bundle_to_csv(
+            asset=asset,
+            exchange=exchange,
+            data_frequency=data_frequency,
+            path=path,
+            filename=period
+        )
+        pass
+
+    def _bundle_to_csv(self, asset, exchange, data_frequency, filename,
+                       path=None, start_dt=None, end_dt=None):
+        bundle = ExchangeBundle(exchange)
+        reader = bundle.get_reader(data_frequency, path=path)
+
+        if start_dt is None:
+            start_dt = reader.first_trading_day
+
+        if end_dt is None:
+            end_dt = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
+
+        arrays = None
+        try:
+            arrays = reader.load_raw_arrays(
+                sids=[asset.sid],
+                fields=['open', 'high', 'low', 'close', 'volume'],
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+        except Exception as e:
+            log.warn('skipping ctable for {} from {} to {}: {}'.format(
+                asset.symbol, start_dt, end_dt, e
+            ))
+
+        periods = bundle.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+        df = get_df_from_arrays(arrays, periods)
+
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', exchange.name, asset.symbol
+        )
+        ensure_directory(folder)
+
+        path = os.path.join(folder, filename + '.csv')
+
+        log.info('creating csv file: {}'.format(path))
+        print('HEAD\n{}'.format(df.head(10)))
+        print('TAIL\n{}'.format(df.tail(10)))
+        df.to_csv(path)
+        pass

tests/exchange/test_clock.py

@@ -1,50 +0,0 @@
-from unittest import TestCase
-from logbook import Logger
-from mock import patch, sentinel
-from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.utils.calendars.trading_calendar import days_at_time
-from datetime import time
-from collections import defaultdict
-from catalyst.utils.calendars import get_calendar
-import pandas as pd
-
-log = Logger('ExchangeClockTestCase')
-
-
-class ExchangeClockTestCase(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.open_calendar = get_calendar("OPEN")
-
-        cls.sessions = pd.Timestamp.utcnow()
-
-    def setUp(self):
-        self.internal_clock = None
-        self.events = defaultdict(list)
-
-    def advance_clock(self, x):
-        """Mock function for sleep. Advances the internal clock by 1 min"""
-        # The internal clock advance time must be 1 minute to match
-        # MinutesSimulationClock's update frequency
-        self.internal_clock += pd.Timedelta('1 min')
-
-    def get_clock(self, arg, *args, **kwargs):
-        """Mock function for pandas.to_datetime which is used to query the
-        current time in RealtimeClock"""
-        assert arg == "now"
-        return self.internal_clock
-
-    def test_clock(self):
-        with patch('catalyst.exchange.simple_clock.pd.to_datetime') as to_dt, \
-                patch('catalyst.exchange.simple_clock.sleep') as sleep:
-            clock = SimpleClock(sessions=self.sessions)
-            to_dt.side_effect = self.get_clock
-            sleep.side_effect = self.advance_clock
-            start_time = pd.Timestamp.utcnow()
-            self.internal_clock = start_time
-
-            events = list(clock)
-
-            # Event 0 is SESSION_START which always happens at 00:00.
-            ts, event_type = events[1]
-        pass

tests/exchange/test_data_portal.py

@@ -1,47 +1,37 @@
 import pandas as pd
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
+    DataPortalExchangeLive
 from logbook import Logger
+from test_utils import rnd_history_date_days, rnd_bar_count
 
 from catalyst import get_calendar
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.bitfinex.bitfinex import Bitfinex
 from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.data_portal_exchange import DataPortalExchangeBacktest, \
-    DataPortalExchangeLive
-from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_common_assets
+from catalyst.exchange.factory import get_exchange, get_exchanges
 
 log = Logger('test_bitfinex')
 
 
-class ExchangeDataPortalTestCase:
+class TestExchangeDataPortal:
     @classmethod
     def setup(self):
         log.info('creating bitfinex exchange')
-        auth_bitfinex = get_exchange_auth('bitfinex')
-        self.bitfinex = Bitfinex(
-            key=auth_bitfinex['key'],
-            secret=auth_bitfinex['secret'],
-            base_currency='usd'
-        )
-
-        log.info('creating bittrex exchange')
-        auth_bitfinex = get_exchange_auth('bittrex')
-        self.bittrex = Bittrex(
-            key=auth_bitfinex['key'],
-            secret=auth_bitfinex['secret'],
-            base_currency='usd'
-        )
-
+        exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
         open_calendar = get_calendar('OPEN')
         asset_finder = AssetFinderExchange()
 
         self.data_portal_live = DataPortalExchangeLive(
-            exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex),
+            exchanges=exchanges,
             asset_finder=asset_finder,
             trading_calendar=open_calendar,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
+
         self.data_portal_backtest = DataPortalExchangeBacktest(
-            exchanges=dict(bitfinex=self.bitfinex),
+            exchanges=exchanges,
             asset_finder=asset_finder,
             trading_calendar=open_calendar,
             first_trading_day=None  # will set dynamically based on assets
@@ -106,3 +96,20 @@ class ExchangeDataPortalTestCase:
             assets, 'close', date, 'minute')
         log.info('found spot value {}'.format(value))
         pass
+
+    def test_history_compare_exchanges(self):
+        exchanges = get_exchanges(['bittrex', 'bitfinex', 'poloniex'])
+        assets = get_common_assets(exchanges)
+
+        date = rnd_history_date_days()
+        bar_count = rnd_bar_count()
+        data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count,
+            frequency='1d',
+            field='close',
+            data_frequency='daily'
+        )
+
+        log.info('found history window: {}'.format(data))

tests/exchange/test_poloniex.py

@@ -8,7 +8,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 log = Logger('test_poloniex')
 
 
-class PoloniexTestCase(BaseExchangeTestCase):
+class TestPoloniex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
         print ('creating poloniex object')
@@ -21,7 +21,7 @@ class PoloniexTestCase(BaseExchangeTestCase):
 
     def test_order(self):
         log.info('creating order')
-        asset = self.exchange.get_asset('neo_btc')
+        asset = self.exchange.get_asset('neos_btc')
         order_id = self.exchange.order(
             asset=asset,
             limit_price=0.0005,
@@ -33,7 +33,7 @@ class PoloniexTestCase(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        asset = self.exchange.get_asset('neo_btc')
+        asset = self.exchange.get_asset('neos_btc')
         orders = self.exchange.get_open_orders(asset)
         pass
 
@@ -52,14 +52,14 @@ class PoloniexTestCase(BaseExchangeTestCase):
     def test_get_candles(self):
         log.info('retrieving candles')
         ohlcv_neo = self.exchange.get_candles(
-            data_frequency='5m',
-            assets=self.exchange.get_asset('neo_btc')
+            freq='5T',
+            assets=self.exchange.get_asset('eth_btc')
         )
         ohlcv_neo_ubq = self.exchange.get_candles(
-            data_frequency='5m',
+            freq='5T',
             assets=[
-                self.exchange.get_asset('neo_btc'),
-                self.exchange.get_asset('ubq_btc')
+                self.exchange.get_asset('neos_btc'),
+                self.exchange.get_asset('via_btc')
             ],
             bar_count=14
         )

tests/exchange/test_server_bundle.py

@@ -0,0 +1,124 @@
+import os
+import tarfile
+import importlib
+import pandas as pd
+
+from catalyst import get_calendar
+
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
+from catalyst.data.minute_bars import BcolzMinuteBarMetadata
+from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
+
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.finance import candlestick2_ohlc
+from matplotlib.finance import volume_overlay
+import matplotlib.ticker as ticker
+
+from catalyst.exchange.factory import get_exchange
+
+EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
+exchanges = dict((e, getattr(importlib.import_module(
+    'catalyst.exchange.{0}.{0}'.format(e)), e.capitalize()))
+                 for e in EXCHANGE_NAMES)
+
+
+class ValidateChunks(object):
+    def __init__(self):
+        self.columns = ['open', 'high', 'low', 'close', 'volume']
+
+    def chunk_to_df(self, exchange_name, symbol, data_frequency, period):
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset(symbol)
+
+        filename = get_bcolz_chunk(
+            exchange_name=exchange_name,
+            symbol=symbol,
+            data_frequency=data_frequency,
+            period=period
+        )
+
+        reader = BcolzExchangeBarReader(rootdir=filename,
+                                        data_frequency=data_frequency)
+
+        # metadata = BcolzMinuteBarMetadata.read(filename)
+
+        start = reader.first_trading_day
+        end = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end = end - pd.Timedelta(hours=23, minutes=59)
+
+        print start, end, data_frequency
+
+        arrays = reader.load_raw_arrays(self.columns, start, end,
+                                        [asset.sid, ])
+
+        bundle = ExchangeBundle(exchange_name)
+
+        periods = bundle.get_calendar_periods_range(
+            start, end, data_frequency
+        )
+
+        return get_df_from_arrays(arrays, periods)
+
+    def plot_ohlcv(self, df):
+
+        fig, ax = plt.subplots()
+
+        # Plot the candlestick
+        candlestick2_ohlc(ax, df['open'], df['high'], df['low'], df['close'],
+                          width=1, colorup='g', colordown='r', alpha=0.5)
+
+        # shift y-limits of the candlestick plot so that there is space
+        # at the bottom for the volume bar chart
+        pad = 0.25
+        yl = ax.get_ylim()
+        ax.set_ylim(yl[0] - (yl[1] - yl[0]) * pad, yl[1])
+
+        # Add a seconds axis for the volume overlay
+        ax2 = ax.twinx()
+
+        ax2.set_position(
+            matplotlib.transforms.Bbox([[0.125, 0.1], [0.9, 0.26]]))
+
+        # Plot the volume overlay
+        bc = volume_overlay(ax2, df['open'], df['close'], df['volume'],
+                            colorup='g', alpha=0.5, width=1)
+
+        ax.xaxis.set_major_locator(ticker.MaxNLocator(6))
+
+        def mydate(x, pos):
+            try:
+                return df.index[int(x)]
+            except IndexError:
+                return ''
+
+        ax.xaxis.set_major_formatter(ticker.FuncFormatter(mydate))
+        plt.margins(0)
+        plt.show()
+
+    def plot(self, filename):
+        df = self.chunk_to_df(filename)
+        self.plot_ohlcv(df)
+
+    def to_csv(self, filename):
+        df = self.chunk_to_df(filename)
+        df.to_csv(os.path.basename(filename).split('.')[0] + '.csv')
+
+
+v = ValidateChunks()
+
+df = v.chunk_to_df(
+    exchange_name='bitfinex',
+    symbol='eth_btc',
+    data_frequency='daily',
+    period='2016'
+)
+print(df.tail())
+v.plot_ohlcv(df)
+# v.plot(
+#     ex
+# )

tests/exchange/test_utils.py

@@ -0,0 +1,17 @@
+from datetime import timedelta
+from random import randint
+
+import pandas as pd
+
+
+def rnd_history_date_days(max_days=30):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_days)
+
+    return now - timedelta(days=days)
+
+
+def rnd_bar_count(max_bars=21):
+    now = pd.Timestamp.utcnow()
+
+    return randint(0, max_bars)