BUG: fixed a get_candles issue with the Poloniex exchange

Fully tested on v0.3.8

catalyst/__main__.py

@@ -9,6 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_utils import delete_algo_folder
 from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
@@ -490,8 +491,19 @@ def live(ctx,
     default=True,
     help='Print progress information to the terminal.'
 )
+@click.option(
+    '--verbose/--no-verbose`',
+    default=False,
+    help='Show a progress indicator for every currency pair.'
+)
+@click.option(
+    '--validate/--no-validate`',
+    default=False,
+    help='Report potential anomalies found in data bundles.'
+)
 def ingest_exchange(exchange_name, data_frequency, start, end,
-                    include_symbols, exclude_symbols, show_progress):
+                    include_symbols, exclude_symbols, show_progress, verbose,
+                    validate):
     """
     Ingest data for the given exchange.
     """
@@ -509,10 +521,26 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
         exclude_symbols=exclude_symbols,
         start=start,
         end=end,
-        show_progress=show_progress
+        show_progress=show_progress,
+        show_breakdown=verbose,
+        show_report=validate
     )
 
 
+@main.command(name='clean-algo')
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='The label of the algorithm to for which to clean the state.'
+)
+@click.pass_context
+def clean_algo(ctx, algo_namespace):
+    click.echo(
+        'Deleting the state folder of algo: {}...'.format(algo_namespace)
+    )
+    delete_algo_folder(algo_namespace)
+
+
 @main.command(name='clean-exchange')
 @click.option(
     '-x',

catalyst/constants.py

@@ -3,3 +3,7 @@
 import logbook
 
 LOG_LEVEL = logbook.INFO
+
+DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+
+AUTO_INGEST = False

catalyst/curate/poloniex.py

@@ -6,9 +6,8 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 
 DT_START        = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
-DT_END          = int(time.time())
+DT_END          = pd.to_datetime('today').value // 10 ** 9
-CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
+CSV_OUT_FOLDER  = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
-CSV_OUT_FOLDER  = '/Volumes/enigma/data/poloniex/'
 CONN_RETRIES    = 2
 
 logbook.StderrHandler().push_application()
@@ -27,13 +26,15 @@ class PoloniexCurator(object):
             try:
                 os.makedirs(CSV_OUT_FOLDER)
             except Exception as e:
-                log.error('Failed to create data folder: %s' % CSV_OUT_FOLDER)
+                log.error('Failed to create data folder: {}'.format(
+                            CSV_OUT_FOLDER))
                 log.exception(e)
 
+
+    def get_currency_pairs(self):
         '''
         Retrieves and returns all currency pairs from the exchange 
         '''
-    def get_currency_pairs(self):
         url = self._api_path + 'command=returnTicker'
 
         try:
@@ -49,89 +50,136 @@ class PoloniexCurator(object):
             self.currency_pairs.append(ticker)
         self.currency_pairs.sort()
 
-        log.debug('Currency pairs retrieved successfully: %d' % (len(self.currency_pairs)))
+        log.debug('Currency pairs retrieved successfully: {}'.format(
+                    len(self.currency_pairs)
+                    ))
 
 
+
+    def _retrieve_tradeID_date(self, row):
         '''
         Helper function that reads tradeID and date fields from CSV readline
         '''
-    def _retrieve_tradeID_date(self, row):
         tId = int(row.split(',')[0])
-        d   = pd.to_datetime( row.split(',')[1], infer_datetime_format=True).value // 10 ** 9
+        d   = pd.to_datetime(row.split(',')[1], 
+                             infer_datetime_format=True).value // 10 ** 9
         return tId, d
 
+
+    def retrieve_trade_history(self, currencyPair, start=DT_START, 
+                               end=DT_END, temp=None):
         '''
-        Retrieves TradeHistory from exchange for a given currencyPair between start and end dates.
+        Retrieves TradeHistory from exchange for a given currencyPair 
-        If no start date is provided, uses a system-wide one (beginning of time for cryptotrading)
+        between start and end dates. If no start date is provided, uses 
-        If no end date is provided, 'now' is used
+        a system-wide one (beginning of time for cryptotrading).
+        If no end date is provided, 'now' is used.
+
         Stores results in CSV file on disk.
-        This function is called recursively to work around the limitations imposed by the provider API.
+        
+        This function is called recursively to work around the 
+        limitations imposed by the provider API.
         '''
-    def retrieve_trade_history(self, currencyPair, start=DT_START, end=DT_END, temp=None):
         csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
 
         '''
-            Check what data we already have on disk, reading first and last lines from file.
+        Check what data we already have on disk, reading first and last 
-            Data is stored on file from NEWEST to OLDEST.
+        lines from file. Data is stored on file from NEWEST to OLDEST.
         '''
         try:
             with open(csv_fn, 'ab+') as f: 
                 f.seek(0, os.SEEK_END)
-                if(f.tell() > 2):                           # First check file is not zero size
+                if(f.tell() > 2):                   # Check file size is not 0
-                    f.seek(0)                               # Go to the beginning to read first line
+                    f.seek(0)                       # Go to start to read 
                     last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
-                    f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
+                    f.seek(-2, os.SEEK_END)         # Jump to the 2nd last byte
                     while f.read(1) != b"\n":       # Until EOL is found...
                         f.seek(-2, os.SEEK_CUR)     # ...jump back the read byte plus one more.
                     first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
 
-                    if( first_tradeID == 1 and end_file + 3600 > DT_END ):
+                    if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
+                        or (currencyPair == 'BTC_HUC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_RIC' and first_tradeID ==     2) 
+                        or (currencyPair == 'BTC_XCP' and first_tradeID ==     2)
+                        or (currencyPair == 'BTC_NAV' and first_tradeID ==  4569)
+                        or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
                         return
 
         except Exception as e:
-            log.error('Error opening file: %s' % csv_fn)
+            log.error('Error opening file: {}'.format(csv_fn))
             log.exception(e)
 
         '''
-        Poloniex API limits querying TradeHistory to intervals smaller than 1 month,
+        Poloniex API limits querying TradeHistory to intervals smaller 
-        so we make sure that start date is never more than 1 month apart from end date
+        than 1 month, so we make sure that start date is never more than
+        1 month apart from end date
         '''
         if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days     
             newstart = end - 2419200
         else:
             newstart = start
 
-        log.debug(currencyPair+': Retrieving from '+str(newstart)+' to '+str(end) +'\t '
+        log.debug('{}: Retrieving from {} to {}\t {} - {}'.format(
-                    + time.ctime(newstart) + ' - '+ time.ctime(end))
+                    currencyPair, str(newstart), str(end),
+                    time.ctime(newstart), time.ctime(end)))
 
-        url = self._api_path + 'command=returnTradeHistory&currencyPair=' + currencyPair + '&start=' + str(newstart) + '&end=' + str(end)
+        url = '{path}command=returnTradeHistory&currencyPair={pair}' \
+              '&start={start}&end={end}'.format(
+                    path = self._api_path,
+                    pair = currencyPair,
+                    start = str(newstart),
+                    end = str(end)
+                )
+        print url
 
+        attempts = 0
+        success = 0
+        while attempts < CONN_RETRIES:
             try:
                 response = requests.get(url)
             except Exception as e:
-            log.error('Failed to retrieve trade history data for %s' % currencyPair)
+                log.error('Failed to retrieve trade history data for {}'.format(
+                            currencyPair
+                            ))
                 log.exception(e)
-            return None
+                attempts += 1
             else:
+                try:
                     if isinstance(response.json(), dict) and response.json()['error']:
-                log.error('Failed to to retrieve trade history data for %s: %s' % (currencyPair,response.json()['error']))
+                        log.error('Failed to to retrieve trade history data '
-                exit(1)
+                                  'for {}: {}'.format(
+                                    currencyPair,
+                                    response.json()['error']
+                                    ))
+                        attempts += 1
+                except Exception as e:
+                    log.exception(e)
+                    attempts += 1
+                else:
+                    success = 1
+                    break
+
+        if not success:
+            return None
+
 
         '''
-            If we get to transactionId == 1, and we already have that on disk, 
+        If we get to transactionId == 1, and we already have that on 
-            we got to the end of TradeHistory for this coin.
+        disk, we got to the end of TradeHistory for this coin.
         '''
-        if('first_tradeID' in locals() and response.json()[-1]['tradeID'] == first_tradeID): 
+        if('first_tradeID' in locals() 
+                and response.json()[-1]['tradeID'] == first_tradeID): 
             return
 
         '''
             There are primarily two scenarios:
-                a) There is newer data available that we need to add at the beginning
+                a) There is newer data available that we need to add at 
-                   of the file. We'll retrieve all what we need until we get to what 
+                   the beginning of the file. We'll retrieve all what we 
-                   we already have, writing it to a temporary file; and we will write
+                   need until we get to what we already have, writing it 
-                   that at the beginning of our existing file.
+                   to a temporary file; and we will write that at the 
-                b) We are going back in time, appending at the end of our existing
+                   beginning of our existing file.
-                   TradeHistory until the first transaction for this currencyPair
+                b) We are going back in time, appending at the end of 
+                   our existing TradeHistory until the first transaction 
+                   for this currencyPair
         '''
         try: 
             if( 'end_file' in locals() and end_file + 3600 < end):
@@ -151,8 +199,10 @@ class PoloniexCurator(object):
                         item['globalTradeID']        
                     ])
                 if( response.json()[-1]['tradeID'] > last_tradeID ):
-                    end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
+                    end = pd.to_datetime( response.json()[-1]['date'], 
-                    self.retrieve_trade_history(currencyPair, start, end, temp=temp) 
+                                infer_datetime_format=True).value // 10 ** 9
+                    self.retrieve_trade_history(currencyPair, start, 
+                                                end, temp=temp) 
                 else:
                     with open(csv_fn,'rb+') as f:
                         shutil.copyfileobj(f,temp)
@@ -165,7 +215,8 @@ class PoloniexCurator(object):
                 with open(csv_fn, 'ab') as csvfile:
                     csvwriter = csv.writer(csvfile)
                     for item in response.json():
-                        if( 'first_tradeID' in locals() and item['tradeID'] >= first_tradeID ):
+                        if( 'first_tradeID' in locals() 
+                                and item['tradeID'] >= first_tradeID ):
                             continue
                         csvwriter.writerow([
                             item['tradeID'],
@@ -176,48 +227,67 @@ class PoloniexCurator(object):
                             item['total'],
                             item['globalTradeID']
                         ])
-                end = pd.to_datetime( response.json()[-1]['date'], infer_datetime_format=True).value // 10 ** 9
+                end = pd.to_datetime(response.json()[-1]['date'], 
+                            infer_datetime_format=True).value // 10 ** 9
 
         except Exception as e:
-            log.error('Error opening %s' % csv_fn)
+            log.error('Error opening {}'.format(csv_fn))
             log.exception(e)
 
         '''
-            If we got here, we aren't done yet. Call recursively with 'end' times
+            If we got here, we aren't done yet. Call recursively with 
-            that go sequentially back in time.
+            'end' times that go sequentially back in time.
         '''
         self.retrieve_trade_history(currencyPair, start, end)
 
 
+
+    def generate_ohlcv(self, df):
         '''
         Generates OHLCV dataframe from a dataframe containing all TradeHistory
         by resampling with 1-minute period
         '''
-    def generate_ohlcv(self, df):
         df.set_index('date', inplace=True)               # Index by date
-        vol = df['total'].to_frame('volume')                    # Will deal with vol separately, as ohlc() messes it up
+        vol = df['total'].to_frame('volume')             # set Vol aside
-        df.drop('total', axis=1, inplace=True)                  # Drop volume data from dataframe
+        df.drop('total', axis=1, inplace=True)           # Drop volume data 
-        ohlc = df.resample('T').ohlc()                          # Resample OHLC in 1min bins
+        ohlc = df.resample('T').ohlc()                   # Resample OHLC 1min
         ohlc.columns = ohlc.columns.map(lambda t: t[1])  # Raname columns by dropping 'rate'
-        closes = ohlc['close'].fillna(method='pad')             # Pad forward missing 'close'
+        closes = ohlc['close'].fillna(method='pad')      # Pad fwd missing 'close'
         ohlc = ohlc.apply(lambda x: x.fillna(closes))    # Fill N/A with last close
         vol = vol.resample('T').sum().fillna(0)          # Add volumes by bin
-        ohlcv = pd.concat([ohlc,vol], axis=1)                   # Concatenate OHLC + Volume
+        ohlcv = pd.concat([ohlc,vol], axis=1)            # Concatenate OHLC + Vol
         return ohlcv
 
 
+
+    def write_ohlcv_file(self, currencyPair): 
         '''
         Generates OHLCV data file with 1minute bars from TradeHistory on disk
         '''       
-    def write_ohlcv_file(self, currencyPair):        
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
         csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        #if( os.path.isfile(csv_1min) ):
+        if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
-        #    log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
+            log.debug(currencyPair+': 1min data file already up to date. '
-        #else:
+                      'Delete the file if you want to rebuild it.')
-        df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
+        else:
-                dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
+            df = pd.read_csv(csv_trades, 
-        df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
+                                names=['tradeID',
+                                       'date',
+                                       'type',
+                                       'rate',
+                                       'amount',
+                                       'total',
+                                       'globalTradeID'], 
+                                dtype = {'tradeID': int, 
+                                         'date': str, 
+                                         'type': str, 
+                                         'rate': float, 
+                                         'amount': float, 
+                                         'total': float, 
+                                         'globalTradeID': int } 
+                                )
+            df.drop(['tradeID','type','amount','globalTradeID'], 
+                    axis=1, inplace=True)
             df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
             ohlcv = self.generate_ohlcv(df)
             try: 
@@ -235,25 +305,34 @@ class PoloniexCurator(object):
                             item.volume,
                         ])
             except Exception as e:
-            log.error('Error opening %s' % csv_fn)
+                log.error('Error opening {}'.format(csv_fn))
                 log.exception(e)
-        log.debug(currencyPair+': Generated 1min OHLCV data.')
+            log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
 
 
+
+    def onemin_to_dataframe(self, currencyPair, start, end):
         '''
         Returns a data frame for a given currencyPair from data on disk
         '''
-    def onemin_to_dataframe(self, currencyPair, start, end):
         csv_fn     = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        df         = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
+        df         = pd.read_csv(csv_fn, names=['date', 
+                                                'open', 
+                                                'high', 
+                                                'low', 
+                                                'close', 
+                                                'volume']
+                                )
         df['date'] = pd.to_datetime(df['date'],unit='s')
         df.set_index('date', inplace=True)
         return df[start : end]
 
-    '''
+
-    Generates a symbols.json file with corresponding start_date for each currencyPair
-    '''
     def generate_symbols_json(self, filename=None):
+        '''
+        Generates a symbols.json file with corresponding start_date 
+        for each currencyPair
+        '''
         symbol_map = {}
 
         if(filename is None):
@@ -262,14 +341,16 @@ class PoloniexCurator(object):
         with open(filename, 'w') as symbols:
             for currencyPair in self.currency_pairs:
                 start = None
-                csv_fn     = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
+                csv_fn     = '{}crypto_trades-{}.csv'.format(
+                                CSV_OUT_FOLDER, currencyPair)
                 with open(csv_fn, 'r') as f: 
                     f.seek(0, os.SEEK_END)
-                    if(f.tell() > 2):                           # First check file is not zero size
+                    if(f.tell() > 2):               # Check file size is not 0
-                        f.seek(-2, os.SEEK_END)                 # Jump to the second last byte.
+                        f.seek(-2, os.SEEK_END)     # Jump to 2nd last byte
                         while f.read(1) != b"\n":   # Until EOL is found...
                             f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
-                        start = pd.to_datetime( f.readline().split(',')[1], infer_datetime_format=True)
+                        start = pd.to_datetime( f.readline().split(',')[1], 
+                                                infer_datetime_format=True)
 
                 if(start is None):
                     start = time.gmtime()
@@ -279,7 +360,8 @@ class PoloniexCurator(object):
                     symbol = symbol,
                     start_date = start.strftime("%Y-%m-%d")
                 )
-            json.dump(symbol_map, symbols, sort_keys=True, indent=2, separators=(',',':'))    
+            json.dump(symbol_map, symbols, sort_keys=True, indent=2, 
+                        separators=(',',':'))    
 
 
 if __name__ == '__main__':
@@ -289,6 +371,6 @@ if __name__ == '__main__':
     
     for currencyPair in pc.currency_pairs:
         pc.retrieve_trade_history(currencyPair)
+        log.debug('{} up to date.'.format(currencyPair))
         pc.write_ohlcv_file(currencyPair)
 
-    

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -1,173 +0,0 @@
-import talib
-from logbook import Logger
-import pandas as pd
-
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.exchange.stats_utils import get_pretty_stats
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_low_sell_high_neo'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.asset = symbol('neo_btc', 'bitfinex')
-
-    context.TARGET_POSITIONS = 50000
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    price = data.current(context.asset, 'close')
-    log.info('got price {price}'.format(price=price))
-
-    if price is None:
-        log.warn('no pricing data')
-        return
-
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=1,
-        frequency='1m'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 1
-    elif rsi <= 40:
-        buy_increment = 0.5
-    elif rsi <= 70:
-        buy_increment = 0.1
-    else:
-        buy_increment = None
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    record(price=price)
-
-    orders = get_open_orders(context.asset)
-    if len(orders) > 0:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        limit_price = price * (1 + context.SLIPPAGE_ALLOWED)
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=limit_price
-        )
-        pass
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
-
-    pass
-
-
-# run_algorithm(
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=analyze,
-#     exchange_name='bitfinex',
-#     live=True,
-#     algo_namespace=algo_namespace,
-#     base_currency='btc',
-#     live_graph=False
-# )
-
-# Backtest
-run_algorithm(
-    capital_base=250,
-    data_frequency='minute',
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    algo_namespace=algo_namespace,
-    base_currency='btc'
-)

catalyst/examples/mean_reversion.py

@@ -0,0 +1,283 @@
+# For this example, we're going to write a simple momentum script.  When the 
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+from datetime import timedelta
+
+import pandas as pd
+import talib
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+from logbook import Logger
+from talib.common import MA_Type
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, \
+    get_open_orders
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.exchange.stats_utils import extract_transactions, trend_direction
+
+algo_namespace = 'momentum'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Ether in USD Tether.
+    context.eth_btc = symbol('etc_usdt')
+    context.base_price = None
+    context.current_day = None
+    context.trigger = None
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.eth_btc variable.  For this example, we're 
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.eth_btc,
+        fields='close',
+        bar_count=50,
+        frequency='15T'
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+    upper, middle, lower = talib.BBANDS(
+        prices.values,
+        timeperiod=20,
+        nbdevup=2,
+        nbdevdn=2,
+        matype=MA_Type.EMA
+    )
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.eth_btc, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        volume=current['volume'],
+        upper_band=upper[-1],
+        lower_band=lower[-1],
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.eth_btc)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.eth_btc):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.   
+    pos_amount = context.portfolio.positions[context.eth_btc].amount
+
+    # In this example, we're using a trigger instead of buying directly after
+    # a signal. Since this is mean reversion, our signals go against the
+    # momentum. Using a trigger allow us to spot the opportunity but trade
+    # only when a trade reversal begins.
+    if context.trigger is not None:
+        # The tread_direction() method determines the trend based on the last
+        # two bars of the series.
+        direction = trend_direction(rsi)
+        if context.trigger[1] == 'buy' and direction == 'up':
+            log.info(
+                '{}: buying - price: {}, rsi: {}, bband: {}'.format(
+                    data.current_dt, price, rsi[-1], lower[-1]
+                )
+            )
+            order_target_percent(context.eth_btc, 1)
+            context.traded_today = True
+            context.trigger = None
+
+        elif context.trigger[1] == 'sell' and direction == 'down':
+            log.info(
+                '{}: selling - price: {}, rsi: {}, bband: {}'.format(
+                    data.current_dt, price, rsi[-1], upper[-1]
+                )
+            )
+            order_target_percent(context.eth_btc, 0)
+            context.traded_today = True
+            context.trigger = None
+
+        # If we found a signal but no trade reversal within two hours, we
+        # reset the trigger.
+        elif context.trigger[0] + timedelta(hours=2) < data.current_dt:
+            context.trigger = None
+
+    else:
+        # Determining the entry and exit signals based on RSI and SMA
+        if rsi[-1] <= 30 and pos_amount == 0:
+            context.trigger = (data.current_dt, 'buy')
+
+        elif rsi[-1] >= 80 and pos_amount > 0:
+            context.trigger = (data.current_dt, 'sell')
+
+
+def analyze(context=None, perf=None):
+    import matplotlib.pyplot as plt
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+    perf.loc[:, 'upper_band'].plot(ax=ax2, label='Upper')
+    perf.loc[:, 'lower_band'].plot(ax=ax2, label='Lower')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.eth_btc.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.axhline(70, color='darkgoldenrod')
+    ax6.axhline(30, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'backtest'
+
+    if MODE == 'backtest':
+        run_algorithm(
+            capital_base=1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            start=pd.to_datetime('2017-7-1', utc=True),
+            # end=pd.to_datetime('2017-9-30', utc=True),
+            end=pd.to_datetime('2017-10-31', utc=True),
+        )
+
+    elif MODE == 'live':
+        run_algorithm(
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            live_graph=True
+        )

catalyst/examples/mean_reversion_simple.py

@@ -0,0 +1,248 @@
+# For this example, we're going to write a simple momentum script.  When the 
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+from datetime import timedelta
+
+import pandas as pd
+import talib
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+from logbook import Logger
+from talib.common import MA_Type
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, \
+    get_open_orders
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.exchange.stats_utils import extract_transactions, trend_direction
+
+algo_namespace = 'momentum'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Ether in USD Tether.
+    context.eth_btc = symbol('etc_usdt')
+    context.base_price = None
+    context.current_day = None
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.eth_btc variable.  For this example, we're 
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.eth_btc,
+        fields='close',
+        bar_count=50,
+        frequency='15T'
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.eth_btc, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+    record(
+        price=price,
+        volume=current['volume'],
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.eth_btc)
+    if len(orders) > 0:
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.eth_btc):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.   
+    pos_amount = context.portfolio.positions[context.eth_btc].amount
+
+    if rsi[-1] <= 30 and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        order_target_percent(context.eth_btc, 1)
+        context.traded_today = True
+
+    elif rsi[-1] >= 80 and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        order_target_percent(context.eth_btc, 0)
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    import matplotlib.pyplot as plt
+
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.eth_btc.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash ({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.axhline(70, color='darkgoldenrod')
+    ax6.axhline(30, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index, 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index, 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    MODE = 'backtest'
+
+    if MODE == 'backtest':
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py -x poloniex -s 2017-7-1 -e 2017-7-31 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=10000,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            start=pd.to_datetime('2017-7-1', utc=True),
+            end=pd.to_datetime('2017-7-31', utc=True),
+        )
+
+    elif MODE == 'live':
+        run_algorithm(
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=algo_namespace,
+            base_currency='usdt',
+            live_graph=True
+        )

catalyst/examples/rsi_profit_target.py

@@ -0,0 +1,276 @@
+from datetime import timedelta
+
+import pandas as pd
+import numpy as np
+import talib
+from logbook import Logger
+
+from catalyst.api import (
+    order,
+    symbol,
+    record,
+    get_open_orders,
+)
+from catalyst.exchange.stats_utils import crossover, crossunder
+from catalyst.utils.run_algo import run_algorithm
+
+algo_namespace = 'rsi'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('initializing algo')
+    context.asset = symbol('eth_btc')
+    context.base_price = None
+
+    context.MAX_HOLDINGS = 0.2
+    context.RSI_OVERSOLD = 30
+    context.RSI_OVERSOLD_BBANDS = 45
+    context.RSI_OVERBOUGHT_BBANDS = 55
+    context.SLIPPAGE_ALLOWED = 0.03
+
+    context.TARGET = 0.15
+    context.STOP_LOSS = 0.1
+    context.STOP = 0.03
+    context.position = None
+
+    context.last_bar = None
+
+    context.errors = []
+    pass
+
+
+def _handle_buy_sell_decision(context, data, signal, price):
+    orders = get_open_orders(context.asset)
+    if len(orders) > 0:
+        log.info('skipping bar until all open orders execute')
+        return
+
+    positions = context.portfolio.positions
+    if context.position is None and context.asset in positions:
+        position = positions[context.asset]
+        context.position = dict(
+            cost_basis=position['cost_basis'],
+            amount=position['amount'],
+            stop=None
+        )
+
+    action = None
+    if context.position is not None:
+        cost_basis = context.position['cost_basis']
+        amount = context.position['amount']
+        log.info(
+            'found {amount} positions with cost basis {cost_basis}'.format(
+                amount=amount,
+                cost_basis=cost_basis
+            )
+        )
+        stop = context.position['stop']
+
+        target = cost_basis * (1 + context.TARGET)
+        if price >= target:
+            context.position['cost_basis'] = price
+            context.position['stop'] = context.STOP
+
+        stop_target = context.STOP_LOSS if stop is None else context.STOP
+        if price < cost_basis * (1 - stop_target):
+            log.info('executing stop loss')
+            order(
+                asset=context.asset,
+                amount=-amount,
+                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+            action = 0
+            context.position = None
+
+    else:
+        if signal == 'long':
+            log.info('opening position')
+            buy_amount = context.MAX_HOLDINGS / price
+            order(
+                asset=context.asset,
+                amount=buy_amount,
+                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
+            )
+            context.position = dict(
+                cost_basis=price,
+                amount=buy_amount,
+                stop=None
+            )
+            action = 0
+
+
+def _handle_data_rsi_only(context, data):
+    price = data.current(context.asset, 'close')
+    log.info('got price {price}'.format(price=price))
+
+    if price is np.nan:
+        log.warn('no pricing data')
+        return
+
+    if context.base_price is None:
+        context.base_price = price
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=17,
+            frequency='30T'
+        )
+    except Exception as e:
+        log.warn('historical data not available: '.format(e))
+        return
+
+    rsi = talib.RSI(prices.values, timeperiod=16)[-1]
+    log.info('got rsi {}'.format(rsi))
+
+    signal = None
+    if rsi < context.RSI_OVERSOLD:
+        signal = 'long'
+
+    # Making sure that the price is still current
+    price = data.current(context.asset, 'close')
+    cash = context.portfolio.cash
+    log.info(
+        'base currency available: {cash}, cap: {cap}'.format(
+            cash=cash,
+            cap=context.MAX_HOLDINGS
+        )
+    )
+    volume = data.current(context.asset, 'volume')
+    price_change = (price - context.base_price) / context.base_price
+    record(
+        price=price,
+        price_change=price_change,
+        rsi=rsi,
+        volume=volume,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+    )
+
+    _handle_buy_sell_decision(context, data, signal, price)
+
+
+def handle_data(context, data):
+    dt = data.current_dt
+
+    if context.last_bar is None or (
+                context.last_bar + timedelta(minutes=15)) <= dt:
+        context.last_bar = dt
+    else:
+        return
+
+    log.info('BAR {}'.format(dt))
+    try:
+        _handle_data_rsi_only(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    results.loc[:, 'price'].plot(ax=ax2)
+    ax2.set_ylabel('{asset} ({base})'.format(
+        asset=context.asset.symbol, base=base_currency
+    ))
+
+    trans = results.loc[[t != [] for t in results.transactions], :]
+    buys = trans.loc[[t[0]['amount'] > 0 for t in trans.transactions], :]
+    sells = trans.loc[[t[0]['amount'] < 0 for t in trans.transactions], :]
+    # buys = results.loc[results['action'] == 1, :]
+    # sells = results.loc[results['action'] == 0, :]
+
+    ax2.plot(
+        buys.index,
+        results.loc[buys.index, 'price'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax2.plot(
+        sells.index,
+        results.loc[sells.index, 'price'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results.loc[:, ['alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Alpha / Beta ')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results.loc[:, ['starting_cash', 'cash']].plot(ax=ax4)
+    ax4.set_ylabel('Base Currency ({})'.format(base_currency))
+
+    results['algorithm'] = results.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results.loc[:, 'rsi'].plot(ax=ax6)
+    ax6.set_ylabel('RSI')
+
+    ax6.plot(
+        buys.index,
+        results.loc[buys.index, 'rsi'],
+        '^',
+        markersize=10,
+        color='g',
+    )
+    ax6.plot(
+        sells.index,
+        results.loc[sells.index, 'rsi'],
+        'v',
+        markersize=10,
+        color='r',
+    )
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+run_algorithm(
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=analyze,
+    exchange_name='bittrex',
+    live=True,
+    algo_namespace=algo_namespace,
+    base_currency='btc',
+    live_graph=False
+)
+
+# Backtest
+# run_algorithm(
+#     capital_base=0.5,
+#     data_frequency='minute',
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     algo_namespace=algo_namespace,
+#     base_currency='btc',
+#     start=pd.to_datetime('2017-9-1', utc=True),
+#     end=pd.to_datetime('2017-10-1', utc=True),
+# )

catalyst/examples/simple_loop.py

@@ -7,7 +7,7 @@ from catalyst.api import symbol
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('eth_btc')
+    context.asset = symbol('swift_btc')
 
 
 def handle_data(context, data):
@@ -20,8 +20,8 @@ def handle_data(context, data):
         prices = data.history(
             context.asset,
             fields='price',
-            bar_count=16,
+            bar_count=15,
-            frequency='5T'
+            frequency='1D'
         )
         rsi = talib.RSI(prices.values, timeperiod=14)[-1]
         print('got rsi: {}'.format(rsi))
@@ -31,13 +31,13 @@ def handle_data(context, data):
 
 run_algorithm(
     capital_base=250,
-    start=pd.to_datetime('2017-1-1', utc=True),
+    start=pd.to_datetime('2015-4-1', utc=True),
-    end=pd.to_datetime('2017-10-22', utc=True),
+    end=pd.to_datetime('2017-11-1', utc=True),
-    data_frequency='minute',
+    data_frequency='daily',
     initialize=initialize,
     handle_data=handle_data,
     analyze=None,
-    exchange_name='bitfinex',
+    exchange_name='bittrex',
     algo_namespace='simple_loop',
     base_currency='btc'
 )
@@ -50,4 +50,3 @@ run_algorithm(
 #     algo_namespace='simple_loop',
 #     base_currency='eth',
 #     live_graph=False
-# )

catalyst/examples/simple_universe.py

@@ -0,0 +1,129 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.3
+
+These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
+You simply need to specify the exchange and the market that you want to focus on.
+You will all see how to create a universe and filter it base on the exchange and the market you desire.
+
+The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
+The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
+Use this as the backbone to create your own trading strategies.
+
+Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
+"""
+
+import numpy as np
+import pandas as pd
+from datetime import timedelta
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = context.exchanges.values()[0].name.lower()  # exchange name
+    context.base_currency = context.exchanges.values()[0].base_currency.lower()  # market base currency
+
+
+def handle_data(context, data):
+    context.i += 1
+    lookback_days = 7  # 7 days
+
+    # current date formatted into a string
+    today = data.current_dt
+    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = today - timedelta(days=lookback_days)  # subtract the amount of days specified in lookback
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  # get only the date as a string
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+    one_day_in_minutes = 1440  # 1440 assumes data_frequency='minute'
+    lookback = one_day_in_minutes / minutes * lookback_days  # get N lookback_days of history data
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
+            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
+            opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
+            high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
+            low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
+            close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
+            volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
+
+            # close[-1] is the equivalent to current price
+            # displays the minute price for each pair every 30 minutes
+            print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
+
+            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
+            # ----------------------------------------------------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    json_symbols = get_exchange_symbols(context.exchange)  # get all the pairs for the exchange
+    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)  # convert into a dataframe
+    universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
+                                                                       axis=1)
+    universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
+                                                                         axis=1)
+
+    # Filter all the exchange pairs to only the ones for a give base currency
+    universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
+
+    # Filter all the pairs to ensure that pair existed in the current date range
+    universe_df = universe_df[universe_df.start_date < lookback_date]
+    universe_df = universe_df[universe_df.end_daily >= current_date]
+    context.coins = symbols(*universe_df.symbol)  # convert all the pairs to symbols
+
+    # print(universe_df.symbol.tolist())
+    return universe_df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-01-01', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=100.0,  # amount of base_currency, not always in dollars unless usd
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')
+
+"""
+Run in Terminal (inside catalyst environment):
+python simple_universe.py
+"""

catalyst/exchange/bittrex/bittrex.py

@@ -1,7 +1,7 @@
 import json
+import time
 
 import pandas as pd
-import time
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib

catalyst/exchange/bittrex/bittrex_api.py

@@ -3,11 +3,12 @@ import json
 import time
 import hmac
 import hashlib
-
+import ssl
 
 # Workaround for backwards compatibility
 # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
 from six.moves import urllib
+
 urlopen = urllib.request.urlopen
 
 
@@ -48,7 +49,8 @@ class Bittrex_api(object):
             headers = {}
 
         req = urllib.request.Request(url, headers=headers)
-        response = json.loads(urlopen(req).read())
+        response = json.loads(urlopen(
+            req, context=ssl._create_unverified_context()).read())
 
         if response["result"]:
             return response["result"]

catalyst/exchange/bundle_utils.py

@@ -149,7 +149,7 @@ def get_periods(start_dt, end_dt, freq):
     return len(get_periods_range(start_dt, end_dt, freq))
 
 
-def get_start_dt(end_dt, bar_count, data_frequency):
+def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
     """
     The start date based on specified end date and data frequency.
 
@@ -168,6 +168,9 @@ def get_start_dt(end_dt, bar_count, data_frequency):
     if periods > 1:
         delta = get_delta(periods, data_frequency)
         start_dt = end_dt - delta
+
+        if not include_first:
+            start_dt += get_delta(1, data_frequency)
     else:
         start_dt = end_dt
 
@@ -223,6 +226,9 @@ def get_month_start_end(dt, first_day=None, last_day=None):
             dt.year, dt.month, month_range[1], 23, 59, 0, 0
         ), utc=True)
 
+        if month_end > pd.Timestamp.utcnow():
+            month_end = pd.Timestamp.utcnow().floor('1D')
+
     return month_start, month_end
 
 
@@ -247,6 +253,9 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     year_end = last_day if last_day \
         else pd.to_datetime(date(dt.year, 12, 31), utc=True)
 
+    if year_end > pd.Timestamp.utcnow():
+        year_end = pd.Timestamp.utcnow().floor('1D')
+
     return year_start, year_end
 
 
@@ -294,24 +303,17 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
 
     """
     has_data = True
-    if has_data and reader is not None:
+    dates = [start_dt, end_dt]
+
+    while dates and has_data:
         try:
-            start_close = \
+            dt = dates.pop(0)
-                reader.get_value(asset.sid, start_dt, 'close')
+            close = reader.get_value(asset.sid, dt, 'close')
 
-            if np.isnan(start_close):
+            if np.isnan(close):
-                has_data = False
-
-            else:
-                end_close = reader.get_value(asset.sid, end_dt, 'close')
-
-                if np.isnan(end_close):
                 has_data = False
 
         except Exception as e:
             has_data = False
 
-    else:
-        has_data = False
-
     return has_data

catalyst/exchange/exchange_algorithm.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import pickle
 import signal
 import sys
@@ -27,8 +26,6 @@ from catalyst.assets._assets import TradingPair
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
 from catalyst.constants import LOG_LEVEL
-from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
-    BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
 from catalyst.exchange.exchange_blotter import ExchangeBlotter
 from catalyst.exchange.exchange_errors import (
@@ -38,8 +35,8 @@ from catalyst.exchange.exchange_errors import (
     OrphanOrderError)
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
+from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
-    save_algo_object, get_algo_object, get_algo_folder, get_algo_df, \
+    get_algo_folder, get_algo_df, \
     save_algo_df
 from catalyst.exchange.live_graph_clock import LiveGraphClock
 from catalyst.exchange.simple_clock import SimpleClock
@@ -182,17 +179,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
 
         # we want the key to be absent, not just empty
         # Only include transactions for given dt
-        stats['transactions'] = dict()
+        stats['transactions'] = []
         for date in period.processed_transactions:
             if start_dt <= date < end_dt:
-                stats['transactions'][date] = \
+                transactions = period.processed_transactions[date]
-                    period.processed_transactions[date]
+                for t in transactions:
+                    stats['transactions'].append(t.to_dict())
 
-        stats['orders'] = dict()
+        stats['orders'] = []
         for date in period.orders_by_modified:
             if start_dt <= date < end_dt:
-                stats['orders'][date] = \
+                orders = period.orders_by_modified[date]
-                    period.orders_by_modified[date]
+                for order in orders:
+                    stats['orders'].append(orders[order].to_dict())
 
         return stats
 
@@ -201,6 +200,7 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
         super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
 
+        self.frame_stats = list()
         self.blotter = ExchangeBlotter(
             data_frequency=self.data_frequency,
             # Default to NeverCancel in catalyst
@@ -245,6 +245,19 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
         else:
             return MarketOrder()
 
+    def handle_data(self, data):
+        super(ExchangeTradingAlgorithmBacktest, self).handle_data(data)
+
+        minute_stats = self.prepare_period_stats(
+            data.current_dt, data.current_dt + timedelta(minutes=1))
+        self.frame_stats.append(minute_stats)
+
+    def analyze(self, perf):
+        stats = pd.DataFrame(self.frame_stats)
+        stats.set_index('period_close', inplace=True, drop=False)
+
+        super(ExchangeTradingAlgorithmBacktest, self).analyze(stats)
+
 
 class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
     def __init__(self, *args, **kwargs):
@@ -273,34 +286,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.stats_minutes = 5
 
         super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
-        # TODO: fix precision before re-enabling
-        # self._create_minute_writer()
 
         signal.signal(signal.SIGINT, self.signal_handler)
 
         log.info('initialized trading algorithm in live mode')
 
-    def _create_minute_writer(self):
-        root = get_exchange_minute_writer_root(self.exchange.name)
-        filename = os.path.join(root, 'metadata.json')
-
-        if os.path.isfile(filename):
-            writer = BcolzMinuteBarWriter.open(
-                root, self.sim_params.end_session)
-        else:
-            # TODO: need to be able to write more precise numbers
-            writer = BcolzMinuteBarWriter(
-                rootdir=root,
-                calendar=self.trading_calendar,
-                minutes_per_day=1440,
-                start_session=self.sim_params.start_session,
-                end_session=self.sim_params.end_session,
-                write_metadata=True
-            )
-
-        self.exchange.minute_writer = writer
-        self.exchange.minute_reader = BcolzMinuteBarReader(root)
-
     def signal_handler(self, signal, frame):
         """
         Handles the keyboard interruption signal.

catalyst/exchange/exchange_bundle.py

@@ -1,21 +1,24 @@
 import os
-import os
 import shutil
+from functools import partial
 from itertools import chain
+from operator import is_not
 
+import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
+from datetime import datetime, timedelta
 from logbook import Logger
-from pandas.tslib import Timestamp
 from pytz import UTC
 from six import itervalues
 
 from catalyst import get_calendar
+from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_delta, get_month_start_end, \
+    get_bcolz_chunk, get_month_start_end, \
     get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
@@ -218,8 +221,91 @@ class ExchangeBundle:
             if data_frequency == 'minute' \
             else self.calendar.sessions_in_range(start_dt, end_dt)
 
+    def _spot_empty_periods(self, ohlcv_df, asset, data_frequency,
+                            empty_rows_behavior):
+        problems = []
+
+        nan_rows = ohlcv_df[ohlcv_df.isnull().T.any().T].index
+        if len(nan_rows) > 0:
+            dates = []
+            for row_date in nan_rows.values:
+                row_date = pd.to_datetime(row_date, utc=True)
+                if row_date > asset.start_date:
+                    dates.append(row_date)
+
+            if len(dates) > 0:
+                end_dt = asset.end_minute if data_frequency == 'minute' \
+                    else asset.end_daily
+
+                problem = '{name} ({start_dt} to {end_dt}) has empty ' \
+                          'periods: {dates}'.format(
+                    name=asset.symbol,
+                    start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                    dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
+                )
+                if empty_rows_behavior == 'warn':
+                    log.warn(problem)
+
+                elif empty_rows_behavior == 'raise':
+                    raise EmptyValuesInBundleError(
+                        name=asset.symbol,
+                        end_minute=end_dt,
+                        dates=dates
+                    )
+
+                else:
+                    ohlcv_df.dropna(inplace=True)
+
+            else:
+                problem = None
+
+            problems.append(problem)
+
+        return problems
+
+    def _spot_duplicates(self, ohlcv_df, asset, data_frequency, threshold):
+        # TODO: work in progress
+        series = ohlcv_df.reset_index().groupby('close')['index'].apply(
+            np.array
+        )
+
+        ref_delta = timedelta(minutes=1) if data_frequency == 'minute' \
+            else timedelta(days=1)
+
+        dups = series.loc[lambda values: [len(x) > 10 for x in values]]
+
+        for index, dates in dups.iteritems():
+            prev_date = None
+            for date in dates:
+                if prev_date is not None:
+                    delta = (date - prev_date) / 1e9
+                    if delta == ref_delta.seconds:
+                        log.info('pex')
+
+                prev_date = date
+
+        problems = []
+        for index, dates in dups.iteritems():
+            end_dt = asset.end_minute if data_frequency == 'minute' \
+                else asset.end_daily
+
+            problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
+                      'identical close values on: {dates}'.format(
+                name=asset.symbol,
+                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                threshold=threshold,
+                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                       for date in dates]
+            )
+
+            problems.append(problem)
+
+        return problems
+
     def ingest_df(self, ohlcv_df, data_frequency, asset, writer,
-                  empty_rows_behavior='strip'):
+                  empty_rows_behavior='warn', duplicates_threshold=None):
         """
         Ingest a DataFrame of OHLCV data for a given market.
 
@@ -232,50 +318,16 @@ class ExchangeBundle:
         empty_rows_behavior: str
 
         """
+        problems = []
         if empty_rows_behavior is not 'ignore':
-            nan_rows = ohlcv_df[ohlcv_df.isnull().T.any().T].index
+            problems += self._spot_empty_periods(
-
+                ohlcv_df, asset, data_frequency, empty_rows_behavior
-            if len(nan_rows) > 0:
-                dates = []
-                previous_date = None
-                for row_date in nan_rows.values:
-                    row_date = pd.to_datetime(row_date)
-
-                    if previous_date is None:
-                        dates.append(row_date)
-
-                    else:
-                        seq_date = previous_date + get_delta(1, data_frequency)
-
-                        if row_date > seq_date:
-                            dates.append(previous_date)
-                            dates.append(row_date)
-
-                    previous_date = row_date
-
-                dates.append(pd.to_datetime(nan_rows.values[-1]))
-
-                name = '{} from {} to {}'.format(
-                    asset.symbol, ohlcv_df.index[0], ohlcv_df.index[-1]
-                )
-                if empty_rows_behavior == 'warn':
-                    log.warn(
-                        '\n{name} with end minute {end_minute} has empty rows '
-                        'in ranges: {dates}'.format(
-                            name=name,
-                            end_minute=asset.end_minute,
-                            dates=dates
-                        )
             )
 
-                elif empty_rows_behavior == 'raise':
+        # if duplicates_threshold is not None:
-                    raise EmptyValuesInBundleError(
+        #     problems += self._spot_duplicates(
-                        name=name,
+        #         ohlcv_df, asset, data_frequency, duplicates_threshold
-                        end_minute=asset.end_minute,
+        #     )
-                        dates=dates
-                    )
-                else:
-                    ohlcv_df.dropna(inplace=True)
 
         data = []
         if not ohlcv_df.empty:
@@ -284,8 +336,11 @@ class ExchangeBundle:
 
         self._write(data, writer, data_frequency)
 
+        return problems
+
     def ingest_ctable(self, asset, data_frequency, period,
-                      writer, empty_rows_behavior='strip', cleanup=False):
+                      writer, empty_rows_behavior='strip',
+                      duplicates_threshold=100, cleanup=False):
         """
         Merge a ctable bundle chunk into the main bundle for the exchange.
 
@@ -301,9 +356,15 @@ class ExchangeBundle:
         cleanup: bool
             Remove the temp bundle directory after ingestion.
 
-        :return:
+        Returns
-        """
+        -------
+        list[str]
+            A list of problems which occurred during ingestion.
 
+        """
+        problems = []
+
+        # Download and extract the bundle
         path = get_bcolz_chunk(
             exchange_name=self.exchange.name,
             symbol=asset.symbol,
@@ -313,6 +374,14 @@ class ExchangeBundle:
 
         reader = self.get_reader(data_frequency, path=path)
         if reader is None:
+            try:
+                log.warn('the reader is unable to use bundle: {}, '
+                         'deleting it.'.format(path))
+                shutil.rmtree(path)
+
+            except Exception as e:
+                log.warn('unable to remove temp bundle: {}'.format(e))
+
             raise TempBundleNotFoundError(path=path)
 
         start_dt = reader.first_trading_day
@@ -335,27 +404,29 @@ class ExchangeBundle:
             ))
 
         if not arrays:
-            return path
+            return reader._rootdir
 
         periods = self.get_calendar_periods_range(
             start_dt, end_dt, data_frequency
         )
         df = get_df_from_arrays(arrays, periods)
-        self.ingest_df(
+        problems += self.ingest_df(
             ohlcv_df=df,
             data_frequency=data_frequency,
             asset=asset,
             writer=writer,
-            empty_rows_behavior=empty_rows_behavior
+            empty_rows_behavior=empty_rows_behavior,
+            duplicates_threshold=duplicates_threshold
         )
 
         if cleanup:
             log.debug(
-                'removing bundle folder following ingestion: {}'.format(path)
+                'removing bundle folder following ingestion: {}'.format(
+                    reader._rootdir)
             )
-            shutil.rmtree(path)
+            shutil.rmtree(reader._rootdir)
 
-        return path
+        return filter(partial(is_not, None), problems)
 
     def get_adj_dates(self, start, end, assets, data_frequency):
         """
@@ -400,10 +471,10 @@ class ExchangeBundle:
         if end is None or (last_entry is not None and end > last_entry):
             end = last_entry
 
-        if end is None or start is None or start >= end:
+        if end is None or start is None or start > end:
             raise NoDataAvailableOnExchange(
-                exchange=asset.exchange.title(),
+                exchange=[asset.exchange for asset in assets],
-                symbol=[asset.symbol],
+                symbol=[asset.symbol for asset in assets],
                 data_frequency=data_frequency,
             )
 
@@ -429,9 +500,7 @@ class ExchangeBundle:
         get_start_end = get_month_start_end \
             if data_frequency == 'minute' else get_year_start_end
 
-        start_dt, _ = get_start_end(start_dt)
+        # Get a reader for the main bundle to verify if data exists
-        _, end_dt = get_start_end(end_dt)
-
         reader = self.get_reader(data_frequency)
 
         chunks = dict()
@@ -462,7 +531,6 @@ class ExchangeBundle:
 
             chunks[asset] = []
             for index, dt in enumerate(dates):
-
                 period_start, period_end = get_start_end(
                     dt=dt,
                     first_day=dt if index == 0 else None,
@@ -481,42 +549,49 @@ class ExchangeBundle:
                     asset, range_start, period_end, reader
                 )
                 if not has_data:
-                    chunks[asset].append(
-                        dict(
-                            asset=asset,
-                            period_start=period_start,
-                            period_end=period_end,
                     period = get_period_label(dt, data_frequency)
+                    chunk = dict(
+                        asset=asset,
+                        period=period,
                     )
-                    )
+                    chunks[asset].append(chunk)
 
             # We sort the chunks by end date to ingest most recent data first
-            chunks[asset].sort(key=lambda chunk: chunk['period_end'])
+            chunks[asset].sort(
+                key=lambda chunk: pd.to_datetime(chunk['period'])
+            )
 
         return chunks
 
     def ingest_assets(self, assets, data_frequency, start_dt=None, end_dt=None,
-                      show_progress=False, asset_chunks=False):
+                      show_progress=False, show_breakdown=False,
+                      show_report=False):
         """
         Determine if data is missing from the bundle and attempt to ingest it.
 
         Parameters
         ----------
         assets: list[TradingPair]
+        data_frequency: str
         start_dt: datetime
         end_dt: datetime
+        show_progress: bool
+        show_breakdown: bool
 
         """
-
         if start_dt is None:
             start_dt = self.calendar.first_session
 
         if end_dt is None:
             end_dt = pd.Timestamp.utcnow()
 
-        start_dt, end_dt = self.get_adj_dates(
+        get_start_end = get_month_start_end \
-            start_dt, end_dt, assets, data_frequency
+            if data_frequency == 'minute' else get_year_start_end
-        )
+
+        # Assign the first and last day of the period
+        start_dt, _ = get_start_end(start_dt)
+        _, end_dt = get_start_end(end_dt)
+
         chunks = self.prepare_chunks(
             assets=assets,
             data_frequency=data_frequency,
@@ -524,20 +599,11 @@ class ExchangeBundle:
             end_dt=end_dt
         )
 
-        # Since chunks are either monthly or yearly, it is possible that
+        problems = []
-        # our ingestion data range is greater than specified. We adjust
+        # This is the common writer for the entire exchange bundle
-        # the boundaries to ensure that the writer can write all data.
+        # we want to give an end_date far in time
-        all_chunks = list(chain.from_iterable(itervalues(chunks)))
-        for chunk in all_chunks:
-            if chunk['period_start'] < start_dt:
-                start_dt = chunk['period_start']
-
-            if chunk['period_end'] > end_dt:
-                end_dt = chunk['period_end']
-
         writer = self.get_writer(start_dt, end_dt, data_frequency)
-
+        if show_breakdown:
-        if asset_chunks:
             for asset in chunks:
                 with maybe_show_progress(
                         chunks[asset],
@@ -549,7 +615,7 @@ class ExchangeBundle:
                             symbol=asset.symbol
                         )) as it:
                     for chunk in it:
-                        self.ingest_ctable(
+                        problems += self.ingest_ctable(
                             asset=chunk['asset'],
                             data_frequency=data_frequency,
                             period=chunk['period'],
@@ -558,6 +624,12 @@ class ExchangeBundle:
                             cleanup=True
                         )
         else:
+            all_chunks = list(chain.from_iterable(itervalues(chunks)))
+
+            # We sort the chunks by end date to ingest most recent data first
+            all_chunks.sort(
+                key=lambda chunk: pd.to_datetime(chunk['period'])
+            )
             with maybe_show_progress(
                     all_chunks,
                     show_progress,
@@ -567,7 +639,7 @@ class ExchangeBundle:
                         frequency=data_frequency,
                     )) as it:
                 for chunk in it:
-                    self.ingest_ctable(
+                    problems += self.ingest_ctable(
                         asset=chunk['asset'],
                         data_frequency=data_frequency,
                         period=chunk['period'],
@@ -576,9 +648,14 @@ class ExchangeBundle:
                         cleanup=True
                     )
 
+        if show_report and len(problems) > 0:
+            log.info('problems during ingestion:{}\n'.format(
+                '\n'.join(problems)
+            ))
+
     def ingest(self, data_frequency, include_symbols=None,
                exclude_symbols=None, start=None, end=None,
-               show_progress=True, environ=os.environ):
+               show_progress=True, show_breakdown=True, show_report=True):
         """
         Inject data based on specified parameters.
 
@@ -597,15 +674,15 @@ class ExchangeBundle:
 
         for frequency in data_frequency.split(','):
             self.ingest_assets(assets, frequency, start, end,
-                               show_progress)
+                               show_progress, show_breakdown, show_report)
 
     def get_history_window_series_and_load(self,
-                                           assets,  # type: List[TradingPair]
+                                           assets,
-                                           end_dt,  # type: Timestamp
+                                           end_dt,
-                                           bar_count,  # type: int
+                                           bar_count,
-                                           field,  # type: str
+                                           field,
-                                           data_frequency,  # type: str
+                                           data_frequency,
-                                           algo_end_dt=None  # type: Timestamp
+                                           algo_end_dt=None
                                            ):
         """
         Retrieve price data history, ingest missing data.
@@ -624,6 +701,7 @@ class ExchangeBundle:
         Series
 
         """
+        if AUTO_INGEST:
             try:
                 series = self.get_history_window_series(
                     assets=assets,
@@ -650,7 +728,7 @@ class ExchangeBundle:
                     end_dt=algo_end_dt,
                     data_frequency=data_frequency,
                     show_progress=True,
-                asset_chunks=True
+                    show_breakdown=True
                 )
                 series = self.get_history_window_series(
                     assets=assets,
@@ -658,28 +736,43 @@ class ExchangeBundle:
                     bar_count=bar_count,
                     field=field,
                     data_frequency=data_frequency,
-                reset_reader=False
+                    reset_reader=True
                 )
                 return series
 
+        else:
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency
+            )
+            return pd.DataFrame(series)
+
     def get_spot_values(self,
-                        assets,  # type: List[TradingPair]
+                        assets,
-                        field,  # type: str
+                        field,
-                        dt,  # type: Timestamp
+                        dt,
-                        data_frequency,  # type: str
+                        data_frequency,
-                        reset_reader=False  # type: bool
+                        reset_reader=False
                         ):
-        # type: (...) -> List[float]
         """
         The spot values for the gives assets, field and date. Reads from
         the exchange data bundle.
 
-        :param assets:
+        Parameters
-        :param field:
+        ----------
-        :param dt:
+        assets: list[TradingPair]
-        :param data_frequency:
+        field: str
-        :param reset_reader:
+        dt: pd.Timestamp
-        :return:
+        data_frequency: str
+        reset_reader:
+
+        Returns
+        -------
+        float
+
         """
         values = []
         try:
@@ -706,7 +799,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=dt,
+                end_dt=dt
             )
 
     def get_history_window_series(self,
@@ -716,7 +811,7 @@ class ExchangeBundle:
                                   field,
                                   data_frequency,
                                   reset_reader=False):
-        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
         start_dt, end_dt = self.get_adj_dates(
             start_dt, end_dt, assets, data_frequency
         )
@@ -734,7 +829,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=start_dt,
+                end_dt=end_dt
             )
 
         for asset in assets:
@@ -752,7 +849,9 @@ class ExchangeBundle:
                     exchange=self.exchange.name,
                     symbols=asset.symbol,
                     symbol_list=asset.symbol,
-                    data_frequency=data_frequency
+                    data_frequency=data_frequency,
+                    start_dt=asset_start_dt,
+                    end_dt=asset_end_dt
                 )
 
         series = dict()
@@ -772,7 +871,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=start_dt,
+                end_dt=end_dt
             )
 
         periods = self.get_calendar_periods_range(
@@ -788,6 +889,14 @@ class ExchangeBundle:
         return series
 
     def clean(self, data_frequency):
+        """
+        Removing the bundle data from the catalyst folder.
+
+        Parameters
+        ----------
+        data_frequency: str
+
+        """
         log.debug('cleaning exchange {}, frequency {}'.format(
             self.exchange.name, data_frequency
         ))

catalyst/exchange/exchange_data_portal.py

@@ -6,7 +6,7 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
+from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
@@ -378,9 +378,11 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         else:
             dt = dt.floor('1 min')
 
+        if AUTO_INGEST:
             try:
-            return bundle.get_spot_values(assets, field, dt, data_frequency)
+                return bundle.get_spot_values(
-
+                    assets, field, dt, data_frequency
+                )
             except PricingDataNotLoadedError:
                 log.info(
                     'pricing data for {symbol} not found on {dt}'
@@ -399,3 +401,5 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
                 return bundle.get_spot_values(
                     assets, field, dt, data_frequency, True
                 )
+        else:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -211,12 +211,11 @@ class PricingDataBeforeTradingError(ZiplineError):
 
 
 class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Pricing data {field} for trading pairs {symbols} trading on '
+    msg = ('Missing data for {exchange} {symbols} in date range '
-           'exchange {exchange} since {first_trading_day} is unavailable. '
+           '[{start_dt} - {end_dt}]'
-           'The bundle data is either out-of-date or has not been loaded yet. '
+           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
-           'Please ingest data using the command '
+           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
-           '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. '
+           'for details.').strip()
-           'See catalyst documentation for details.').strip()
 
 
 class ApiCandlesError(ZiplineError):

catalyst/exchange/exchange_utils.py

@@ -2,6 +2,7 @@ import json
 import os
 import pickle
 import re
+import shutil
 from datetime import date, datetime
 
 import pandas as pd
@@ -158,6 +159,24 @@ def get_exchange_auth(exchange_name, environ=None):
             return data
 
 
+def delete_algo_folder(algo_name, environ=None):
+    """
+    Delete the folder containing the algo state.
+
+    Parameters
+    ----------
+    algo_name: str
+    environ:
+
+    Returns
+    -------
+    str
+
+    """
+    folder = get_algo_folder(algo_name, environ)
+    shutil.rmtree(folder)
+
+
 def get_algo_folder(algo_name, environ=None):
     """
     The algorithm root folder of the algorithm.
@@ -284,13 +303,12 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
     ----------
     algo_name: str
     key: str
-    df: DataFrame
+    df: pd.DataFrame
     environ:
     rel_path: str
 
     """
     folder = get_algo_folder(algo_name, environ)
-
     if rel_path is not None:
         folder = os.path.join(folder, rel_path)
         ensure_directory(folder)

catalyst/exchange/poloniex/poloniex.py

@@ -1,4 +1,3 @@
-import calendar
 import json
 import json
 import time
@@ -227,10 +226,9 @@ class Poloniex(Exchange):
         ohlc_map = dict()
 
         for asset in asset_list:
+            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
+            end = int(delta.total_seconds())
 
-            # TODO: what's wrong with this?
-            # end = int(time.mktime(end_dt.timetuple()))
-            end = int(time.time())
             if bar_count is None:
                 start = end - 2 * frequency
             else:

catalyst/exchange/poloniex/poloniex_api.py

@@ -3,6 +3,7 @@ import json
 import time
 import hmac
 import hashlib
+import ssl
 
 from six.moves import urllib
 
@@ -104,9 +105,10 @@ class Poloniex_api(object):
         req = urllib.request.Request(
             url,
             data=post_data,
-            headers=headers
+            headers=headers,
         )
-        return json.loads(urlopen(req).read())
+        return json.loads(
+            urlopen(req, context=ssl._create_unverified_context()).read())
 
     def returnticker(self):
         return self.query('returnTicker', {})

catalyst/exchange/stats_utils.py

@@ -1,7 +1,19 @@
+import numbers
+
 import numpy as np
 import pandas as pd
 
 
+def trend_direction(series):
+    if series[-1] is np.nan or series[-1] is np.nan:
+        return None
+
+    if series[-1] > series[-2]:
+        return 'up'
+    else:
+        return 'down'
+
+
 def crossover(source, target):
     """
     The `x`-series is defined as having crossed over `y`-series if the value
@@ -44,16 +56,58 @@ def crossunder(source, target):
     bool
 
     """
+    if isinstance(target, numbers.Number):
+        if source[-1] is np.nan or source[-2] is np.nan \
+                or target is np.nan:
+            return False
+
+        if source[-1] < target <= source[-2]:
+            return True
+        else:
+            return False
+    else:
         if source[-1] is np.nan or source[-2] is np.nan \
                 or target[-1] is np.nan or target[-2] is np.nan:
             return False
 
-    if source[-1] < target[-1] and source[-2] > target[-2]:
+        if source[-1] < target[-1] and source[-2] >= target[-2]:
             return True
         else:
             return False
 
 
+def vwap(df):
+    """
+    Volume-weighted average price (VWAP) is a ratio generally used by
+    institutional investors and mutual funds to make buys and sells so as not
+    to disturb the market prices with large orders. It is the average share
+    price of a stock weighted against its trading volume within a particular
+    time frame, generally one day.
+
+    Read more: Volume Weighted Average Price - VWAP
+    https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+
+    Returns
+    -------
+
+    """
+    if 'close' not in df.columns or 'volume' not in df.columns:
+        raise ValueError('price data must include `volume` and `close`')
+
+    vol_sum = np.nansum(df['volume'].values)
+
+    try:
+        ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
+    except ZeroDivisionError:
+        ret = np.nan
+
+    return ret
+
+
 def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
     """
     Format and print the last few rows of a statistics DataFrame.
@@ -129,3 +183,28 @@ def df_to_string(df):
     pd.set_option('display.max_colwidth', 1000)
 
     return df.to_string()
+
+
+def extract_transactions(perf):
+    """
+    Compute indexes for buy and sell transactions
+
+    Parameters
+    ----------
+    perf: DataFrame
+        The algo performance DataFrame.
+
+    Returns
+    -------
+    DataFrame
+        A DataFrame of transactions.
+
+    """
+    trans_list = perf.transactions.values
+    all_trans = [t for sublist in trans_list for t in sublist]
+    all_trans.sort(key=lambda t: t['dt'])
+
+    transactions = pd.DataFrame(all_trans)
+    if not transactions.empty:
+        transactions.set_index('dt', inplace=True, drop=True)
+    return transactions

catalyst/finance/execution.py

@@ -77,6 +77,7 @@ class LimitOrder(ExecutionStyle):
     Execution style representing an order to be executed at a price equal to or
     better than a specified limit price.
     """
+
     def __init__(self, limit_price, exchange=None):
         """
         Store the given price.
@@ -99,6 +100,7 @@ class StopOrder(ExecutionStyle):
     Execution style representing an order to be placed once the market price
     reaches a specified stop price.
     """
+
     def __init__(self, stop_price, exchange=None):
         """
         Store the given price.
@@ -121,6 +123,7 @@ class StopLimitOrder(ExecutionStyle):
     Execution style representing a limit order to be placed with a specified
     limit price once the market reaches a specified stop price.
     """
+
     def __init__(self, limit_price, stop_price, exchange=None):
         """
         Store the given prices
@@ -144,31 +147,20 @@ class StopLimitOrder(ExecutionStyle):
 def asymmetric_round_price_to_penny(price, prefer_round_down,
                                     diff=(0.0095 - .005)):
     """
-    Asymmetric rounding function for adjusting prices to two places in a way
+    Modified the original function because we do not want to round
-    that "improves" the price.  For limit prices, this means preferring to
+    prices on crypto exchange.
-    round down on buys and preferring to round up on sells.  For stop prices,
-    it means the reverse.
 
-    If prefer_round_down == True:
+    Parameters
-        When .05 below to .95 above a penny, use that penny.
+    ----------
-    If prefer_round_down == False:
+    price: float
-        When .95 below to .05 above a penny, use that penny.
+
+    Returns
+    -------
+    float
 
-    In math-speak:
-    If prefer_round_down: [<X-1>.0095, X.0195) -> round to X.01.
-    If not prefer_round_down: (<X-1>.0005, X.0105] -> round to X.01.
     """
-    # Subtracting an epsilon from diff to enforce the open-ness of the upper
+    # TODO: consider overriding outside of the original function
-    # bound on buys and the lower bound on sells.  Using the actual system
+    return price
-    # epsilon doesn't quite get there, so use a slightly less epsilon-ey value.
-    epsilon = float_info.epsilon * 10
-    diff = diff - epsilon
-
-    # relies on rounding half away from zero, unlike numpy's bankers' rounding
-    rounded = round(price - (diff if prefer_round_down else -diff), 2)
-    if zp_math.tolerant_equals(rounded, 0.0):
-        return 0.0
-    return rounded
 
 
 def check_stoplimit_prices(price, label):

catalyst/finance/risk/cumulative.py

@@ -37,12 +37,11 @@ from empyrical import (
     sharpe_ratio,
     sortino_ratio,
 )
-
+import warnings
 from catalyst.constants import LOG_LEVEL
 
 log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
-
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',
                                     compound=False)
 
@@ -145,6 +144,8 @@ class RiskMetricsCumulative(object):
         self.num_trading_days = 0
 
     def update(self, dt, algorithm_returns, benchmark_returns, leverage):
+        warnings.filterwarnings('error')
+
         # Keep track of latest dt for use in to_dict and other methods
         # that report current state.
         self.latest_dt = dt
@@ -191,9 +192,12 @@ class RiskMetricsCumulative(object):
             if len(self.benchmark_returns) == 1:
                 self.benchmark_returns = np.append(0.0, self.benchmark_returns)
 
+        try:
             self.benchmark_cumulative_returns[dt_loc] = cum_returns(
                 self.benchmark_returns
             )[-1]
+        except Exception as e:
+            log.debug('cumulative returns error: {}'.format(e))
 
         benchmark_cumulative_returns_to_date = \
             self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -268,10 +272,15 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.downside_risk[dt_loc] = downside_risk(
             self.algorithm_returns
         )
+
+        try:
             self.sortino[dt_loc] = sortino_ratio(
                 self.algorithm_returns,
                 _downside_risk=self.downside_risk[dt_loc]
             )
+        except Exception as e:
+            log.debug('sortino ratio error: {}'.format(e))
+
         self.information[dt_loc] = information_ratio(
             self.algorithm_returns,
             self.benchmark_returns,
@@ -283,6 +292,8 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
         self.max_leverage = self.calculate_max_leverage()
         self.max_leverages[dt_loc] = self.max_leverage
 
+        warnings.resetwarnings()
+
     def to_dict(self):
         """
         Creates a dictionary representing the state of the risk report.

catalyst/support/issue_55.py

@@ -0,0 +1,42 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('xcp_btc')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=1,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2015-3-2', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='poloniex',
+    algo_namespace='issue_55',
+    base_currency='btc'
+)

catalyst/support/issue_57.py

@@ -0,0 +1,46 @@
+import talib
+import pandas as pd
+
+from catalyst import run_algorithm
+from catalyst.api import symbol
+
+
+def initialize(context):
+    print('initializing')
+    context.asset = symbol('btc_usdt')
+
+
+def handle_data(context, data):
+    print('handling bar: {}'.format(data.current_dt))
+
+    price = data.current(context.asset, 'close')
+    print('got price {price}'.format(price=price))
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='close',
+            bar_count=60,
+            frequency='1D'
+        )
+        print('got {} price entries\n'.format(len(prices), prices))
+    except Exception as e:
+        print(e)
+
+
+run_algorithm(
+    capital_base=1,
+    start=pd.to_datetime('2016-2-11', utc=True),
+    end=pd.to_datetime('2017-8-31', utc=True),
+    data_frequency='daily',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=None,
+    exchange_name='bittrex',
+    algo_namespace='issue_57',
+    base_currency='btc'
+<<<<<<< HEAD
+)
+=======
+)
+>>>>>>> develop

docs/source/releases.rst

@@ -2,24 +2,90 @@
 Release Notes
 =============
 
-Version 0.3.4
+Version 0.3.8
 ^^^^^^^^^^^^^
-**Release Date**: 2017-10-31
+**Release Date**: 2017-11-14
 
 Bug Fixes
 ~~~~~~~~~
 
-- Fixed issue with auto-ingestion of minute data
+- Fixed a warning filter issue introduced with the latest release
+
+Version 0.3.7
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-14
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an SSL cert issue (:issue:`64`)
+- Fixed cumulative stats warnings (:issue:`63`)
+- Disabled auto-ingestion because of unresolved caching issues (:issue:`47`)
+- Standardized live-trading stats (:issue:`61`)
+
+Build
+~~~~~
+
+- Added a mean-reversion sample algo
+- Added minutely stats in the analyze() function (:issue:`62`)
+- Added specificity to some error messages
+
+Version 0.3.6
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed an issue with single bar data.history() (:issue:`55`)
+
+Version 0.3.5
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-4
+
+Bug Fixes
+~~~~~~~~~
+
+- Added workaround for: KeyError: Timestamp error (:issue:`53`)
+
+Version 0.3.4
+^^^^^^^^^^^^^
+**Release Date**: 2017-11-2
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed issue with auto-ingestion of minute data (:issue:`47`)
 - Fixed issue with sell orders in backtesting
 - Fixed data frequency issues with data.history() in backtesting
 - Fixed an issue with can_trade()
-
+- Reduced the commission and slippage values to account for lower volume 
+  transactions
 
 Build
 ~~~~~
 
 - Added more unit tests
 
+Documentation
+~~~~~~~~~~~~~
+
+- Improved installation notes for Windows C++ compiler and Conda
+- Addition of 
+  `Jupyter Notebook guide <https://enigmampc.github.io/catalyst/jupyter.html>`_
+- Addition of 
+  `Live Trading page <https://enigmampc.github.io/catalyst/live-trading.html>`_
+- Addition of 
+  `Videos page <https://enigmampc.github.io/catalyst/videos.html>`_
+- Addition of 
+  `Resources page <https://enigmampc.github.io/catalyst/resources.html>`_
+- Addition of `Development Guidelines 
+  <https://enigmampc.github.io/catalyst/development-guidelines.html>`_
+- Addition of 
+  `Release Notes <https://enigmampc.github.io/catalyst/releases.html>`_
+- Updated code docstrings
+
+
 Version 0.3.3
 ^^^^^^^^^^^^^
 **Release Date**: 2017-10-26
@@ -66,9 +132,11 @@ Bug Fixes
 ~~~~~~~~~
 
 - Fixed OS-dependent path issue in data bundle
-- Changed handling of empty ``auth.json``, instead of throwing an error for missing file
+- Changed handling of empty ``auth.json``, instead of throwing an error for 
+  missing file
 - Updated ``etc/python2.7-environment.yml`` to work with Catalyst version 0.3
-- Updated ``catalyst/examples/buy_and_hodl.py``  and ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
+- Updated ``catalyst/examples/buy_and_hodl.py``  and 
+  ``catalyst/examples/buy_low_sell_high.py`` to work with Catalyst version 0.3
 
 
 Version 0.3
@@ -87,15 +155,19 @@ Version 0.2.dev5
 ^^^^^^^^^^^^^^^^
 **Release Date**: 2017-10-03
 
-- Fixes bug in data.history function that was formatting 'volume' data as integers, now they are returned as floats with up to 9 decimals of precision. Data bundles redone.
+- Fixes bug in data.history function that was formatting 'volume' data as 
+  integers, now they are returned as floats with up to 9 decimals of precision. 
+  Data bundles redone.
 
 Version 0.2.dev4 
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-09-20
 
-- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal places. Pricing resolution of daily data remains set to 9 decimal places.
+- Fixes bug in the pricing resolution of 1-minute data, now set to 8 decimal 
-- The current data bundle takes 340MB compressed for download, and 460MB uncompressed on disk for Catalyst to use.
+  places. Pricing resolution of daily data remains set to 9 decimal places.
+- The current data bundle takes 340MB compressed for download, and 460MB 
+  uncompressed on disk for Catalyst to use.
 
 Version 0.2.dev3
 ^^^^^^^^^^^^^^^^
@@ -104,9 +176,12 @@ Version 0.2.dev3
 
 - 1-minute resolution OHLCV data bundle for backtesting from Poloniex exchange
 - Implementation of trading of fractional crypto assets (i.e. 0.01 BTC)
-- Minimum trade size of a coin can be configured on a per-coin basis, defaults to 0.00000001 in backtesting (most exchanges set the minimum trade to larger amounts, which will impact live trading)
+- Minimum trade size of a coin can be configured on a per-coin basis, defaults 
+  to 0.00000001 in backtesting (most exchanges set the minimum trade to larger 
+  amounts, which will impact live trading)
 - Increased pricing resolution from 3 to 9 decimal places
-- The current data bundle takes 40MB compressed for download, and 99MB uncompressed on disk for Catalyst to use.
+- The current data bundle takes 40MB compressed for download, and 99MB 
+  uncompressed on disk for Catalyst to use.
 
 Version 0.2.dev2 
 ^^^^^^^^^^^^^^^^
@@ -124,19 +199,24 @@ Version 0.2.dev1
 
   - Comprehensive trading functionality against exchanges Bitfinex and Bittrex.
   - Support for all trading pairs available on each exchange.
-  - Multiple algorithms can trade simultaneously against a single exchange using the same account.
+  - Multiple algorithms can trade simultaneously against a single exchange 
-  - Each algorithm has a persisted state (i.e. algorithm can be stopped and restarted preserving the state without data loss) that tracks all open orders, executed transactions and portfolio positions.
+    using the same account.
+  - Each algorithm has a persisted state (i.e. algorithm can be stopped and 
+    restarted preserving the state without data loss) that tracks all open 
+    orders, executed transactions and portfolio positions.
 
 - Minute by minute portfolio performance metrics.
 
-  - Daily summary performance statistics compatible with pyfolio, a Python library for performance and risk analysis of financial portfolios
+  - Daily summary performance statistics compatible with pyfolio, a Python 
+    library for performance and risk analysis of financial portfolios
 
 Version 0.1.dev9
 ^^^^^^^^^^^^^^^^
 
 **Release Date**: 2017-08-28
 
-- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex exchange directly
+- Retrieval of crypto benchmark from bundle, instead of hitting Poloniex 
+  exchange directly
 - Change of bundle storage provider from Dropbox to AWS
 - Fix issue with 1/1000 scaling issue of prices in bundle
 

tests/exchange/test_bundle.py

@@ -1,14 +1,13 @@
 import hashlib
+import os
 import tempfile
 from logging import getLogger
 
-import os
 import pandas as pd
 
 from catalyst import get_calendar
 from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
-    get_periods_range, get_start_dt, get_month_start_end, get_df_from_arrays, \
+    get_start_dt, get_df_from_arrays
-    get_year_start_end
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
@@ -43,17 +42,16 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
         assets = [
-            exchange.get_asset('xmr_btc')
+            exchange.get_asset('eth_btc')
         ]
 
-        # start = pd.to_datetime('2017-09-01', utc=True)
+        start = pd.to_datetime('2016-03-01', utc=True)
-        start = pd.to_datetime('2016-01-01', utc=True)
+        end = pd.to_datetime('2017-11-1', utc=True)
-        end = pd.to_datetime('2017-9-30', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -122,18 +120,20 @@ class TestExchangeBundle:
         pass
 
     def test_ingest_daily(self):
-        # exchange_name = 'bitfinex'
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+        include_symbols = 'neo_btc'
+
+        # exchange_name = 'poloniex'
         # data_frequency = 'daily'
-        # include_symbols = 'neo_btc,bch_btc,eth_btc'
+        # include_symbols = 'eth_btc'
 
-        exchange_name = 'poloniex'
+        # start = pd.to_datetime('2017-1-1', utc=True)
-        data_frequency = 'daily'
+        # end = pd.to_datetime('2017-10-16', utc=True)
-        include_symbols = 'eth_btc'
+        # periods = get_periods_range(start, end, data_frequency)
-
-        start = pd.to_datetime('2017-1-1', utc=True)
-        end = pd.to_datetime('2017-10-16', utc=True)
-        periods = get_periods_range(start, end, data_frequency)
 
+        start = None
+        end = None
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
 
@@ -153,12 +153,18 @@ class TestExchangeBundle:
             assets.append(exchange.get_asset(pair_symbol))
 
         reader = exchange_bundle.get_reader(data_frequency)
+        start_dt = reader.first_trading_day
+        end_dt = reader.last_available_dt
+
+        if data_frequency == 'daily':
+            end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
+
         for asset in assets:
             arrays = reader.load_raw_arrays(
                 sids=[asset.sid],
                 fields=['close'],
-                start_dt=start,
+                start_dt=start_dt,
-                end_dt=end
+                end_dt=end_dt
             )
             print('found {} rows for {} ingestion\n{}'.format(
                 len(arrays[0]), asset.symbol, arrays[0])
@@ -415,7 +421,8 @@ class TestExchangeBundle:
                 data_frequency=data_frequency,
                 asset=asset,
                 writer=writer,
-                empty_rows_behavior='raise'
+                empty_rows_behavior='raise',
+                duplicates_behavior='raise'
             )
 
         bundle_series = bundle.get_history_window_series(
@@ -430,23 +437,59 @@ class TestExchangeBundle:
         print('\n' + df_to_string(df))
         pass
 
-    def bundle_to_csv(self):
+    def main_bundle_to_csv(self):
         exchange_name = 'bitfinex'
-        data_frequency = 'daily'
+        data_frequency = 'minute'
-        period = '2016'
 
         exchange = get_exchange(exchange_name)
-        bundle = ExchangeBundle(exchange)
         asset = exchange.get_asset('eth_btc')
 
+        start_dt = pd.to_datetime('2016-5-31', utc=True)
+        end_dt = pd.to_datetime('2016-6-1', utc=True)
+        self._bundle_to_csv(
+            asset=asset,
+            exchange=exchange,
+            data_frequency=data_frequency,
+            filename='{}_{}_{}'.format(
+                exchange_name, data_frequency, asset.symbol
+            ),
+            start_dt=start_dt,
+            end_dt=end_dt
+        )
+
+    def bundle_to_csv(self):
+        exchange_name = 'poloniex'
+        data_frequency = 'minute'
+        period = '2017-09'
+        symbol = 'eth_btc'
+
+        exchange = get_exchange(exchange_name)
+        asset = exchange.get_asset(symbol)
+
         path = get_bcolz_chunk(
             exchange_name=exchange.name,
             symbol=asset.symbol,
             data_frequency=data_frequency,
             period=period
         )
+        self._bundle_to_csv(
+            asset=asset,
+            exchange=exchange,
+            data_frequency=data_frequency,
+            path=path,
+            filename=period
+        )
+        pass
+
+    def _bundle_to_csv(self, asset, exchange, data_frequency, filename,
+                       path=None, start_dt=None, end_dt=None):
+        bundle = ExchangeBundle(exchange)
         reader = bundle.get_reader(data_frequency, path=path)
+
+        if start_dt is None:
             start_dt = reader.first_trading_day
+
+        if end_dt is None:
             end_dt = reader.last_available_dt
 
         if data_frequency == 'daily':
@@ -475,7 +518,7 @@ class TestExchangeBundle:
         )
         ensure_directory(folder)
 
-        path = os.path.join(folder, period + '.csv')
+        path = os.path.join(folder, filename + '.csv')
 
         log.info('creating csv file: {}'.format(path))
         print('HEAD\n{}'.format(df.head(10)))

tests/exchange/test_data_portal.py

@@ -113,3 +113,40 @@ class TestExchangeDataPortal:
         )
 
         log.info('found history window: {}'.format(data))
+
+    def test_validate_resample(self):
+        symbol = ['eth_btc']
+        exchange_name = 'poloniex'
+        exchange = get_exchange(exchange_name, base_currency=symbol)
+
+        assets = exchange.get_assets(symbols=symbol)
+
+        date = rnd_history_date_days(
+            max_days=10,
+            last_dt=pd.to_datetime('2017-11-1', utc=True)
+        )
+        bar_count = rnd_bar_count(max_bars=10)
+        sample_minutes = 15
+        sample_data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count,
+            frequency='{}T'.format(sample_minutes),
+            field='close',
+            data_frequency='daily'
+        )
+        minute_data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count * sample_minutes,
+            frequency='1T',
+            field='close',
+            data_frequency='daily'
+        )
+        resampled_minute_data = minute_data.resample(
+            '{}T'.format(sample_minutes))
+
+        print(sample_data.tail(10))
+        print(resampled_minute_data.tail(10))
+        print(minute_data.tail(10))
+        pass

tests/exchange/test_poloniex.py

@@ -1,9 +1,10 @@
-from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.exchange.poloniex.poloniex import Poloniex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+import pandas as pd
+from test_utils import output_df
 
 log = Logger('test_poloniex')
 
@@ -51,18 +52,19 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
-            freq='5T',
         assets = self.exchange.get_asset('eth_btc')
+        ohlcv = self.exchange.get_candles(
+            end_dt=pd.to_datetime('2017-11-01', utc=True),
+            freq='30T',
+            assets=assets,
+            bar_count=200
         )
-        ohlcv_neo_ubq = self.exchange.get_candles(
+        df = pd.DataFrame(ohlcv)
-            freq='5T',
+        df.set_index('last_traded', drop=True, inplace=True)
-            assets=[
+        log.info(df.tail(25))
-                self.exchange.get_asset('neos_btc'),
+
-                self.exchange.get_asset('via_btc')
+        path = output_df(df, assets, 'candles')
-            ],
+        log.info('saved candles: {}'.format(path))
-            bar_count=14
-        )
         pass
 
     def test_tickers(self):

tests/exchange/test_utils.py

@@ -4,11 +4,20 @@ from random import randint
 import pandas as pd
 
 
-def rnd_history_date_days(max_days=30):
+def rnd_history_date_days(max_days=30, last_dt=None):
-    now = pd.Timestamp.utcnow()
+    if last_dt is None:
+        last_dt = pd.Timestamp.utcnow()
+
     days = randint(0, max_days)
 
-    return now - timedelta(days=days)
+    return last_dt - timedelta(days=days)
+
+
+def rnd_history_date_minutes(max_minutes=1440):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_minutes)
+
+    return now - timedelta(minutes=days)
 
 
 def rnd_bar_count(max_bars=21):