Merge branch 'develop' - Release 0.3.3

[MIG] Migrated buy_and_hodl and buy_low_sell_high to version 0.3 to work with Poloniex exchange

catalyst/__main__.py

@@ -38,7 +38,7 @@ except NameError:
     '--default-extension/--no-default-extension',
     is_flag=True,
     default=True,
-    help="Don't load the default catalyst extension.py file in $ZIPLINE_HOME.",
+    help="Don't load the default catalyst extension.py file in $CATALYST_HOME.",
 )
 @click.version_option()
 def main(extension, strict_extensions, default_extension):
@@ -495,10 +495,14 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     """
     Ingest data for the given exchange.
     """
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
     exchange = get_exchange(exchange_name)
     exchange_bundle = ExchangeBundle(exchange)
 
-    click.echo('ingesting exchange bundle {}'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,
@@ -509,6 +513,40 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     )
 
 
+@main.command(name='clean-exchange')
+@click.option(
+    '-x',
+    '--exchange-name',
+    type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
+    help='The name of the exchange bundle to ingest (supported: bitfinex,'
+         ' bittrex, poloniex).',
+)
+@click.option(
+    '-f',
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute'}),
+    default=None,
+    help='The bundle data frequency to remove. If not specified, it will '
+         'remove both daily and minute bundles.',
+)
+@click.pass_context
+def clean_exchange(ctx, exchange_name, data_frequency):
+    """Clean up bundles from 'ingest-exchange'.
+    """
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    exchange = get_exchange(exchange_name)
+    exchange_bundle = ExchangeBundle(exchange)
+
+    click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
+    exchange_bundle.clean(
+        data_frequency=data_frequency,
+    )
+    click.echo('Done')
+
+
 @main.command()
 @click.option(
     '-b',
@@ -598,7 +636,7 @@ def ingest(ctx, bundle, exchange_name, compile_locally, assets_version,
          ' This may not be passed with -e / --before or -a / --after',
 )
 def clean(bundle, before, after, keep_last):
-    """Clean up data downloaded with the ingest command.
+    """Clean up bundles from 'ingest'.
     """
     bundles_module.clean(
         bundle,

catalyst/algorithm.py

@@ -138,8 +138,9 @@ from catalyst.gens.sim_engine import MinuteSimulationClock
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
+from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger("ZiplineLog")
+log = logbook.Logger("CatalystLog", level=LOG_LEVEL)
 
 
 class TradingAlgorithm(object):

catalyst/assets/_assets.pyx

@@ -17,6 +17,8 @@
 """
 Cythonized Asset object.
 """
+import hashlib
+
 cimport cython
 from cpython.number cimport PyNumber_Index
 from cpython.object cimport (
@@ -501,7 +503,11 @@ cdef class TradingPair(Asset):
 
         if sid == 0 or sid is None:
             try:
-                sid = abs(hash(symbol)) % (10 ** 4)
+                # sid = abs(hash(symbol)) % (10 ** 4)
+                # TODO: try to encode the symbol in the main scope
+                sid = int(
+                    hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+                ) % 10 ** 6
             except Exception as e:
                 raise SidHashError(symbol=symbol)
 

catalyst/assets/assets.py

@@ -76,7 +76,9 @@ from catalyst.utils.numpy_utils import as_column
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.sqlite_utils import group_into_chunks, coerce_string_to_eng
 
-log = Logger('assets.py')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('assets.py', level=LOG_LEVEL)
 
 # A set of fields that need to be converted to strings before building an
 # Asset to avoid unicode fields

catalyst/constants.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+import logbook
+
+LOG_LEVEL = logbook.INFO

catalyst/curate/poloniex.py

@@ -212,32 +212,32 @@ class PoloniexCurator(object):
     def write_ohlcv_file(self, currencyPair):        
         csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
         csv_1min   = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
-        if( os.path.isfile(csv_1min) ):
+        #if( os.path.isfile(csv_1min) ):
-            log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
+        #    log.debug(currencyPair+': 1min data already present. Delete the file if you want to rebuild it.')
-        else:
+        #else:
-            df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
+        df = pd.read_csv(csv_trades, names=['tradeID','date','type','rate','amount','total','globalTradeID'], 
-                    dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
+                dtype = {'tradeID': int, 'date': str, 'type': str, 'rate': float, 'amount': float, 'total': float, 'globalTradeID': int } )
-            df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
+        df.drop(['tradeID','type','amount','globalTradeID'], axis=1, inplace=True)
-            df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
+        df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) 
-            ohlcv = self.generate_ohlcv(df)
+        ohlcv = self.generate_ohlcv(df)
-            try: 
+        try: 
-                with open(csv_1min, 'ab') as csvfile:
+            with open(csv_1min, 'w') as csvfile:
-                    csvwriter = csv.writer(csvfile)
+                csvwriter = csv.writer(csvfile)
-                    for item in ohlcv.itertuples():
+                for item in ohlcv.itertuples():
-                        if item.Index == 0:
+                    if item.Index == 0:
-                            continue
+                        continue
-                        csvwriter.writerow([
+                    csvwriter.writerow([
-                            item.Index.value // 10 ** 9,
+                        item.Index.value // 10 ** 9,
-                            item.open,
+                        item.open,
-                            item.high,
+                        item.high,
-                            item.low,
+                        item.low,
-                            item.close,
+                        item.close,
-                            item.volume,
+                        item.volume,
-                        ])
+                    ])
-            except Exception as e:
+        except Exception as e:
-                log.error('Error opening %s' % csv_fn)
+            log.error('Error opening %s' % csv_fn)
-                log.exception(e)
+            log.exception(e)
-            log.debug(currencyPair+': Generated 1min OHLCV data.')
+        log.debug(currencyPair+': Generated 1min OHLCV data.')
 
 
     '''

catalyst/data/_equities.pyx

@@ -215,7 +215,7 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in ['open', 'high', 'low', 'close']:
+        if column_name in ['open', 'high', 'low', 'close', 'volume']:
             where_nan = (outbuf == 0)
             outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN

catalyst/data/bundles/base.py

@@ -30,8 +30,10 @@ from catalyst.utils.cli import (
 )
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
+
 logbook.StderrHandler().push_application()
-log = logbook.Logger(__name__)
+log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 

catalyst/data/bundles/quandl.py

@@ -40,7 +40,9 @@ from catalyst.utils.cli import maybe_show_progress
 
 from . import core as bundles
 
-log = Logger(__name__)
+from catalyst.constants import LOG_LEVEL
+
+log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
 class QuandlBundle(BaseEquityPricingBundle):

catalyst/data/data_portal.py

@@ -68,7 +68,9 @@ from catalyst.errors import (
     HistoryWindowStartsBeforeData,
 )
 
-log = Logger('DataPortal')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('DataPortal', level=LOG_LEVEL)
 
 BASE_FIELDS = frozenset([
     "open",

catalyst/data/loader.py

@@ -32,7 +32,9 @@ from ..utils.paths import (
     data_root,
 )
 
-logger = logbook.Logger('Loader')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
 INDEX_MAPPING = {
@@ -95,7 +97,8 @@ def has_data_for_dates(series_or_df, first_date, last_date):
 
 def load_crypto_market_data(trading_day=None, trading_days=None,
                             bm_symbol=None, bundle=None, bundle_data=None,
-                            environ=None, exchange=None):
+                            environ=None, exchange=None, start_dt=None,
+                            end_dt=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
 
@@ -104,8 +107,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
-    first_date = get_calendar('OPEN').first_trading_session
+    # if start_dt is None:
-    now = pd.Timestamp.utcnow()
+    start_dt = get_calendar('OPEN').first_trading_session
+
+    if end_dt is None:
+        end_dt = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
     # **two** full trading days prior to the most recently completed trading
@@ -131,7 +137,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 1]
+    last_date = trading_days[trading_days.get_loc(end_dt, method='ffill') - 1]
 
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
@@ -146,14 +152,14 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     br = exchange.get_history_window(
         assets=[benchmark_asset],
         end_dt=last_date,
-        bar_count=pd.Timedelta(last_date - first_date).days,
+        bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
         data_frequency='daily')
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
-    br.loc[first_date]=0
+    br.loc[start_dt] = 0
-    br=br.sort_index()
+    br = br.sort_index()
 
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
@@ -162,10 +168,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         bm_symbol,
         first_date_treasury,
         last_date,
-        now,
+        end_dt,
         environ,
     )
-    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
+    benchmark_returns = br[br.index.slice_indexer(start_dt, last_date)]
     treasury_curves = tc[
         tc.index.slice_indexer(first_date_treasury, last_date)]
     return benchmark_returns, treasury_curves

catalyst/data/minute_bars.py

@@ -44,8 +44,9 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
 
-logger = logbook.Logger('MinuteBars')
+logger = logbook.Logger('MinuteBars', level=LOG_LEVEL)
 
 US_EQUITIES_MINUTES_PER_DAY = 390
 FUTURES_MINUTES_PER_DAY = 1440
@@ -1125,7 +1126,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             else:
                 return np.nan
 
-        #if field != 'volume':
+        # if field != 'volume':
         value *= self._ohlc_ratio_inverse_for_sid(sid)
         return value
 
@@ -1206,7 +1207,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             minute_dt.value / NANOS_IN_MINUTE,
             self._minutes_per_day,
             False,
-            )
+        )
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
         """
@@ -1262,10 +1263,10 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 where = values != 0
                 # first slice down to len(where) because we might not have
                 # written data for all the minutes requested
-                #if field != 'volume':
+                # if field != 'volume':
                 out[:len(where), i][where] = (
                     values[where] * self._ohlc_ratio_inverse_for_sid(sid))
-                #else:
+                # else:
                 #    out[:len(where), i][where] = values[where]
 
             results.append(out)
@@ -1353,9 +1354,10 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
     path : str
         The path of the HDF5 file from which to source data.
     """
+
     def __init__(self, path):
         self._panel = pd.read_hdf(path)
 
     def read(self, dts, sids):
         panel = self._panel[sids, dts, :]
-        return panel.iteritems()
+        return panel.iteritems()

catalyst/data/us_equity_pricing.py

@@ -83,7 +83,9 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-logger = logbook.Logger('UsEquityPricing')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('UsEquityPricing', level=LOG_LEVEL)
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
 OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])

catalyst/examples/buy_and_hodl.py

@@ -24,7 +24,7 @@ from catalyst.api import (
 )
 
 def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
+    context.ASSET_NAME = 'BTC_USDT'
     context.TARGET_HODL_RATIO = 0.8
     context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
 
@@ -49,14 +49,14 @@ def handle_data(context, data):
     orders = get_open_orders(context.asset) or []
     for order in orders:
         cancel_order(order)
-    
+
     # Stop buying after passing the reserve threshold
     cash = context.portfolio.cash
     if cash <= reserve_value:
         context.is_buying = False
 
     # Retrieve current asset price from pricing data
-    price = data[context.asset].price
+    price = data.current(context.asset, 'price')
 
     # Check if still buying and could (approximately) afford another purchase
     if context.is_buying and cash > price:
@@ -70,7 +70,7 @@ def handle_data(context, data):
 
     record(
         price=price,
-        volume=data[context.asset].volume,
+        volume=data.current(context.asset, 'volume'),
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,

catalyst/examples/buy_btc.py

@@ -0,0 +1,10 @@
+from catalyst.api import order, record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc=data.current(context.asset, 'price'))

catalyst/examples/buy_btc_simple.py

@@ -0,0 +1,8 @@
+from catalyst.api import order, record, symbol
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc = data.current(context.asset, 'price'))

catalyst/examples/buy_low_sell_high.py

@@ -27,7 +27,7 @@ log = Logger(algo_namespace)
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USD'
+    context.ASSET_NAME = 'XRP_USDT'
     context.asset = symbol(context.ASSET_NAME)
 
     context.TARGET_POSITIONS = 5000

catalyst/examples/buy_low_sell_high_live.py

@@ -1,6 +1,7 @@
 import talib
 from logbook import Logger
 
+import pandas as pd
 from catalyst.api import (
     order,
     order_target_percent,
@@ -17,10 +18,10 @@ log = Logger('buy low sell high')
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_BTC'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 300
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
     context.SLIPPAGE_ALLOWED = 0.02
 
@@ -33,31 +34,31 @@ def initialize(context):
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1d'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
-    # elif rsi <= 70:
+    elif rsi <= 70:
-    #    buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -146,11 +147,22 @@ def analyze(context, stats):
 
 
 run_algorithm(
+    capital_base=100000,
     initialize=initialize,
     handle_data=handle_data,
     analyze=analyze,
-    exchange_name='bitfinex',
+    exchange_name='poloniex',
-    live=True,
+    start=pd.to_datetime('2017-5-01', utc=True),
-    algo_namespace=algo_namespace,
+    end=pd.to_datetime('2017-10-16', utc=True),
-    base_currency='btc'
+    base_currency='usdt',
+    data_frequency='daily'
 )
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc'
+# )

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -163,8 +163,6 @@ def analyze(context, stats):
 # Backtest
 run_algorithm(
     capital_base=250,
-    start=pd.to_datetime('2017-10-01', utc=True),
-    end=pd.to_datetime('2017-10-15', utc=True),
     data_frequency='minute',
     initialize=initialize,
     handle_data=handle_data,

catalyst/examples/simple_loop.py

@@ -1,13 +1,13 @@
-import pandas as pd
 import talib
 
+import pandas as pd
 from catalyst import run_algorithm
 from catalyst.api import symbol
 
 
 def initialize(context):
     print('initializing')
-    context.asset = symbol('xrp_btc')
+    context.asset = symbol('burst_btc')
 
 
 def handle_data(context, data):
@@ -27,25 +27,25 @@ def handle_data(context, data):
     pass
 
 
-# run_algorithm(
-#     capital_base=250,
-#     start=pd.to_datetime('2015-08-01', utc=True),
-#     end=pd.to_datetime('2017-9-30', utc=True),
-#     data_frequency='daily',
-#     initialize=initialize,
-#     handle_data=handle_data,
-#     analyze=None,
-#     exchange_name='poloniex',
-#     algo_namespace='simple_loop',
-#     base_currency='eth'
-# )
 run_algorithm(
+    capital_base=250,
+    start=pd.to_datetime('2017-08-01', utc=True),
+    end=pd.to_datetime('2017-9-30', utc=True),
+    data_frequency='minute',
     initialize=initialize,
     handle_data=handle_data,
     analyze=None,
-    exchange_name='bitfinex',
+    exchange_name='poloniex',
-    live=True,
     algo_namespace='simple_loop',
-    base_currency='eth',
+    base_currency='btc'
-    live_graph=False
 )
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=None,
+#     exchange_name='bitfinex',
+#     live=True,
+#     algo_namespace='simple_loop',
+#     base_currency='eth',
+#     live_graph=False
+# )

catalyst/exchange/asset_finder_exchange.py

@@ -1,6 +1,8 @@
 from logbook import Logger
 
-log = Logger('AssetFinderExchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('AssetFinderExchange', level=LOG_LEVEL)
 
 
 class AssetFinderExchange(object):
@@ -41,9 +43,9 @@ class AssetFinderExchange(object):
         """
         for sid in sids:
             if sid in self._asset_cache:
-                log.info('got asset from cache: {}'.format(sid))
+                log.debug('got asset from cache: {}'.format(sid))
             else:
-                log.info('fetching asset: {}'.format(sid))
+                log.debug('fetching asset: {}'.format(sid))
         return list()
 
     def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,10 +1,10 @@
 import base64
+import datetime
 import hashlib
 import hmac
 import json
 import re
 import time
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import (
     InvalidOrderStyle, OrderCancelError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.protocol import Account
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -33,7 +33,9 @@ requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-log = Logger('Bitfinex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 
@@ -56,7 +58,7 @@ class Bitfinex(Exchange):
 
         # Max is 90 but playing it safe
         # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -253,6 +255,16 @@ class Bitfinex(Exchange):
         '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
          '1M'
         """
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=data_frequency,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
 
         freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
         if freq_match:
@@ -665,10 +677,11 @@ class Bitfinex(Exchange):
         return time.strftime('%Y-%m-%d',
                              time.gmtime(int(response.json()[-1][0] / 1000)))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         exchange_symbol = asset.exchange_symbol
         try:
             self.ask_request()
+            # TODO: implement limit
             response = self._request(
                 'book/{}'.format(exchange_symbol), None)
             data = response.json()

catalyst/exchange/bittrex/bittrex.py

@@ -1,29 +1,33 @@
 import json
 
 import pandas as pd
+import time
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
+    download_exchange_symbols, get_symbols_string
 from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
 
-log = Logger('Bittrex')
+# TODO: consider using this: https://github.com/mondeja/bittrex_v2
+
+log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 
 
 class Bittrex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Bittrex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Bittrex_api(key=key, secret=secret)
         self.name = 'bittrex'
         self.color = 'blue'
         self.base_currency = base_currency
@@ -64,10 +68,10 @@ class Bittrex(Exchange):
         return exchange_symbol.lower()
 
     def get_balances(self):
+        balances = self.api.getbalances()
         try:
             log.debug('retrieving wallet balances')
             self.ask_request()
-            balances = self.api.getbalances()
 
         except Exception as e:
             raise ExchangeRequestError(error=e)
@@ -207,7 +211,7 @@ class Bittrex(Exchange):
             )
 
     def get_candles(self, data_frequency, assets, bar_count=None,
-                    start_date=None):
+                    start_dt=None, end_dt=None):
         """
         Supported Intervals
         -------------------
@@ -216,10 +220,27 @@ class Bittrex(Exchange):
         :param data_frequency:
         :param assets:
         :param bar_count:
+        :param start_dt
+        :param end_dt
         :return:
         """
-        log.info('retrieving candles')
 
+        # TODO: this has no effect at the moment
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
+
+        log.debug(
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=data_frequency,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        data_frequency = data_frequency.lower()
         if data_frequency == 'minute' or data_frequency == '1m':
             frequency = 'oneMin'
         elif data_frequency == '5m':
@@ -228,7 +249,7 @@ class Bittrex(Exchange):
             frequency = 'thirtyMin'
         elif data_frequency == '1h':
             frequency = 'hour'
-        elif data_frequency == 'daily' or data_frequency == '1D':
+        elif data_frequency == 'daily' or data_frequency == '1d':
             frequency = 'day'
         else:
             raise InvalidHistoryFrequencyError(
@@ -237,13 +258,14 @@ class Bittrex(Exchange):
 
         # Making sure that assets are iterable
         asset_list = [assets] if isinstance(assets, TradingPair) else assets
-        ohlc_map = dict()
         for asset in asset_list:
+            end = int(time.mktime(end_dt.timetuple()))
             url = '{url}/pub/market/GetTicks?marketName={symbol}' \
-                  '&tickInterval={frequency}&_=1499127220008'.format(
+                  '&tickInterval={frequency}&_={end}'.format(
                 url=URL2,
                 symbol=self.get_symbol(asset),
-                frequency=frequency
+                frequency=frequency,
+                end=end
             )
 
             try:
@@ -271,6 +293,7 @@ class Bittrex(Exchange):
                 return ohlc
 
             ordered_candles = list(reversed(candles))
+            ohlc_map = dict()
             if bar_count is None:
                 ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
             else:
@@ -358,7 +381,7 @@ class Bittrex(Exchange):
             json.dump(symbol_map, f, sort_keys=True, indent=2,
                       separators=(',', ':'))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         if order_type == 'all':
             order_type = 'both'
         elif order_type == 'bid':
@@ -369,7 +392,11 @@ class Bittrex(Exchange):
             raise ValueError('invalid type')
 
         exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(market=exchange_symbol, type=order_type)
+        data = self.api.getorderbook(
+            market=exchange_symbol,
+            type=order_type,
+            depth=100
+        )
 
         result = dict()
         for exchange_type in data:

catalyst/exchange/bittrex/bittrex_api.py

@@ -4,10 +4,10 @@ import time
 import hmac
 import hashlib
 
-from six.moves import urllib
 
 # Workaround for backwards compatibility
 # https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
+from six.moves import urllib
 urlopen = urllib.request.urlopen
 
 
@@ -39,7 +39,10 @@ class Bittrex_api(object):
         if method not in self.public:
             url += '&apikey=' + self.key
             url += '&nonce=' + str(int(time.time()))
-            signature = hmac.new(self.secret, url, hashlib.sha512).hexdigest()
+
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 url.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
             headers = {'apisign': signature}
         else:
             headers = {}

catalyst/exchange/bundle_utils.py

@@ -1,18 +1,15 @@
 import calendar
-import tarfile
-
-import requests
-from datetime import timedelta, datetime, date
 import os
-import pandas as pd
+import tarfile
-import numpy as np
+from datetime import timedelta, datetime, date
 
+import numpy as np
+import pandas as pd
 import pytz
 
 from catalyst.data.bundles import from_bundle_ingest_dirname
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_errors import ApiCandlesError, \
+from catalyst.exchange.exchange_errors import NoDataAvailableOnExchange
-    PricingDataBeforeTradingError, NoDataAvailableOnExchange
 from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_path
@@ -106,42 +103,6 @@ def get_start_dt(end_dt, bar_count, data_frequency):
     return start_dt
 
 
-def get_adj_dates(start, end, assets, data_frequency):
-    """
-    Contains a date range to the trading availability of the specified pairs.
-
-    :param start:
-    :param end:
-    :param assets:
-    :param data_frequency:
-    :return:
-    """
-    earliest_trade = None
-    last_entry = None
-    for asset in assets:
-        if earliest_trade is None or earliest_trade > asset.start_date:
-            earliest_trade = asset.start_date
-
-        end_asset = asset.end_minute if data_frequency == 'minute' else \
-            asset.end_daily
-        if end_asset is not None and \
-                (last_entry is None or end_asset > last_entry):
-            last_entry = end_asset
-
-    if start is None or earliest_trade > start:
-        start = earliest_trade
-
-    if end is None or (last_entry is not None and end > last_entry):
-        end = last_entry
-
-    if end is None or start >= end:
-        raise NoDataAvailableOnExchange(
-            exchange=asset.exchange.title(),
-            symbol=[asset.symbol.encode('utf-8')],
-            data_frequency=data_frequency,
-        )
-
-    return start, end
 
 
 def get_month_start_end(dt):
@@ -189,60 +150,6 @@ def get_df_from_arrays(arrays, periods):
     return df
 
 
-def get_df_from_candles(candles, bar_count, end_dt, data_frequency,
-                        previous_candle=None):
-    """
-    Create candles for each period of the specified range, forward-filling
-    missing candles with the previous value.
-
-    :param candles:
-    :param bar_count:
-    :param end_dt:
-    :param data_frequency:
-    :param previous_candle:
-
-    :return:
-    """
-    all_dates = []
-    all_candles = []
-
-    start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-    date = start_dt
-
-    # TODO: this works well with a small number of candles, consider using numpy as needed
-    while date <= end_dt:
-        candle = next((
-            candle for candle in candles if candle['last_traded'] == date
-        ), previous_candle)
-
-        if candle is None:
-            candle = candles[0]
-
-        all_dates.append(date)
-        all_candles.append(candle)
-
-        previous_candle = candle
-
-        date += get_delta(1, data_frequency)
-
-    return all_dates, all_candles
-
-
-def get_trailing_candles_dt(asset, start_dt, end_dt, data_frequency):
-    missing_start = None
-
-    if asset.end_minute is not None and start_dt < asset.end_minute:
-        if asset.end_minute < end_dt:
-            delta = get_delta(1, data_frequency)
-
-            missing_start = asset.end_minute + delta
-
-    else:
-        missing_start = start_dt
-
-    return missing_start
-
-
 def range_in_bundle(asset, start_dt, end_dt, reader):
     """
     Evaluate whether price data of an asset is included has been ingested in
@@ -278,6 +185,7 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
     return has_data
 
 
+@deprecated
 def find_most_recent_time(bundle_name):
     """
     Find most recent "time folder" for a given bundle.
@@ -299,92 +207,12 @@ def find_most_recent_time(bundle_name):
     for folder in bundle_folders:
         date = from_bundle_ingest_dirname(folder)
         if not most_recent_bundle or date > \
-                most_recent_bundle[most_recent_bundle.keys()[0]]:
+                most_recent_bundle[list(most_recent_bundle.keys())[0]]:
             most_recent_bundle = dict()
             most_recent_bundle[folder] = date
 
     if most_recent_bundle:
-        return most_recent_bundle.keys()[0]
+        return list(most_recent_bundle.keys())[0]
     else:
         return None
 
-
-@deprecated
-def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
-    """
-    History API provides OHLCV data for any of the supported exchanges up to yesterday.
-
-    :param exchange_name: string
-        Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex).
-    :param data_frequency: string
-        Required: The bar frequency (minute or daily)
-    :param symbol: string
-        Required: The trading pair symbol, using Catalyst naming convention
-    :param start: datetime
-        Optional: The start date.
-    :param end: datetime
-        Optional: The end date.
-
-    :return ohlcv: list[dict[string, float]]
-        Each row contains the following dictionary for the resulting bars:
-        'ts'     : int, the timestamp in seconds
-        'open'   : float
-        'high'   : float
-        'low'    : float
-        'close'  : float
-        'volume' : float
-
-    Notes
-    =====
-    Using seconds for the start and end dates for ease of use in the
-    function query parameters.
-
-    Sometimes, one minute goes by without completing a trade of the given
-    trading pair on the given exchange. To minimize the payload size, we
-    don't return identical sequential bars. Post-processing code will
-    forward fill missing bars outside of this function.
-    """
-
-    start_seconds = get_seconds_from_date(start) if start else None
-    end_seconds = get_seconds_from_date(end) if end else None
-
-    if exchange_name not in EXCHANGE_NAMES:
-        raise ValueError(
-            'get_history function only supports the following exchanges: {}'.format(
-                list(EXCHANGE_NAMES)))
-
-    if data_frequency != 'daily' and data_frequency != 'minute':
-        raise ValueError(
-            'get_history currently only supports daily and minute data.'
-        )
-
-    url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format(
-        api_url=API_URL,
-        exchange=exchange_name,
-        symbol=symbol,
-        data_frequency=data_frequency,
-    )
-
-    if start_seconds:
-        url += '&start={}'.format(start_seconds)
-
-    if end_seconds:
-        url += '&end={}'.format(end_seconds)
-
-    try:
-        response = requests.get(url)
-    except Exception as e:
-        raise ValueError(e)
-
-    data = response.json()
-
-    if 'error' in data:
-        raise ApiCandlesError(error=data['error'])
-
-    for candle in data:
-        last_traded = pd.Timestamp.utcfromtimestamp(candle['ts'])
-        last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-        candle['last_traded'] = last_traded
-
-    return data

catalyst/exchange/data_portal_exchange.py

@@ -12,24 +12,22 @@
 # limitations under the License.
 
 import abc
-from datetime import timedelta
 from time import sleep
 
+import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import DataPortal
-from catalyst.errors import HistoryWindowStartsBeforeData
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
-    PricingDataBeforeTradingError,
+    PricingDataNotLoadedError)
-    PricingDataNotLoadedError, InvalidHistoryFrequencyError,
-    BundleNotFoundError)
 
-log = Logger('DataPortalExchange')
+log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
@@ -82,7 +80,7 @@ class DataPortalExchangeBase(DataPortal):
                 return pd.concat(df_list)
 
             else:
-                exchange = self.exchanges[exchange_assets.keys()[0]]
+                exchange = self.exchanges[list(exchange_assets.keys())[0]]
                 return self.get_exchange_history_window(
                     exchange,
                     assets,
@@ -153,6 +151,10 @@ class DataPortalExchangeBase(DataPortal):
                 exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
                     exchange, [assets], field, dt, data_frequency)
+
+                if not spot_values:
+                    return np.nan
+
                 return spot_values[0]
 
             else:
@@ -163,8 +165,8 @@ class DataPortalExchangeBase(DataPortal):
 
                     exchange_assets[asset.exchange].append(asset)
 
-                if len(exchange_assets.keys()) == 1:
+                if len(list(exchange_assets.keys())) == 1:
-                    exchange = self.exchanges[exchange_assets.keys()[0]]
+                    exchange = self.exchanges[list(exchange_assets.keys())[0]]
                     return self.get_exchange_spot_value(
                         exchange, assets, field, dt, data_frequency)
 
@@ -282,109 +284,60 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange bundle.
+
+        Using a try... except approach to minimize reads most of the time,
+        when the data exists.
+
+        :param exchange:
+        :param assets:
+        :param end_dt:
+        :param bar_count:
+        :param frequency:
+        :param field:
+        :param data_frequency:
+        :param ffill:
+        :return:
+        """
+
         bundle = self.exchange_bundles[exchange.name]
-
+        series = bundle.get_history_window_series_and_load(
-        if data_frequency == 'minute':
+            assets=assets,
-            dts = self.trading_calendar.minutes_window(
+            end_dt=end_dt,
-                end_dt, -bar_count
+            bar_count=bar_count,
-            )
+            field=field,
-
+            data_frequency=data_frequency
-            self.ensure_after_first_day(dts[0], assets)
+        )
-
-        elif data_frequency == 'daily':
-            session = self.trading_calendar.minute_to_session_label(end_dt)
-            dts = self._get_days_for_window(session, bar_count)
-
-            if len(dts) == 0:
-                symbols = [asset.symbol for asset in assets]
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    symbols=symbols,
-                    exchange=exchange.name,
-                    first_trading_day= \
-                        min([asset.start_date for asset in assets]),
-                    data_frequency=data_frequency,
-                    symbol_list=','.join(symbols)
-                )
-
-            self.ensure_after_first_day(dts[0], assets)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=data_frequency)
-
-        reader = bundle.get_reader(data_frequency)
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=exchange.name.title(),
-                data_frequency=data_frequency
-            )
-
-        try:
-            values = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=dts[0],
-                end_dt=dts[-1]
-            )[0]
-
-        except Exception:
-            first_trading_day = self._get_first_trading_day(assets)
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-
-            symbol_list = ','.join(symbols)
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=first_trading_day,
-                exchange=exchange.name.title(),
-                symbols=symbols,
-                symbol_list=symbol_list,
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        for index, asset in enumerate(assets):
-            asset_values = values[:, index]
-
-            value_series = pd.Series(asset_values, index=dts)
-            series[asset] = value_series
-
         return pd.DataFrame(series)
 
-    def ensure_after_first_day(self, dt, assets):
-        first_trading_day = self._get_first_trading_day(assets)
-        if dt < first_trading_day:
-            raise PricingDataBeforeTradingError(
-                first_trading_day=first_trading_day,
-                exchange=assets[0].exchange.title(),
-                symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                dt=dt,
-            )
-
     def get_exchange_spot_value(self, exchange, assets, field, dt,
                                 data_frequency):
         bundle = self.exchange_bundles[exchange.name]
-        reader = bundle.get_reader(data_frequency)
 
-        self.ensure_after_first_day(dt, assets)
+        if data_frequency == 'daily':
+            dt = dt.floor('1D')
+        else:
+            dt = dt.floor('1 min')
 
-        values = []
+        try:
-        for asset in assets:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)
-            try:
+
-                value = reader.get_value(
+        except PricingDataNotLoadedError:
-                    sid=asset.sid,
+            log.info(
-                    dt=dt,
+                'pricing data for {symbol} not found on {dt}'
-                    field=field
+                ', updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    dt=dt
                 )
-                values.append(value)
+            )
-            except Exception:
+            bundle.ingest_assets(
-                raise PricingDataNotLoadedError(
+                assets=assets,
-                    field=field,
+                start_dt=self._first_trading_day,
-                    first_trading_day=self._get_first_trading_day(assets),
+                end_dt=self._last_available_session,
-                    exchange=exchange.name.title(),
+                data_frequency=data_frequency,
-                    symbols=[asset.symbol.encode('utf-8') for asset in assets],
+                show_progress=True
-                    symbol_list=''.join(
+            )
-                        [asset.symbol.encode('utf-8') for asset in assets]),
+            return bundle.get_spot_values(
-                    data_frequency=data_frequency
+                assets, field, dt, data_frequency, True
-                )
+            )
-
-            return values

catalyst/exchange/exchange.py

@@ -9,14 +9,14 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_adj_dates
+    get_delta, get_periods
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
-    InvalidHistoryFrequencyError, MismatchingFrequencyError, \
+    InvalidHistoryFrequencyError, PricingDataNotLoadedError
-    BundleNotFoundError, NoDataAvailableOnExchange
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,7 +24,7 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
-log = Logger('Exchange')
+log = Logger('Exchange', level=LOG_LEVEL)
 
 
 class Exchange:
@@ -87,7 +87,7 @@ class Exchange:
             self.request_cpt[now] = 0
             return True
 
-        cpt_date = self.request_cpt.keys()[0]
+        cpt_date = list(self.request_cpt.keys())[0]
         cpt = self.request_cpt[cpt_date]
 
         if now > cpt_date + timedelta(minutes=1):
@@ -167,8 +167,10 @@ class Exchange:
                 asset = self.assets[key]
 
         if not asset:
-            supported_symbols = [pair.symbol.encode('utf-8') for pair in
+            supported_symbols = [
-                                 self.assets.values()]
+                pair.symbol for pair in list(self.assets.values())
+            ]
+
             raise SymbolNotFoundOnExchange(
                 symbol=symbol,
                 exchange=self.name.title(),
@@ -370,46 +372,8 @@ class Exchange:
 
         return value
 
-    def get_series_from_bundle(self, assets, start_dt, end_dt, data_frequency,
-                               field):
-        """
-
-        :return:
-        """
-        reader = self.bundle.get_reader(data_frequency)
-
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=self.name.title(),
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        try:
-            arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=start_dt,
-                end_dt=end_dt
-            )
-
-            periods = self.bundle.get_calendar_periods_range(
-                start_dt, end_dt, data_frequency
-            )
-
-            for asset_index, asset in enumerate(assets):
-                asset_values = arrays[asset_index]
-
-                value_series = pd.Series(asset_values[0], index=periods)
-                series[asset] = value_series
-
-        except Exception as e:
-            log.debug('unable to retrieve from bundle: {}'.format(e))
-
-        return series
-
     def get_series_from_candles(self, candles, start_dt, end_dt,
-                                field, previous_value=None):
+                                data_frequency, field, previous_value=None):
         """
         Get a series of field data for the specified candles.
 
@@ -424,9 +388,12 @@ class Exchange:
         dates = [candle['last_traded'] for candle in candles]
         values = [candle[field] for candle in candles]
 
-        periods = pd.date_range(start_dt, end_dt)
+        periods = self.bundle.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
         series = pd.Series(values, index=dates)
 
+        #TODO: ensure that this working as expected, if not use fillna
         series.reindex(periods, method='ffill', fill_value=previous_value)
 
         return series
@@ -487,11 +454,6 @@ class Exchange:
                 data_frequency = 'daily'
 
         elif unit.lower() == 'm':
-            # if data_frequency != 'minute':
-            # raise MismatchingFrequencyError(
-            #     frequency=frequency,
-            #     data_frequency=data_frequency
-            # )
             if data_frequency == 'daily':
                 data_frequency = 'minute'
 
@@ -499,42 +461,15 @@ class Exchange:
             raise InvalidHistoryFrequencyError(frequency)
 
         adj_bar_count = candle_size * bar_count
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         try:
-            adj_start_dt, adj_end_dt = get_adj_dates(
+            series = self.bundle.get_history_window_series_and_load(
-                start_dt, end_dt, assets, data_frequency
-            )
-            in_bundle = True
-
-        except NoDataAvailableOnExchange:
-            in_bundle = False
-
-        if in_bundle:
-            missing_assets = self.bundle.filter_existing_assets(
                 assets=assets,
-                start_dt=adj_start_dt,
+                end_dt=end_dt,
-                end_dt=adj_end_dt,
+                bar_count=adj_bar_count,
+                field=field,
                 data_frequency=data_frequency
             )
-
+        except PricingDataNotLoadedError:
-            if missing_assets:
-                self.bundle.ingest_assets(
-                    assets=assets,
-                    start_dt=adj_start_dt,
-                    end_dt=adj_end_dt,
-                    data_frequency=data_frequency
-                )
-
-            series = self.get_series_from_bundle(
-                assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
-                data_frequency=data_frequency,
-                field=field
-            )
-
-        else:
             series = dict()
 
         for asset in assets:
@@ -542,7 +477,7 @@ class Exchange:
                 # Adding bars too recent to be contained in the consolidated
                 # exchanges bundles. We go directly against the exchange
                 # to retrieve the candles.
-
+                start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
                         if asset in series else start_dt
@@ -555,6 +490,7 @@ class Exchange:
                     data_frequency=data_frequency,
                     assets=asset,
                     bar_count=trailing_bar_count,
+                    start_dt=start_dt,
                     end_dt=end_dt
                 )
 
@@ -565,6 +501,7 @@ class Exchange:
                     candles=candles,
                     start_dt=trailing_dt,
                     end_dt=end_dt,
+                    data_frequency=data_frequency,
                     field=field,
                     previous_value=last_value
                 )
@@ -622,7 +559,7 @@ class Exchange:
             portfolio.starting_cash = portfolio.cash
 
         if portfolio.positions:
-            assets = portfolio.positions.keys()
+            assets = list(portfolio.positions.keys())
             tickers = self.tickers(assets)
 
             portfolio.positions_value = 0.0
@@ -852,13 +789,14 @@ class Exchange:
         pass
 
     @abc.abstractmethod
-    def get_orderbook(self, asset, order_type):
+    def get_orderbook(self, asset, order_type, limit):
         """
         Retrieve the the orderbook for the given trading pair.
 
         :param asset: TradingPair
         :param order_type: str
             The type of orders: bid, ask or all
+        :param limit
 
         :return:
         """

catalyst/exchange/exchange_algorithm.py

@@ -26,6 +26,7 @@ from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
     BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
@@ -51,10 +52,10 @@ from catalyst.utils.api_support import (
     disallowed_in_before_trading_start)
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
     expect_types
-from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
+from catalyst.utils.preprocess import preprocess
 
-log = logbook.Logger('exchange_algorithm')
+log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
 
 class ExchangeAlgorithmExecutor(AlgorithmSimulator):
@@ -112,7 +113,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
             else self.sim_params.end_session
 
         if exchange_name is None:
-            exchange = self.exchanges.values()[0]
+            exchange = list(self.exchanges.values())[0]
         else:
             exchange = self.exchanges[exchange_name]
 
@@ -523,7 +524,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             self.add_pnl_stats(minute_stats)
             if self.recorded_vars:
                 self.add_custom_signals_stats(minute_stats)
-                recorded_cols = self.recorded_vars.keys()
+                recorded_cols = list(self.recorded_vars.keys())
             else:
                 recorded_cols = None
 
@@ -555,6 +556,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         except Exception as e:
             log.warn('unable to calculate performance: {}'.format(e))
 
+        # TODO: pickle does not seem to work in python 3
         try:
             save_algo_object(
                 algo_name=self.algo_namespace,

catalyst/exchange/exchange_bcolz.py

@@ -3,7 +3,6 @@ import numpy as np
 from catalyst import get_calendar
 from catalyst.data.minute_bars import BcolzMinuteBarReader, \
     BcolzMinuteBarWriter
-from catalyst.exchange.bundle_utils import get_periods, get_periods_range
 
 
 class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
@@ -17,7 +16,7 @@ class BcolzExchangeBarWriter(BcolzMinuteBarWriter):
             end_session = end_session.floor('1d')
 
         minutes_per_day = 1440 if self._data_frequency == 'minute' else 1
-        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 1000000)
+        default_ohlc_ratio = kwargs.pop('default_ohlc_ratio', 100000000)
         calendar = get_calendar('OPEN')
 
         super(BcolzExchangeBarWriter, self) \
@@ -48,9 +47,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
         # else:
         #     return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
 
-        return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
+        return self._load_raw_arrays(fields, start_dt, end_dt, sids)
 
-    def _load_daily_raw_arrays(self, fields, start_dt, end_dt, sids):
+    def _load_raw_arrays(self, fields, start_dt, end_dt, sids):
         start_idx = self._find_position_of_minute(start_dt)
         end_idx = self._find_position_of_minute(end_dt)
 
@@ -80,8 +79,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
                 if mask is None:
                     mask = a != 0
 
+                inverse_ratio = self._ohlc_ratio_inverse_for_sid(sid)
                 out[:len(mask), i][mask] = (
-                    a[mask] * self._ohlc_ratio_inverse_for_sid(sid)
+                    a[mask] * inverse_ratio
                 )
 
             if field in fields:

catalyst/exchange/exchange_blotter.py

@@ -1,12 +1,13 @@
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import Transaction
 
-log = Logger('exchange_blotter')
+log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos
 # Orders won't often close at Equity levels.

catalyst/exchange/exchange_bundle.py

@@ -3,33 +3,34 @@ import shutil
 from datetime import timedelta
 
 import pandas as pd
-from logbook import Logger, INFO
+from logbook import Logger
 
 from catalyst import get_calendar
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
-    get_bcolz_chunk, get_delta, get_adj_dates, get_month_start_end, \
+    get_bcolz_chunk, get_delta, get_month_start_end, \
-    get_year_start_end, get_periods_range, get_df_from_arrays
+    get_year_start_end, get_df_from_arrays, get_start_dt
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
-    InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
+    InvalidHistoryFrequencyError, TempBundleNotFoundError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange
+    NoDataAvailableOnExchange, \
+    PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
+log = Logger('exchange_bundle', level=LOG_LEVEL)
+
+BUNDLE_NAME_TEMPLATE = os.path.join('{root}', '{frequency}_bundle')
+
 
 def _cachpath(symbol, type_):
     return '-'.join([symbol, type_])
 
 
-BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
-log = Logger('exchange_bundle')
-log.level = INFO
-
-
 class ExchangeBundle:
     def __init__(self, exchange):
         self.exchange = exchange
@@ -172,16 +173,13 @@ class ExchangeBundle:
                 invalid_data_behavior='raise'
             )
         except BcolzMinuteOverlappingData as e:
-            log.warn('chunk already exists: {}'.format(e))
+            log.debug('chunk already exists: {}'.format(e))
         except Exception as e:
             log.warn('error when writing data: {}, trying again'.format(e))
 
             # This is workaround, there is an issue with empty
             # session_label when using a newly created writer
-            key = writer._rootdir if data_frequency == 'minute' \
+            del self._writers[writer._rootdir]
-                else writer._filename
-
-            del self._writers[key]
 
             writer = self.get_writer(writer._start_session,
                                      writer._end_session, data_frequency)
@@ -196,6 +194,71 @@ class ExchangeBundle:
             if data_frequency == 'minute' \
             else self.calendar.sessions_in_range(start_dt, end_dt)
 
+    def ingest_df(self, ohlcv_df, data_frequency, asset, writer,
+                  empty_rows_behavior='strip'):
+        """
+        Ingest a DataFrame of OHLCV data for a given market.
+
+        :param ohlcv_df:
+        :param data_frequency:
+        :param asset:
+        :param writer:
+        :param path:
+        :param empty_rows_behavior:
+        :return:
+        """
+        if empty_rows_behavior is not 'ignore':
+            nan_rows = ohlcv_df[ohlcv_df.isnull().T.any().T].index
+
+            if len(nan_rows) > 0:
+                dates = []
+                previous_date = None
+                for row_date in nan_rows.values:
+                    row_date = pd.to_datetime(row_date)
+
+                    if previous_date is None:
+                        dates.append(row_date)
+
+                    else:
+                        seq_date = previous_date + get_delta(1, data_frequency)
+
+                        if row_date > seq_date:
+                            dates.append(previous_date)
+                            dates.append(row_date)
+
+                    previous_date = row_date
+
+                dates.append(pd.to_datetime(nan_rows.values[-1]))
+
+                name = '{} from {} to {}'.format(
+                    asset.symbol, ohlcv_df.index[0], ohlcv_df.index[-1]
+                )
+                if empty_rows_behavior == 'warn':
+                    log.warn(
+                        '\n{name} with end minute {end_minute} has empty rows '
+                        'in ranges: {dates}'.format(
+                            name=name,
+                            end_minute=asset.end_minute,
+                            dates=dates
+                        )
+                    )
+
+                elif empty_rows_behavior == 'raise':
+                    raise EmptyValuesInBundleError(
+                        name=name,
+                        end_minute=asset.end_minute,
+                        dates=dates
+                    )
+                else:
+                    ohlcv_df.dropna(inplace=True)
+
+        data = []
+        if not ohlcv_df.empty:
+            ohlcv_df.sort_index(inplace=True)
+            data.append((asset.sid, ohlcv_df))
+
+        self._write(data, writer, data_frequency)
+
     def ingest_ctable(self, asset, data_frequency, period, start_dt, end_dt,
                       writer, empty_rows_behavior='strip', cleanup=False):
         """
@@ -225,12 +288,18 @@ class ExchangeBundle:
         if reader is None:
             raise TempBundleNotFoundError(path=path)
 
-        arrays = reader.load_raw_arrays(
+        arrays = None
-            sids=[asset.sid],
+        try:
-            fields=['open', 'high', 'low', 'close', 'volume'],
+            arrays = reader.load_raw_arrays(
-            start_dt=start_dt,
+                sids=[asset.sid],
-            end_dt=end_dt
+                fields=['open', 'high', 'low', 'close', 'volume'],
-        )
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+        except Exception as e:
+            log.warn('skipping ctable for {} from {} to {}: {}'.format(
+                asset.symbol, start_dt, end_dt, e
+            ))
 
         if not arrays:
             return path
@@ -238,65 +307,69 @@ class ExchangeBundle:
         periods = self.get_calendar_periods_range(
             start_dt, end_dt, data_frequency
         )
-
         df = get_df_from_arrays(arrays, periods)
-
+        self.ingest_df(
-        if empty_rows_behavior is not 'ignore':
+            ohlcv_df=df,
-            nan_rows = df[df.isnull().T.any().T].index
+            data_frequency=data_frequency,
-
+            asset=asset,
-            if len(nan_rows) > 0:
+            writer=writer,
-                dates = []
+            empty_rows_behavior=empty_rows_behavior
-                previous_date = None
+        )
-                for row_date in nan_rows.values:
-                    row_date = pd.to_datetime(row_date)
-
-                    if previous_date is None:
-                        dates.append(row_date)
-
-                    else:
-                        seq_date = previous_date + get_delta(1, data_frequency)
-
-                        if row_date > seq_date:
-                            dates.append(previous_date)
-                            dates.append(row_date)
-
-                    previous_date = row_date
-
-                dates.append(pd.to_datetime(nan_rows.values[-1]))
-
-                name = path.split('/')[-1]
-                if empty_rows_behavior == 'warn':
-                    log.warn(
-                        '\n{name} with end minute {end_minute} has empty rows '
-                        'in ranges: {dates}'.format(
-                            name=name,
-                            end_minute=asset.end_minute,
-                            dates=dates
-                        )
-                    )
-
-                elif empty_rows_behavior == 'raise':
-                    raise EmptyValuesInBundleError(
-                        name=name,
-                        end_minute=asset.end_minute,
-                        dates=dates
-                    )
-                else:
-                    df.dropna(inplace=True)
-
-        data = []
-        if not df.empty:
-            df.sort_index(inplace=True)
-            data.append((asset.sid, df))
-        self._write(data, writer, data_frequency)
 
         if cleanup:
-            log.debug('removing bundle folder following '
+            log.debug(
-                      'ingestion: {}'.format(path))
+                'removing bundle folder following ingestion: {}'.format(path)
+            )
             shutil.rmtree(path)
 
         return path
 
+    def get_adj_dates(self, start, end, assets, data_frequency):
+        """
+        Contains a date range to the trading availability of the specified pairs.
+
+        :param start:
+        :param end:
+        :param assets:
+        :param data_frequency:
+        :return:
+        """
+        earliest_trade = None
+        last_entry = None
+        for asset in assets:
+            if earliest_trade is None or earliest_trade > asset.start_date:
+                if asset.start_date >= self.calendar.first_session:
+                    earliest_trade = asset.start_date
+
+                else:
+                    earliest_trade = self.calendar.first_session
+
+            end_asset = asset.end_minute if data_frequency == 'minute' else \
+                asset.end_daily
+            if end_asset is not None:
+                if last_entry is None or end_asset > last_entry:
+                    last_entry = end_asset
+
+            else:
+                end = None
+                last_entry = None
+
+        if start is None or \
+                (earliest_trade is not None and earliest_trade > start):
+            start = earliest_trade
+
+        if end is None or (last_entry is not None and end > last_entry):
+            end = last_entry
+
+        if end is None or start is None or start >= end:
+            raise NoDataAvailableOnExchange(
+                exchange=asset.exchange.title(),
+                symbol=[asset.symbol],
+                data_frequency=data_frequency,
+            )
+
+        return start, end
+
     def prepare_chunks(self, assets, data_frequency, start_dt, end_dt):
         """
         Split a price data request into chunks corresponding to individual
@@ -313,23 +386,27 @@ class ExchangeBundle:
         chunks = []
         for asset in assets:
             try:
-                asset_start, asset_end = \
+                # Checking if the the asset has price data in the specified
-                    get_adj_dates(start_dt, end_dt, [asset], data_frequency)
+                # date range
+                adj_start, adj_end = self.get_adj_dates(
+                    start_dt, end_dt, [asset], data_frequency
+                )
 
-            except NoDataAvailableOnExchange:
+            except NoDataAvailableOnExchange as e:
+                # If not, we continue to the next asset
+                log.debug('skipping {}: {}'.format(asset.symbol, e))
                 continue
 
+            # This is either the first trading day of the asset or the
+            # first session available in the calendar
+            first_trading_dt = asset.start_date \
+                if asset.start_date > self.calendar.first_session \
+                else self.calendar.first_session
+
             # Aligning start / end dates with the daily calendar
-            sessions = get_periods_range(start_dt, end_dt, data_frequency) \
+            sessions = self.calendar.sessions_in_range(adj_start, adj_end)
-                if data_frequency == 'minute' \
-                else self.calendar.sessions_in_range(start_dt, end_dt)
-
-            if asset_start < sessions[0]:
-                asset_start = sessions[0]
-
-            if asset_end > sessions[-1]:
-                asset_end = sessions[-1]
 
+            # We loop through each session to create chunks for each period
             chunk_labels = []
             dt = sessions[0]
             while dt <= sessions[-1]:
@@ -343,29 +420,39 @@ class ExchangeBundle:
                     # of the trading pair
                     if data_frequency == 'minute':
                         period_start, period_end = get_month_start_end(dt)
-                        asset_start_month, _ = get_month_start_end(asset_start)
 
+                        asset_start_month, _ = get_month_start_end(
+                            first_trading_dt
+                        )
                         if asset_start_month == period_start \
-                                and period_start < asset_start:
+                                and period_start < first_trading_dt:
-                            period_start = asset_start
+                            period_start = first_trading_dt
 
-                        _, asset_end_month = get_month_start_end(asset_end)
+                        # TODO: need to filter closed pairs?
+                        _, asset_end_month = get_month_start_end(
+                            asset.end_minute
+                        )
                         if asset_end_month == period_end \
-                                and period_end > asset_end:
+                                and period_end > asset.end_minute:
-                            period_end = asset_end
+                            period_end = asset.end_minute
 
                     elif data_frequency == 'daily':
                         period_start, period_end = get_year_start_end(dt)
-                        asset_start_year, _ = get_year_start_end(asset_start)
 
+                        asset_start_year, _ = get_year_start_end(
+                            first_trading_dt
+                        )
                         if asset_start_year == period_start \
-                                and period_start < asset_start:
+                                and period_start < first_trading_dt:
-                            period_start = asset_start
+                            period_start = first_trading_dt
 
-                        _, asset_end_year = get_year_start_end(asset_end)
+                        _, asset_end_year = get_year_start_end(
+                            asset.end_daily
+                        )
                         if asset_end_year == period_end \
-                                and period_end > asset_end:
+                                and period_end > asset.end_daily:
-                            period_end = asset_end
+                            period_end = asset.end_daily
+
                     else:
                         raise InvalidHistoryFrequencyError(
                             frequency=data_frequency
@@ -375,10 +462,13 @@ class ExchangeBundle:
                     # Checking the last minute of the day instead.
                     range_start = period_start.replace(hour=23, minute=59) \
                         if data_frequency == 'minute' else period_start
+
+                    # Checking if the data already exists in the bundle
+                    # for the date range of the chunk. If not, we create
+                    # a chunk for ingestion.
                     has_data = range_in_bundle(
                         asset, range_start, period_end, reader
                     )
-
                     if not has_data:
                         log.debug('adding period: {}'.format(label))
                         chunks.append(
@@ -392,6 +482,7 @@ class ExchangeBundle:
 
                 dt += timedelta(days=1)
 
+        # We sort the chunks by end date to ingest most recent data first
         chunks.sort(key=lambda chunk: chunk['period_end'])
 
         return chunks
@@ -406,13 +497,24 @@ class ExchangeBundle:
         :param end_dt:
         :return:
         """
-        writer = self.get_writer(start_dt, end_dt, data_frequency)
         chunks = self.prepare_chunks(
             assets=assets,
             data_frequency=data_frequency,
             start_dt=start_dt,
             end_dt=end_dt
         )
+
+        # Since chunks are either monthly or yearly, it is possible that
+        # our ingestion data range is greater than specified. We adjust
+        # the boundaries to ensure that the writer can write all data.
+        for chunk in chunks:
+            if chunk['period_start'] < start_dt:
+                start_dt = chunk['period_start']
+
+            if chunk['period_end'] > end_dt:
+                end_dt = chunk['period_end']
+
+        writer = self.get_writer(start_dt, end_dt, data_frequency)
         with maybe_show_progress(
                 chunks,
                 show_progress,
@@ -428,7 +530,8 @@ class ExchangeBundle:
                     start_dt=chunk['period_start'],
                     end_dt=chunk['period_end'],
                     writer=writer,
-                    empty_rows_behavior='strip'
+                    empty_rows_behavior='strip',
+                    cleanup=True
                 )
 
     def ingest(self, data_frequency, include_symbols=None,
@@ -446,8 +549,192 @@ class ExchangeBundle:
         :return:
         """
         assets = self.get_assets(include_symbols, exclude_symbols)
-        start_dt, end_dt = get_adj_dates(start, end, assets, data_frequency)
+        start_dt, end_dt = self.get_adj_dates(
+            start, end, assets, data_frequency
+        )
 
         for frequency in data_frequency.split(','):
             self.ingest_assets(assets, start_dt, end_dt, frequency,
                                show_progress)
+
+    def get_history_window_series_and_load(self,
+                                           assets,
+                                           end_dt,
+                                           bar_count,
+                                           field,
+                                           data_frequency):
+        try:
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency
+            )
+            return pd.DataFrame(series)
+
+        except PricingDataNotLoadedError:
+            start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+            log.info(
+                'pricing data for {symbol} not found in range '
+                '{start} to {end}, updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    start=start_dt,
+                    end=end_dt
+                )
+            )
+            self.ingest_assets(
+                assets=assets,
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                show_progress=True
+            )
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency,
+                reset_reader=True
+            )
+            return series
+
+    def get_spot_values(self, assets, field, dt, data_frequency,
+                        reset_reader=False):
+        values = []
+        try:
+            reader = self.get_reader(data_frequency)
+            if reset_reader:
+                del self._readers[reader._rootdir]
+                reader = self.get_reader(data_frequency)
+
+            for asset in assets:
+                value = reader.get_value(
+                    sid=asset.sid,
+                    dt=dt,
+                    field=field
+                )
+                values.append(value)
+
+            return values
+
+        except Exception:
+            symbols = [asset.symbol for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+    def get_history_window_series(self,
+                                  assets,
+                                  end_dt,
+                                  bar_count,
+                                  field,
+                                  data_frequency,
+                                  reset_reader=False):
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        start_dt, end_dt = self.get_adj_dates(
+            start_dt, end_dt, assets, data_frequency
+        )
+
+        reader = self.get_reader(data_frequency)
+        if reset_reader:
+            del self._readers[reader._rootdir]
+            reader = self.get_reader(data_frequency)
+
+        if reader is None:
+            symbols = [asset.symbol for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        for asset in assets:
+            asset_start_dt, asset_end_dt = self.get_adj_dates(
+                start_dt, end_dt, assets, data_frequency
+            )
+
+            in_bundle = range_in_bundle(
+                asset, asset_start_dt, asset_end_dt, reader
+            )
+            if not in_bundle:
+                raise PricingDataNotLoadedError(
+                    field=field,
+                    first_trading_day=asset.start_date,
+                    exchange=self.exchange.name,
+                    symbols=asset.symbol,
+                    symbol_list=asset.symbol,
+                    data_frequency=data_frequency
+                )
+
+        series = dict()
+        try:
+            arrays = reader.load_raw_arrays(
+                sids=[asset.sid for asset in assets],
+                fields=[field],
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+
+        except Exception:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        periods = self.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+
+        for asset_index, asset in enumerate(assets):
+            asset_values = arrays[asset_index]
+
+            value_series = pd.Series(asset_values.flatten(), index=periods)
+            series[asset] = value_series
+
+        return series
+
+    def clean(self, data_frequency):
+        log.debug('cleaning exchange {}, frequency {}'.format(
+            self.exchange.name, data_frequency
+        ))
+        root = get_exchange_folder(self.exchange.name)
+
+        symbols = os.path.join(root, 'symbols.json')
+        if os.path.isfile(symbols):
+            os.remove(symbols)
+
+        temp_bundles = os.path.join(root, 'temp_bundles')
+
+        if os.path.isdir(temp_bundles):
+            log.debug('removing folder and content: {}'.format(temp_bundles))
+            shutil.rmtree(temp_bundles)
+            log.debug('{} removed'.format(temp_bundles))
+
+        frequencies = ['daily', 'minute'] if data_frequency is None \
+            else [data_frequency]
+
+        for frequency in frequencies:
+            label = '{}_bundle'.format(frequency)
+            frequency_bundle = os.path.join(root, label)
+
+            if os.path.isdir(frequency_bundle):
+                log.debug(
+                    'removing folder and content: {}'.format(frequency_bundle)
+                )
+                shutil.rmtree(frequency_bundle)
+                log.debug('{} removed'.format(frequency_bundle))

catalyst/exchange/exchange_errors.py

@@ -1,14 +1,17 @@
-import sys, traceback
+import sys
+import traceback
+
 from catalyst.errors import ZiplineError
 
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange,
+                   ExchangeAuthEmpty]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
-        print "Error traceback: {1} (line {2})\n" \
+        print("Error traceback: {1} (line {2})\n"
-              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue)
+              "{0.__name__}:  {3}".format(exctype, fn, ln, excvalue))
     else:
         sys.__excepthook__(exctype, excvalue, exctraceback)
 
@@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
+class ExchangeAuthEmpty(ZiplineError):
+    msg = (
+        'Please enter your API token key and secret for exchange {exchange} '
+        'in the following file: {filename}'
+    ).strip()
+
+
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '
@@ -204,7 +214,9 @@ class PricingDataNotLoadedError(ZiplineError):
 class ApiCandlesError(ZiplineError):
     msg = ('Unable to fetch candles from the remote API: {error}.').strip()
 
+
 class NoDataAvailableOnExchange(ZiplineError):
-    msg = ('Requested data for trading pair {symbol} is not available on exchange {exchange} '
+    msg = (
-           'in `{data_frequency}` frequency at this time. '
+        'Requested data for trading pair {symbol} is not available on exchange {exchange} '
-           'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
+        'in `{data_frequency}` frequency at this time. '
+        'Check `http://enigma.co/catalyst/status` for market coverage.').strip()

catalyst/exchange/exchange_portfolio.py

@@ -1,9 +1,10 @@
 import numpy as np
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
 
-log = Logger('ExchangePortfolio')
+log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
 
 class ExchangePortfolio(Portfolio):

catalyst/exchange/exchange_utils.py

@@ -1,14 +1,16 @@
 import json
 import os
 import pickle
-import urllib
+
+from catalyst.assets._assets import TradingPair
+from six.moves.urllib import request
 from datetime import date, datetime
 
 import pandas as pd
 
-from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
-    ExchangeSymbolsNotFound
+from catalyst.utils.paths import data_root, ensure_directory, \
-from catalyst.utils.paths import data_root, ensure_directory, last_modified_time
+    last_modified_time
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
@@ -33,7 +35,7 @@ def get_exchange_symbols_filename(exchange_name, environ=None):
 def download_exchange_symbols(exchange_name, environ=None):
     filename = get_exchange_symbols_filename(exchange_name)
     url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = urllib.urlretrieve(url=url, filename=filename)
+    response = request.urlretrieve(url=url, filename=filename)
     return response
 
 
@@ -41,7 +43,9 @@ def get_exchange_symbols(exchange_name, environ=None):
     filename = get_exchange_symbols_filename(exchange_name)
 
     if not os.path.isfile(filename) or \
-            pd.Timedelta(pd.Timestamp('now', tz='UTC') - last_modified_time(filename)).days > 1:
+                    pd.Timedelta(pd.Timestamp('now',
+                                              tz='UTC') - last_modified_time(
+                        filename)).days > 1:
         download_exchange_symbols(exchange_name, environ)
 
     if os.path.isfile(filename):
@@ -55,6 +59,11 @@ def get_exchange_symbols(exchange_name, environ=None):
         )
 
 
+def get_symbols_string(assets):
+    array = [assets] if isinstance(assets, TradingPair) else assets
+    return ', '.join([asset.symbol for asset in array])
+
+
 def get_exchange_auth(exchange_name, environ=None):
     exchange_folder = get_exchange_folder(exchange_name, environ)
     filename = os.path.join(exchange_folder, 'auth.json')
@@ -64,10 +73,11 @@ def get_exchange_auth(exchange_name, environ=None):
             data = json.load(data_file)
             return data
     else:
-        raise ExchangeAuthNotFound(
+        data = dict(name=exchange_name, key='', secret='')
-            exchange=exchange_name,
+        with open(filename, 'w') as f:
-            filename=filename
+            json.dump(data, f, sort_keys=False, indent=2,
-        )
+                      separators=(',', ':'))
+            return data
 
 
 def get_algo_folder(algo_name, environ=None):
@@ -151,8 +161,8 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
 
     filename = os.path.join(folder, key + '.csv')
 
-    with open(filename, 'wb') as handle:
+    with open(filename, 'wt') as handle:
-        df.to_csv(handle)
+        df.to_csv(handle, encoding='UTF_8')
 
 
 def get_exchange_minute_writer_root(exchange_name, environ=None):
@@ -163,6 +173,7 @@ def get_exchange_minute_writer_root(exchange_name, environ=None):
 
     return minute_data_folder
 
+
 def get_exchange_bundles_folder(exchange_name, environ=None):
     exchange_folder = get_exchange_folder(exchange_name, environ)
 

catalyst/exchange/live_graph_clock.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from datetime import timedelta
 
 import pandas as pd
 from catalyst.gens.sim_engine import (
@@ -19,11 +18,11 @@ from catalyst.gens.sim_engine import (
 )
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
-
+log = Logger('LiveGraphClock', level=LOG_LEVEL)
-log = Logger('LiveGraphClock')
 
 
 class LiveGraphClock(object):

catalyst/exchange/poloniex/poloniex.py

@@ -1,44 +1,39 @@
-import base64
-import hashlib
-import hmac
 import json
-import re
+import json
 import time
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 import pytz
-import requests
-# import six
-from six import iteritems
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+# import six
+from six import iteritems
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError,
+    InvalidOrderStyle, OrphanOrderReverseError)
-    OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
+    ExchangeStopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
-    download_exchange_symbols
+    download_exchange_symbols, get_symbols_string
+from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
+from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
+from catalyst.protocol import Account
 
-log = Logger('Poloniex')
+log = Logger('Poloniex', level=LOG_LEVEL)
 
 
 class Poloniex(Exchange):
     def __init__(self, key, secret, base_currency, portfolio=None):
-        self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8'))
+        self.api = Poloniex_api(key=key, secret=secret)
         self.name = 'poloniex'
         self.assets = {}
         self.load_assets()
@@ -49,7 +44,7 @@ class Poloniex(Exchange):
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -124,9 +119,9 @@ class Poloniex(Exchange):
         return order, executed_price
 
     def get_balances(self):
-        log.debug('retrieving wallets balances')
+        balances = self.api.returnbalances()
         try:
-            balances = self.api.returnbalances()
+            log.debug('retrieving wallets balances')
         except Exception as e:
             log.debug(e)
             raise ExchangeRequestError(error=e)
@@ -191,22 +186,35 @@ class Poloniex(Exchange):
         '5m', '15m', '30m', '2h', '4h', '1D'
         """
 
-        # TODO: implement end_dt and start_dt filters
+        if end_dt is None:
+            end_dt = pd.Timestamp.utcnow()
 
-        if (
+        log.debug(
-                        data_frequency == '5m' or data_frequency == 'minute'):  # TODO: Polo does not have '1m'
+            'retrieving {bars} {freq} candles on {exchange} from '
+            '{end_dt} for markets {symbols}, '.format(
+                bars=bar_count,
+                freq=data_frequency,
+                exchange=self.name,
+                end_dt=end_dt,
+                symbols=get_symbols_string(assets)
+            )
+        )
+
+        if data_frequency == '5m':
             frequency = 300
-        elif (data_frequency == '15m'):
+        elif data_frequency == '15m':
             frequency = 900
-        elif (data_frequency == '30m'):
+        elif data_frequency == '30m':
             frequency = 1800
-        elif (data_frequency == '2h'):
+        elif data_frequency == '2h':
             frequency = 7200
-        elif (data_frequency == '4h'):
+        elif data_frequency == '4h':
             frequency = 14400
-        elif (data_frequency == '1D' or data_frequency == 'daily'):
+        elif data_frequency == '1D' or data_frequency == 'daily':
             frequency = 86400
         else:
+            # Poloniex does not offer 1m data candles
+            # It is likely to error out there frequently
             raise InvalidHistoryFrequencyError(
                 frequency=data_frequency
             )
@@ -217,8 +225,8 @@ class Poloniex(Exchange):
 
         for asset in asset_list:
 
-            end = int(time.time())
+            end = int(time.mktime(end_dt.timetuple()))
-            if (bar_count is None):
+            if bar_count is None:
                 start = end - 2 * frequency
             else:
                 start = end - bar_count * frequency

catalyst/exchange/poloniex/poloniex_api.py

@@ -19,19 +19,25 @@ class Poloniex_api(object):
         self.max_requests_per_second = 6
         self.request_cpt = dict()
 
-        self.public  = ['returnTicker', 'return24Volume', 'returnOrderBook',
+        self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
-                        'returnTradeHistory', 'returnChartData',
+                       'returnTradeHistory', 'returnChartData',
-                        'returnCurrencies', 'returnLoanOrders']
+                       'returnCurrencies', 'returnLoanOrders']
-        self.trading = ['returnBalances','returnCompleteBalances','returnDepositAddresses',
+        self.trading = ['returnBalances', 'returnCompleteBalances',
-                        'generateNewAddress','returnDepositsWithdrawals','returnOpenOrders',
+                        'returnDepositAddresses',
-                        'returnTradeHistory','returnOrderTrades',
+                        'generateNewAddress', 'returnDepositsWithdrawals',
+                        'returnOpenOrders',
+                        'returnTradeHistory', 'returnOrderTrades',
                         'buy', 'sell', 'cancelOrder', 'moveOrder',
-                        'withdraw', 'returnFeeInfo','returnAvailableAccountBalances',
+                        'withdraw', 'returnFeeInfo',
+                        'returnAvailableAccountBalances',
                         'returnTradableBalances', 'transferBalance',
-                        'returnMarginAccountSummary','marginBuy','marginSell',
+                        'returnMarginAccountSummary', 'marginBuy',
-                        'getMarginPosition', 'closeMarginPosition','createLoanOffer',
+                        'marginSell',
-                        'cancelLoanOffer','returnOpenLoanOffers','returnActiveLoans',
+                        'getMarginPosition', 'closeMarginPosition',
-                        'returnLendingHistory','toggleAutoRenew']
+                        'createLoanOffer',
+                        'cancelLoanOffer', 'returnOpenLoanOffers',
+                        'returnActiveLoans',
+                        'returnLendingHistory', 'toggleAutoRenew']
 
     def ask_request(self):
         """
@@ -50,7 +56,7 @@ class Poloniex_api(object):
             self.request_cpt[now] = 0
             return True
 
-        cpt_date = self.request_cpt.keys()[0]
+        cpt_date = list(self.request_cpt.keys())[0]
         cpt = self.request_cpt[cpt_date]
 
         if now > cpt_date + 1:
@@ -59,9 +65,8 @@ class Poloniex_api(object):
             return True
 
         if cpt >= self.max_requests_per_second:
-            
+
-            log.debug('max requests 6 reached, sleeping for 1 seconds')
+            time.sleep(1)
-            sleep(1)
 
             now = time.time()
             self.request_cpt = dict()
@@ -73,21 +78,34 @@ class Poloniex_api(object):
     def query(self, method, req={}):
 
         if method in self.public:
-            url = 'https://poloniex.com/public?command=' + method + '&' + urllib.parse.urlencode(req)
+            url = 'https://poloniex.com/public?command=' + method + '&' + \
+                  urllib.parse.urlencode(req)
             headers = {}
             post_data = None
         elif method in self.trading:
             url = 'https://poloniex.com/tradingApi'
             req['command'] = method
-            req['nonce']   = int(time.time()*1000)
+            req['nonce'] = int(time.time() * 1000)
-            post_data      = urllib.parse.urlencode(req)
+            post_data = urllib.parse.urlencode(req)
-            signature      = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
+
-            headers        = { 'Sign': signature, 'Key': self.key}
+            signature = hmac.new(self.secret.encode('utf-8'),
+                                 post_data.encode('utf-8'),
+                                 hashlib.sha512).hexdigest()
+            headers = {'Sign': signature, 'Key': self.key}
+
+            post_data = post_data.encode('utf-8')
         else:
-            raise ValueError('Method "' + method + '" not found in neither the Public API or Trading API endpoints')
+            raise ValueError(
+                'Method "' + method + '" not found in neither the Public API '
+                                      'or Trading API endpoints'
+            )
 
         self.ask_request()
-        req = urllib.request.Request(url, data=post_data, headers=headers)
+        req = urllib.request.Request(
+            url,
+            data=post_data,
+            headers=headers
+        )
         return json.loads(urlopen(req).read())
 
     def returnticker(self):
@@ -100,15 +118,17 @@ class Poloniex_api(object):
         return self.query('returnOrderBook', {'currencyPair': market})
 
     def returntradehistory(self, market, start=None, end=None):
-        if(start is not None and end is not None):
+        if (start is not None and end is not None):
-            return self.query('returntradehistory', 
+            return self.query('returntradehistory',
-                              {'currencyPair': market, 'start': start, 'end': end })
+                              {'currencyPair': market, 'start': start,
+                               'end': end})
         else:
-            return self.query('returntradehistory', {'currencyPair': market })
+            return self.query('returntradehistory', {'currencyPair': market})
 
     def returnchartdata(self, market, period, start, end=9999999999):
-        return self.query('returnChartData', {'currencyPair': market, 'period': period,
+        return self.query('returnChartData',
-                          'start': start, 'end': end})
+                          {'currencyPair': market, 'period': period,
+                           'start': start, 'end': end})
 
     def returncurrencies(self):
         return self.query('returnCurrencies', {})
@@ -120,7 +140,7 @@ class Poloniex_api(object):
         return self.query('returnBalances')
 
     def returncompletebalances(self, account):
-        if(account):
+        if (account):
             return self.query('returnCompleteBalances', {'account': account})
         else:
             return self.query('returnCompleteBalances')
@@ -132,43 +152,54 @@ class Poloniex_api(object):
         return self.query('generateNewAddress', {'currency': currency})
 
     def returnDepositsWithdrawals(self, start, end):
-        return self.query('returnDepositsWithdrawals', {'start': start, 'end': end})
+        return self.query('returnDepositsWithdrawals',
+                          {'start': start, 'end': end})
 
     def returnopenorders(self, market):
         return self.query('returnOpenOrders', {'currencyPair': market})
 
     def returntradehistory(self, market):
-        #TODO: optional start and/or end and limit
+        # TODO: optional start and/or end and limit
         return self.query('returnTradeHistory', {'currencyPair': market})
 
     def returnordertrades(self, ordernumber):
         return self.query('returnOrderTrades', {'orderNumber': ordernumber})
 
-    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
+    def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-        if(fillorkill):
+            postonly=0):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+        if (fillorkill):
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'fillOrKill': fillorkill, })
-        elif(immediateorcancel):
+        elif (immediateorcancel):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'immediateOrCancel': immediateorcancel, })
-        elif(postonly):
+        elif (postonly):
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount,
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount,
                                       'postOnly': postonly, })
         else:
-            return self.query('buy', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('buy', {'currencyPair': market, 'rate': rate,
+                                      'amount': amount, })
 
-    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, postonly=0):
+    def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
-        if(fillorkill):
+             postonly=0):
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
+        if (fillorkill):
-                                      'fillOrKill': fillorkill, })
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
-        elif(immediateorcancel):
+                                       'amount': amount,
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
+                                       'fillOrKill': fillorkill, })
-                                      'immediateOrCancel': immediateorcancel, })
+        elif (immediateorcancel):
-        elif(postonly):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount,
+                                       'amount': amount,
-                                      'postOnly': postonly, })
+                                       'immediateOrCancel': immediateorcancel, })
+        elif (postonly):
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount,
+                                       'postOnly': postonly, })
         else:
-            return self.query('sell', {'currencyPair': market, 'rate':rate, 'amount': amount, })
+            return self.query('sell', {'currencyPair': market, 'rate': rate,
+                                       'amount': amount, })
 
     def cancelorder(self, ordernumber):
         return self.query('cancelOrder', {'orderNumber': ordernumber})
@@ -180,4 +211,3 @@ class Poloniex_api(object):
 
     def returnfeeinfo(self):
         return self.query('returnFeeInfo')
-

catalyst/exchange/simple_clock.py

@@ -16,13 +16,13 @@ from time import sleep
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START,
+    SESSION_START
-    MINUTE_END,
-    SESSION_END
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
 class SimpleClock(object):

catalyst/exchange/stats_utils.py

@@ -49,3 +49,12 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
         columns=columns,
         formatters=formatters
     )
+
+
+def df_to_string(df):
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+    return df.to_string()

catalyst/finance/blotter.py

@@ -34,7 +34,9 @@ from catalyst.finance.commission import (
 from catalyst.finance.cancel_policy import NeverCancel
 from catalyst.utils.input_validation import expect_types
 
-log = Logger('Blotter')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Blotter', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/finance/controls.py

@@ -24,7 +24,9 @@ from catalyst.errors import (
     TradingControlViolation,
 )
 
-log = logbook.Logger('TradingControl')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('TradingControl', level=LOG_LEVEL)
 
 
 class TradingControl(with_metaclass(abc.ABCMeta)):

catalyst/finance/performance/period.py

@@ -88,7 +88,10 @@ from six import itervalues, iteritems
 
 import catalyst.protocol as zp
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
+
 TRADE_TYPE = zp.DATASOURCE_TYPE.TRADE
 
 

catalyst/finance/performance/position.py

@@ -40,7 +40,9 @@ import logbook
 from catalyst.assets import Future, Asset
 from catalyst.utils.input_validation import expect_types
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class Position(object):

catalyst/finance/performance/position_tracker.py

@@ -32,7 +32,9 @@ from catalyst.assets import (
 )
 from . position import positiondict
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 PositionStats = namedtuple('PositionStats',

catalyst/finance/performance/tracker.py

@@ -70,7 +70,9 @@ import catalyst.finance.risk as risk
 
 from . position_tracker import PositionTracker
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class PerformanceTracker(object):

catalyst/finance/risk/cumulative.py

@@ -38,7 +38,9 @@ from empyrical import (
     sortino_ratio,
 )
 
-log = logbook.Logger('Risk Cumulative')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
 
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',

catalyst/finance/risk/period.py

@@ -36,7 +36,9 @@ from empyrical import (
     sortino_ratio
 )
 
-log = logbook.Logger('Risk Period')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Period', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(risk.choose_treasury,
                                     risk.select_treasury_duration)

catalyst/finance/risk/report.py

@@ -63,7 +63,9 @@ from dateutil.relativedelta import relativedelta
 
 from . period import RiskMetricsPeriod
 
-log = logbook.Logger('Risk Report')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Report', level=LOG_LEVEL)
 
 
 class RiskReport(object):

catalyst/finance/risk/risk.py

@@ -61,7 +61,9 @@ Risk Report
 import logbook
 import numpy as np
 
-log = logbook.Logger('Risk')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk', level=LOG_LEVEL)
 
 
 TREASURY_DURATIONS = [

catalyst/finance/trading.py

@@ -26,7 +26,9 @@ from catalyst.data.loader import load_market_data
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.memoize import remember_last
 
-log = logbook.Logger('Trading')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Trading', level=LOG_LEVEL)
 
 
 DEFAULT_CAPITAL_BASE = 1e5

catalyst/gens/tradesimulation.py

@@ -27,7 +27,9 @@ from catalyst.gens.sim_engine import (
     BEFORE_TRADING_START_BAR
 )
 
-log = Logger('Trade Simulation')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Trade Simulation', level=LOG_LEVEL)
 
 
 class AlgorithmSimulator(object):

catalyst/sources/benchmark_source.py

@@ -72,7 +72,13 @@ class BenchmarkSource(object):
                             "benchmark_returns.")
 
     def get_value(self, dt):
-        return self._precalculated_series.loc[dt]
+        try:
+            series = self._precalculated_series
+            value = series.loc[dt]
+            return value
+        except Exception:
+            # TODO: workaround, find permanent fix
+            return 0
 
     def get_range(self, start_dt, end_dt):
         return self._precalculated_series.loc[start_dt:end_dt]

catalyst/sources/requests_csv.py

@@ -23,7 +23,9 @@ from catalyst.protocol import (
 )
 from catalyst.assets import Equity
 
-logger = Logger('Requests Source Logger')
+from catalyst.constants import LOG_LEVEL
+
+logger = Logger('Requests Source Logger', level=LOG_LEVEL)
 
 
 def roll_dts_to_midnight(dts, trading_day):

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,4 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-02-19', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/paths.py

@@ -126,7 +126,7 @@ def catalyst_root(environ=None):
 
     root = environ.get('ZIPLINE_ROOT', None)
     if root is None:
-        root = expanduser('~/.catalyst')
+        root = os.path.join(expanduser('~'),'.catalyst')
 
     return root
 

catalyst/utils/run_algo.py

@@ -36,14 +36,16 @@ from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
+    ExchangeRequestError, ExchangeAuthEmpty,
     ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, ExchangeNotFoundError)
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
+    get_algo_object, get_exchange_folder
 from logbook import Logger
 
-log = Logger('run_algo')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('run_algo', level=LOG_LEVEL)
 
 
 class _RunAlgoError(click.ClickException, ValueError):
@@ -164,6 +166,12 @@ def _run(handle_data,
 
         # This corresponds to the json file containing api token info
         exchange_auth = get_exchange_auth(exchange_name)
+
+        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
+            raise ExchangeAuthEmpty(
+                    exchange=exchange_name.title(), 
+                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
+
         if exchange_name == 'bitfinex':
             exchanges[exchange_name] = Bitfinex(
                 key=exchange_auth['key'],
@@ -191,7 +199,12 @@ def _run(handle_data,
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
-        load=partial(load_crypto_market_data, environ=environ),
+        load=partial(
+            load_crypto_market_data,
+            environ=environ,
+            start_dt=start,
+            end_dt=end
+        ),
         environ=environ,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
@@ -230,8 +243,11 @@ def _run(handle_data,
                 balances = exchange.get_balances()
             except ExchangeRequestError as e:
                 if attempt_index < 20:
-                    log.warn('exchange error when retrieving balances, {} '
+                    log.warn(
-                             'trying again in 5 seconds'.format(e))
+                        'could not retrieve balances on {}: {}'.format(
+                            exchange.name, e
+                        )
+                    )
                     sleep(5)
                     return fetch_capital_base(exchange, attempt_index + 1)
 
@@ -284,7 +300,8 @@ def _run(handle_data,
             exchanges=exchanges,
             asset_finder=None,
             trading_calendar=open_calendar,
-            first_trading_day=None,
+            first_trading_day=start,
+            last_available_session=end
         )
 
         sim_params = create_simulation_parameters(

docs/source/conf.py

@@ -41,7 +41,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC'
+copyright = u'2017, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
@@ -94,6 +94,6 @@ intersphinx_mapping = {
     'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None),
 }
 
-doctest_global_setup = "import zipline"
+doctest_global_setup = "import catalyst"
 
 todo_include_todos = True

docs/source/index.rst

@@ -1,12 +1,17 @@
-.. include:: ../../README.rst
+.. include:: welcome.rst
+|
+|
+Table of Contents
+-----------------
 
 .. toctree::
    :maxdepth: 1
 
    install
    beginner-tutorial
-   bundles
+   naming-convention
-   development-guidelines
+..   bundles
-   appendix
+..   development-guidelines
-   release-process
+..   appendix
-   releases
+..   release-process
+..   releases

docs/source/install.rst

@@ -4,16 +4,16 @@ Install
 Installing with ``pip``
 -----------------------
 
-Installing Zipline via ``pip`` is slightly more involved than the average
+Installing Catalyst via ``pip`` is slightly more involved than the average
 Python package.
 
 There are two reasons for the additional complexity:
 
-1. Zipline ships several C extensions that require access to the CPython C API.
+1. Catalyst ships several C extensions that require access to the CPython C API.
    In order to build the C extensions, ``pip`` needs access to the CPython
    header files for your Python installation.
 
-2. Zipline depends on `numpy <http://www.numpy.org/>`_, the core library for
+2. Catalyst depends on `numpy <http://www.numpy.org/>`_, the core library for
    numerical array computing in Python.  Numpy depends on having the `LAPACK
    <http://www.netlib.org/lapack>`_ linear algebra routines available.
 
@@ -28,13 +28,28 @@ your particular platform), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install zipline
+   $ pip install enigma-catalyst
 
-If you use Python for anything other than Zipline, we **strongly** recommend
+If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
 Python`_ provides an `excellent tutorial on virtualenv
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_.
+<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a summarized
+version:
+
+.. code-block:: bash
+
+   $ virtualenv catalyst-venv
+   $ source ./catalyst-venv/bin/activate
+   $ pip install enigma-
+
+Though not required by Catalyst directly, our example algorithms use matplotlib 
+to visually display the results of the trading algorithms. If you wish to run 
+any examples or use matplotlib during development, it can be installed using:
+
+.. code-block:: bash
+
+    $ pip install matplotlib
 
 GNU/Linux
 ~~~~~~~~~
@@ -60,15 +75,17 @@ On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
    $ pacman -S lapack gcc gcc-fortran pkg-config
 
-There are also AUR packages available for installing `Python 3.4
+.. Commenting it out until Catalyst fully supports Python 3.X
-<https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+..
-3.5, but Zipline only currently supports 3.4), and `ta-lib
+.. There are also AUR packages available for installing `Python 3.4
-<https://aur.archlinux.org/packages/ta-lib/>`_, an optional Zipline dependency.
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-Python 2 is also installable via:
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-.. code-block:: bash
+.. 
 
-   $ pacman -S python2
+..   $ pacman -S python2
 
 OSX
 ~~~
@@ -87,36 +104,238 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
+OSX + virtualenv + matplotlib
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A note about using matplotlib in virtual enviroments on OSX: it may be necessary to run
+
+.. code-block:: bash
+
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+
+in order to override the default ``macosx`` backend for your system, which may not 
+be accessible from inside the virtual environment. This will allow Catalyst to open 
+matplotlib charts from within a virtual environment, which is useful for displaying 
+the performance of your backtests.  To learn more about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+
 Windows
 ~~~~~~~
 
-For windows, the easiest and best supported way to install zipline is to use
+In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This package 
+contains the compiler and the set of system headers necessary for producing 
+binary wheels for Python 2.7 packages. If it's not already in your system, download
+it and install it before proceeding to the next step.
+
+For windows, the easiest and best supported way to install Catalyst is to use
 :ref:`Conda <conda>`.
 
+Amazon Linux AMI
+~~~~~~~~~~~~~~~~
+
+The packages ``pip`` and ``setuptools`` that come shipped by default are very outdated. 
+Thus, you first need to run:
+
+.. code-block:: bash
+
+   pip install --upgrade pip setuptools
+
+The default installation is also missing the C and C++ compilers, which you install by:
+
+.. code-block:: bash
+
+   sudo yum install gcc gcc-c++
+
+Then you should follow the regular installation instructions outlined at the beginning 
+of this page.
+
+
+Troubleshooting ``pip`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Issue**: 
+   Package enigma-catalyst cannot be found 
+ 
+**Solution**: 
+   Make sure you have the most up-to-date version of pip installed, by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip
+
+   On Windows, the recommended command is:
+
+   .. code-block:: bash
+
+      python -m pip install --upgrade pip
+
+----
+
+**Issue**: 
+   Package enigma-catalyst cannot still be found, even after upgrading pip (see above), with an error similar to:
+
+   .. code-block:: bash
+
+      Downloading/unpacking enigma-catalyst
+      Could not find a version that satisfies the requirement enigma-catalyst (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 0.2.dev1, 0.1.dev8, 0.1.dev6)
+      Cleaning up...
+      No distributions matching the version for enigma-catalyst
+
+**Solution**:
+   In some systems (this error has been reported in Ubuntu), pip is configured to only find stable versions by default. Since Catalyst is in alpha version, pip cannot find a matching version that satisfies the installation requirements. The solution is to include the `--pre` flag to include pre-release and development versions:
+
+   .. code-block:: bash
+
+      pip install --pre enigma-catalyst
+
+----
+
+**Issue**: 
+   Package enigma-catalyst fails to install because of outdated setuptools
+
+**Solution**: 
+   Upgrade to the most up-to-date setuptools package by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip setuptools
+
+----
+
+**Issue**:
+   Missing required packages  
+
+**Solution**:
+   Download `requirements.txt 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/requirements.txt>`_ 
+   (click on the *Raw* button and Right click -> Save As...) and use it to
+   install all the required dependencies by running:
+
+   .. code-block:: bash
+
+      pip install -r requirements.txt
+
+----
+
+**Issue**: 
+   Installation fails with error: ``fatal error: Python.h: No such file or directory``
+
+**Solution**: 
+   Some systems (this issue has been reported in Ubuntu) require `python-dev` for the proper build and installation of package dependencies. The solution is to install python-dev, which is independent of the virtual environment. In Ubuntu, you would need to run:
+
+   .. code-block:: bash
+
+      sudo apt-get install python-dev
+
+
 .. _conda:
 
 Installing with ``conda``
 -------------------------
 
-Another way to install Zipline is via the ``conda`` package manager, which
+Another way to install Catalyst is via the ``conda`` package manager, which
 comes as part of Continuum Analytics' `Anaconda
 <http://continuum.io/downloads>`_ distribution.
 
 The primary advantage of using Conda over ``pip`` is that conda natively
 understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Zipline and its dependencies
+``scipy``.  This means that ``conda`` can install Catalyst and its dependencies
-without requiring the use of a second tool to acquire Zipline's non-Python
+without requiring the use of a second tool to acquire Catalyst's non-Python
 dependencies.
 
 For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, you 
+can install MiniConda, which is a smaller footprint (fewer packages and smaller 
+size) than its big brother Anaconda, but it still contains all the main packages 
+needed. To install MiniConda, you can follow these steps:
 
-Once conda has been set up you can install Zipline from our ``Quantopian``
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 for 
-channel:
+   your Operating System.
+2. Install MiniConda. See the `Installation Instructions <https://conda.io/docs/user-guide/install/index.html>`_
+   if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal window,
+   which should print the list of packages installed with Conda.
 
-.. code-block:: bash
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you saved
+   the above ``python2.7-environment.yml`` file.
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new session
+   to run Catalyst):
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 above failed 
+for any reason, you can try setting up the environment manually with the following steps:
+
+1. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy
+
+2. Activate the environment:
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+3. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+Getting Help
+------------
+
+If after following the instructions above, and going through the *Troubleshooting* sections, 
+you still experience problems installing Catalyst, you can seek additional help through the 
+following channels:
+
+- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over the #catalyst_dev 
+  channel where many other users (as well as the project developers) hang out, and can assist 
+  you with your particular issue. The more descriptive and the more information you can provide, 
+  the easiest will be for others to help you out.
+
+- Report the problem you are experiencing on our 
+  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ following the guidelines 
+  provided therein. Before you do so, take a moment to browse through all `previous reported issues
+  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ in the likely case
+  that someone else experienced that same issue before, and you get a hint on how to solve it.
 
-    conda install -c Quantopian zipline
 
 .. _`Debian-derived`: https://www.debian.org/misc/children-distros
 .. _`RHEL-derived`: https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux_derivatives

docs/source/naming-convention.rst

@@ -0,0 +1,66 @@
+Naming Convention
+=================
+
+Catalyst introduces a standardized naming convention for all asset pairs 
+trading on any exchange in the following form:
+
+
+    **{market_currency}_{base_currency}**
+
+Where {market_currency} is the asset to be traded using {base_currency} as 
+the reference, both written in lowercase and separated with an underscore.
+
+This standardization is needed to overcome the lack of consistency in the 
+naming of assets across different exchanges, and making it easier to the user
+to refer to the asset pairs that you want to trade.
+
+Catalyst maintains a `Market Coverage Overview <https://www.enigma.co/catalyst/status>`_ 
+where you can check the mapping between Catalyst naming pairs and that of each 
+exchange. Catalyst will always expect in all its functions that you will refer to 
+the asset pairs by using the Catalyst naming convention.
+
+If at any point, you input the wrong name for an asset pair, you will get an error 
+of that pair not found in the given exchange, and a list of pairs available on that exchange:
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usd
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	Error traceback: /Volumes/Data/Users/victoris/Desktop/Enigma/user-install/catalyst-dev/catalyst/exchange/exchange.py (line 175)
+	SymbolNotFoundOnExchange:  Symbol btc_usd not found on exchange Poloniex. 
+	Choose from: ['rep_usdt', 'gno_btc', 'xvc_btc', 'pink_btc', 'sys_btc', 
+	'emc2_btc', 'rads_btc', 'note_btc', 'maid_btc', 'bch_btc', 'gnt_btc', 
+	'bcn_btc', 'rep_btc', 'bcy_btc', 'cvc_btc', 'nxt_xmr', 'zec_usdt', 
+	'fct_btc', 'gas_btc', 'pot_btc', 'eth_usdt', 'btc_usdt', 'lbc_btc', 
+	'dcr_btc', 'etc_usdt', 'omg_eth', 'amp_btc', 'xpm_btc', 'nxt_btc', 
+	'vtc_btc', 'steem_eth', 'blk_xmr', 'pasc_btc', 'zec_xmr', 'grc_btc', 
+	'nxc_btc', 'btcd_btc', 'ltc_btc', 'dash_btc', 'naut_btc', 'zec_eth', 
+	'zec_btc', 'burst_btc', 'zrx_eth', 'bela_btc', 'steem_btc', 'etc_btc', 
+	'eth_btc', 'huc_btc', 'strat_btc', 'lsk_btc', 'exp_btc', 'clam_btc', 
+	'rep_eth', 'dash_xmr', 'cvc_eth', 'bch_usdt', 'zrx_btc', 'dash_usdt', 
+	'blk_btc', 'xrp_btc', 'nxt_usdt', 'neos_btc', 'omg_btc', 'bts_btc', 
+	'doge_btc', 'gnt_eth', 'sbd_btc', 'gno_eth', 'xcp_btc', 'ltc_usdt', 
+	'btm_btc', 'xmr_usdt', 'lsk_eth', 'omni_btc', 'nav_btc', 'fldc_btc', 
+	'ppc_btc', 'xbc_btc', 'dgb_btc', 'sc_btc', 'btcd_xmr', 'vrc_btc', 
+	'ric_btc', 'str_btc', 'maid_xmr', 'xmr_btc', 'sjcx_btc', 'via_btc', 
+	'xem_btc', 'nmc_btc', 'etc_eth', 'ltc_xmr', 'ardr_btc', 'gas_eth', 
+	'flo_btc', 'xrp_usdt', 'game_btc', 'bch_eth', 'bcn_xmr', 'str_usdt']
+
+In the example above, exchange Poloniex does not use USD, but uses instead the 
+USDT cryptocurrency asset that is issued on the Bitcoin blockchain via the Omni
+Layer Protocol. Each USDT unit is backed by a U.S Dollar held in the reserves of 
+Tether Limited. USDT can be transferred, stored, and spent, just like bitcoins 
+or any other cryptocurrency. Given its 1:1 mapping to the USD, is a viable alternative.
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usdt
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	    [====================================]  Fetching poloniex daily candles: :  100%
+

docs/source/welcome.rst

@@ -0,0 +1,28 @@
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+|
+Catalyst is a data-driven crypto investment platform. It supports both 
+backtesting and live-trading in a number of different crypto-exchanges. 
+Catalyst empowers users to share and curate data and build profitable, 
+data-driven investment strategies.
+
+Features
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.

etc/python2.7-environment.yml

@@ -1,30 +1,22 @@
 name: catalyst
 channels:
-- statiskit
 - defaults
 dependencies:
 - certifi=2016.2.28=py27_0
-- coverage=4.4.1=py27_0
+- mkl=2017.0.3=0
-- nose=1.3.7=py27_1
+- numpy=1.13.1=py27_0
-- openssl=1.0.2l=0
+- openssl=1.0.2l
-- path.py=10.3.1=py27_0
 - pip=9.0.1=py27_1
 - python=2.7.13=0
-- pyyaml=3.12=py27_0
+- scipy=0.19.1=np113py27_0
-- readline=6.2=2
+- setuptools=36.4.0=py27_1
-- setuptools=36.4.0=py27_0
+- sqlite=3.13.0
-- six=1.10.0=py27_0
+- tk=8.5.18
-- sqlite=3.13.0=0
-- tk=8.5.18=0
 - wheel=0.29.0=py27_0
-- yaml=0.1.6=0
 - zlib=1.2.11=0
-- libdev=1.0.0=py27_0
-- python-dev=1.0.0=py27_0
-- python-scons=3.0.0=py27_0
 - pip:
-  - alembic==0.9.5
+  - alembic==0.9.6
-  - backports.shutil-get-terminal-size==1.0.0
+  - backports.functools-lru-cache==1.4
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
@@ -32,36 +24,22 @@ dependencies:
   - contextlib2==0.5.5
   - cycler==0.10.0
   - cyordereddict==1.0.0
-  - cython==0.26.1
+  - cython==0.27.1
   - decorator==4.1.2
   - empyrical==0.2.1
-  - enigma-catalyst>=0.2.dev2
-  - enum34==1.1.6
-  - functools32==3.2.3.post2
   - idna==2.6
   - intervaltree==2.1.0
-  - ipdb==0.10.3
-  - ipdbplugin==1.4.5
-  - ipython==5.5.0
-  - ipython-genutils==0.2.0
   - logbook==1.1.0
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.0.2
+  - matplotlib==2.1.0
   - multipledispatch==0.4.9
-  - networkx==1.11
+  - networkx==2.0
   - numexpr==2.6.4
-  - numpy==1.13.1
   - pandas==0.19.2
   - pandas-datareader==0.5.0
-  - pathlib2==2.3.0
   - patsy==0.4.1
-  - pexpect==4.2.1
-  - pickleshare==0.7.4
-  - prompt-toolkit==1.0.15
-  - ptyprocess==0.5.2
-  - pygments==2.2.0
   - pyparsing==2.2.0
   - python-dateutil==2.6.1
   - python-editor==1.0.3
@@ -69,16 +47,12 @@ dependencies:
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1
-  - scandir==1.5
+  - six==1.11.0
-  - scipy==0.19.1
-  - scons==3.0.0a20170821
-  - simplegeneric==0.8.1
   - sortedcontainers==1.5.7
   - sqlalchemy==1.1.14
   - statsmodels==0.8.0
   - subprocess32==3.2.7
   - tables==3.4.2
   - toolz==0.8.2
-  - traitlets==4.3.2
   - urllib3==1.22
-  - wcwidth==0.1.7
+  - enigma-catalyst>=0.3

etc/requirements.txt

@@ -1,7 +1,7 @@
 # Incompatible with earlier PIP versions
 pip>=7.1.0
 # bcolz fails to install if this is not in the build_requires.
-setuptools>18.0
+setuptools>36.0
 
 # Logging
 Logbook==0.12.5

etc/requirements_docs.txt

@@ -1,4 +1,3 @@
 Sphinx>=1.3.2
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
-enigma-catalyst		# readthedocs.org

setup.py

@@ -304,7 +304,7 @@ setup(
                   if '__pycache__' not in root},
     license='Apache 2.0',
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
+        'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Programming Language :: Python',

tests/exchange/test_bcolz.py

@@ -0,0 +1,150 @@
+import shutil
+import random
+import tempfile
+import pandas as pd
+
+from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \
+    BcolzExchangeBarReader
+
+from catalyst.exchange.bundle_utils import get_df_from_arrays
+
+from nose.tools import assert_equals
+
+
+class TestBcolzWriter(object):
+    @classmethod
+    def setup_class(cls):
+        cls.columns = ['open', 'high', 'low', 'close', 'volume']
+
+    def setUp(self):
+        self.root_dir = tempfile.mkdtemp()  # Create a temporary directory
+
+    def tearDown(self):
+        shutil.rmtree(self.root_dir)  # Remove the directory after the test
+
+    def generate_df(self, exchange_name, freq, start, end):
+        bundle = ExchangeBundle(exchange_name)
+        index = bundle.get_calendar_periods_range(start, end, freq)
+        df = pd.DataFrame(index=index, columns=self.columns)
+        df.fillna(random.random(), inplace=True)
+        return df
+
+    def test_bcolz_write_daily_past(self):
+        start = pd.to_datetime('2016-01-01')
+        end = pd.to_datetime('2016-12-31')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_daily_present(self):
+        start = pd.to_datetime('2017-01-01')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def test_bcolz_write_minute_past(self):
+        start = pd.to_datetime('2015-04-01 00:00')
+        end = pd.to_datetime('2015-04-30 23:59')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        pass
+
+    def test_bcolz_write_minute_present(self):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'minute'
+
+        df = self.generate_df('bitfinex', freq, start, end)
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=start,
+            end_session=end,
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+        pass
+
+    def bcolz_exchange_daily_write_read(self, exchange_name):
+        start = pd.to_datetime('2017-10-01 00:00')
+        end = pd.to_datetime('today')
+        freq = 'daily'
+
+        bundle = ExchangeBundle(exchange_name)
+
+        df = self.generate_df(exchange_name, freq, start, end)
+
+        print df.index[0],df.index[-1]
+
+        writer = BcolzExchangeBarWriter(
+            rootdir=self.root_dir,
+            start_session=df.index[0],
+            end_session=df.index[-1],
+            data_frequency=freq,
+            write_metadata=True)
+
+        data = []
+        data.append((1, df))
+        writer.write(data)
+
+        reader = BcolzExchangeBarReader(rootdir=self.root_dir,
+                                        data_frequency=freq)
+
+        arrays = reader.load_raw_arrays(self.columns, start, end, [1, ])
+
+        periods = bundle.get_calendar_periods_range(
+            start, end, freq
+        )
+
+        dx = get_df_from_arrays(arrays, periods)
+
+        assert_equals(df.equals(df), True)
+        pass
+
+    def test_bcolz_bitfinex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('bitfinex')
+
+    def test_bcolz_poloniex_daily_write_read(self):
+        self.bcolz_exchange_daily_write_read('poloniex')

tests/exchange/test_bitfinex.py

@@ -8,7 +8,7 @@ from catalyst.finance.execution import (LimitOrder)
 log = Logger('test_bitfinex')
 
 
-class BitfinexTestCase(BaseExchangeTestCase):
+class TestBitfinexTestCase(BaseExchangeTestCase):
     @classmethod
     def setup(self):
         log.info('creating bitfinex object')

tests/exchange/test_bittrex.py

@@ -1,3 +1,4 @@
+import pandas as pd
 from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
@@ -7,15 +8,15 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 log = Logger('test_bittrex')
 
 
-class BittrexTestCase(BaseExchangeTestCase):
+class TestBittrex(BaseExchangeTestCase):
     @classmethod
     def setup(self):
-        print ('creating bittrex object')
         auth = get_exchange_auth('bittrex')
         self.exchange = Bittrex(
             key=auth['key'],
             secret=auth['secret'],
-            base_currency='btc'
+            base_currency=None,
+            portfolio=None
         )
 
     def test_order(self):
@@ -52,15 +53,18 @@ class BittrexTestCase(BaseExchangeTestCase):
         log.info('retrieving candles')
         ohlcv_neo = self.exchange.get_candles(
             data_frequency='5m',
-            assets=self.exchange.get_asset('neo_btc')
+            assets=self.exchange.get_asset('neo_btc'),
+            bar_count=20,
+            end_dt=pd.to_datetime('2017-10-20', utc=True)
         )
         ohlcv_neo_ubq = self.exchange.get_candles(
-            data_frequency='5m',
+            data_frequency='1d',
             assets=[
                 self.exchange.get_asset('neo_btc'),
                 self.exchange.get_asset('ubq_btc')
             ],
-            bar_count=14
+            bar_count=14,
+            end_dt=pd.to_datetime('2017-10-20', utc=True)
         )
         pass
 

tests/exchange/test_bundle.py

@@ -1,21 +1,43 @@
-from logging import Logger
+import hashlib
+from logging import getLogger
 
 import pandas as pd
 
 from catalyst import get_calendar
-from catalyst.exchange.bundle_utils import get_bcolz_chunk
+from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
+    get_periods_range, get_start_dt
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.stats_utils import df_to_string
 from catalyst.utils.paths import ensure_directory
 
-log = Logger('test_exchange_bundle')
+log = getLogger('test_exchange_bundle')
 
 
-class ExchangeBundleTestCase:
+class TestExchangeBundle:
+    def test_spot_value(self):
+        data_frequency = 'daily'
+        exchange_name = 'poloniex'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [
+            exchange.get_asset('btc_usdt')
+        ]
+        dt = pd.to_datetime('2017-10-14', utc=True)
+
+        values = exchange_bundle.get_spot_values(
+            assets=assets,
+            field='close',
+            dt=dt,
+            data_frequency=data_frequency
+        )
+        pass
+
     def test_ingest_minute(self):
         data_frequency = 'minute'
         exchange_name = 'bitfinex'
@@ -23,11 +45,11 @@ class ExchangeBundleTestCase:
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
         assets = [
-            exchange.get_asset('neo_eth')
+            exchange.get_asset('iot_btc')
         ]
 
         # start = pd.to_datetime('2017-09-01', utc=True)
-        start = pd.to_datetime('2017-9-15', utc=True)
+        start = pd.to_datetime('2017-9-01', utc=True)
         end = pd.to_datetime('2017-9-30', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
@@ -73,17 +95,41 @@ class ExchangeBundleTestCase:
         )
         pass
 
-    def test_ingest_daily(self):
+    def test_ingest_exchange(self):
         # exchange_name = 'bitfinex'
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
         exchange_name = 'bitfinex'
-        data_frequency = 'daily'
+        data_frequency = 'minute'
-        include_symbols = 'etc_btc'
 
-        start = pd.to_datetime('2016-11-01', utc=True)
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+
+        log.info('ingesting exchange bundle {}'.format(exchange_name))
+        exchange_bundle.ingest(
+            data_frequency=data_frequency,
+            include_symbols=None,
+            exclude_symbols=None,
+            start=None,
+            end=None,
+            show_progress=True
+        )
+
+        pass
+
+    def test_ingest_daily(self):
+        # exchange_name = 'bitfinex'
+        # data_frequency = 'daily'
+        # include_symbols = 'neo_btc,bch_btc,eth_btc'
+
+        exchange_name = 'bittrex'
+        data_frequency = 'daily'
+        include_symbols = 'wings_eth'
+
+        start = pd.to_datetime('2017-1-1', utc=True)
         end = pd.to_datetime('2017-10-16', utc=True)
+        periods = get_periods_range(start, end, data_frequency)
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
@@ -253,7 +299,7 @@ class ExchangeBundleTestCase:
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)
-        asset = exchange.get_asset('neo_btc')
+        asset = exchange.get_asset('neos_btc')
 
         path = get_bcolz_chunk(
             exchange_name=exchange_name,
@@ -263,3 +309,120 @@ class ExchangeBundleTestCase:
         )
 
         pass
+
+    def test_hash_symbol(self):
+        symbol = 'etc_btc'
+        sid = int(
+            hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
+        ) % 10 ** 6
+        pass
+
+    def test_validate_data(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-9-2 1:00', utc=True)
+        bar_count = 60
+
+        bundle_series = exchange_bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count * 5,
+            field='close',
+            data_frequency='minute',
+        )
+        candles = exchange.get_candles(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            data_frequency='minute'
+        )
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+
+        frames = []
+        for asset in assets:
+            bundle_df = pd.DataFrame(
+                data=dict(bundle_price=bundle_series[asset]),
+                index=bundle_series[asset].index
+            )
+            exchange_series = exchange.get_series_from_candles(
+                candles=candles[asset],
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                field='close'
+            )
+            exchange_df = pd.DataFrame(
+                data=dict(exchange_price=exchange_series),
+                index=exchange_series.index
+            )
+
+            df = exchange_df.join(bundle_df, how='left')
+            df['last_traded'] = df.index
+            df['asset'] = asset.symbol
+            df.set_index(['asset', 'last_traded'], inplace=True)
+
+            frames.append(df)
+
+        df = pd.concat(frames)
+        print('\n' + df_to_string(df))
+        pass
+
+    def test_ingest_candles(self):
+        exchange_name = 'bitfinex'
+        data_frequency = 'minute'
+
+        exchange = get_exchange(exchange_name)
+        bundle = ExchangeBundle(exchange)
+        assets = [exchange.get_asset('iot_btc')]
+
+        end_dt = pd.to_datetime('2017-10-20', utc=True)
+        bar_count = 100
+
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        candles = exchange.get_candles(
+            assets=assets,
+            start_dt=start_dt,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            data_frequency=data_frequency
+        )
+
+        writer = bundle.get_writer(start_dt, end_dt, data_frequency)
+        for asset in assets:
+            dates = [candle['last_traded'] for candle in candles[asset]]
+
+            values = dict()
+            for field in ['open', 'high', 'low', 'close', 'volume']:
+                values[field] = [candle[field] for candle in candles[asset]]
+
+            periods = bundle.get_calendar_periods_range(
+                start_dt, end_dt, data_frequency
+            )
+            df = pd.DataFrame(values, index=dates)
+            df = df.loc[periods].fillna(method='ffill')
+
+            # TODO: why do I get an extra bar?
+            bundle.ingest_df(
+                ohlcv_df=df,
+                data_frequency=data_frequency,
+                asset=asset,
+                writer=writer,
+                empty_rows_behavior='raise'
+            )
+
+        bundle_series = bundle.get_history_window_series(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            field='close',
+            data_frequency=data_frequency,
+            reset_reader=True
+        )
+        df = pd.DataFrame(bundle_series)
+        print('\n' + df_to_string(df))
+        pass

tests/exchange/test_clock.py

@@ -1,50 +0,0 @@
-from unittest import TestCase
-from logbook import Logger
-from mock import patch, sentinel
-from catalyst.exchange.simple_clock import SimpleClock
-from catalyst.utils.calendars.trading_calendar import days_at_time
-from datetime import time
-from collections import defaultdict
-from catalyst.utils.calendars import get_calendar
-import pandas as pd
-
-log = Logger('ExchangeClockTestCase')
-
-
-class ExchangeClockTestCase(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.open_calendar = get_calendar("OPEN")
-
-        cls.sessions = pd.Timestamp.utcnow()
-
-    def setUp(self):
-        self.internal_clock = None
-        self.events = defaultdict(list)
-
-    def advance_clock(self, x):
-        """Mock function for sleep. Advances the internal clock by 1 min"""
-        # The internal clock advance time must be 1 minute to match
-        # MinutesSimulationClock's update frequency
-        self.internal_clock += pd.Timedelta('1 min')
-
-    def get_clock(self, arg, *args, **kwargs):
-        """Mock function for pandas.to_datetime which is used to query the
-        current time in RealtimeClock"""
-        assert arg == "now"
-        return self.internal_clock
-
-    def test_clock(self):
-        with patch('catalyst.exchange.simple_clock.pd.to_datetime') as to_dt, \
-                patch('catalyst.exchange.simple_clock.sleep') as sleep:
-            clock = SimpleClock(sessions=self.sessions)
-            to_dt.side_effect = self.get_clock
-            sleep.side_effect = self.advance_clock
-            start_time = pd.Timestamp.utcnow()
-            self.internal_clock = start_time
-
-            events = list(clock)
-
-            # Event 0 is SESSION_START which always happens at 00:00.
-            ts, event_type = events[1]
-        pass

tests/exchange/test_data_portal.py

@@ -12,7 +12,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 log = Logger('test_bitfinex')
 
 
-class ExchangeDataPortalTestCase:
+class TestExchangeDataPortalTestCase:
     @classmethod
     def setup(self):
         log.info('creating bitfinex exchange')

tests/exchange/test_poloniex.py

@@ -8,7 +8,7 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 log = Logger('test_poloniex')
 
 
-class PoloniexTestCase(BaseExchangeTestCase):
+class TestPoloniexTestCase(BaseExchangeTestCase):
     @classmethod
     def setup(self):
         print ('creating poloniex object')
@@ -21,7 +21,7 @@ class PoloniexTestCase(BaseExchangeTestCase):
 
     def test_order(self):
         log.info('creating order')
-        asset = self.exchange.get_asset('neo_btc')
+        asset = self.exchange.get_asset('neos_btc')
         order_id = self.exchange.order(
             asset=asset,
             limit_price=0.0005,
@@ -33,7 +33,7 @@ class PoloniexTestCase(BaseExchangeTestCase):
 
     def test_open_orders(self):
         log.info('retrieving open orders')
-        asset = self.exchange.get_asset('neo_btc')
+        asset = self.exchange.get_asset('neos_btc')
         orders = self.exchange.get_open_orders(asset)
         pass
 
@@ -53,13 +53,13 @@ class PoloniexTestCase(BaseExchangeTestCase):
         log.info('retrieving candles')
         ohlcv_neo = self.exchange.get_candles(
             data_frequency='5m',
-            assets=self.exchange.get_asset('neo_btc')
+            assets=self.exchange.get_asset('neos_btc')
         )
         ohlcv_neo_ubq = self.exchange.get_candles(
             data_frequency='5m',
             assets=[
-                self.exchange.get_asset('neo_btc'),
+                self.exchange.get_asset('neos_btc'),
-                self.exchange.get_asset('ubq_btc')
+                self.exchange.get_asset('via_btc')
             ],
             bar_count=14
         )