Merge branch 'develop' -> release 0.3

catalyst/__main__.py

@@ -498,7 +498,7 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     exchange = get_exchange(exchange_name)
     exchange_bundle = ExchangeBundle(exchange)
 
-    click.echo('ingesting exchange bundle {}'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,

catalyst/algorithm.py

@@ -138,8 +138,9 @@ from catalyst.gens.sim_engine import MinuteSimulationClock
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
+from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger("ZiplineLog")
+log = logbook.Logger("CatalystLog", level=LOG_LEVEL)
 
 
 class TradingAlgorithm(object):

catalyst/assets/assets.py

@@ -76,7 +76,9 @@ from catalyst.utils.numpy_utils import as_column
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.sqlite_utils import group_into_chunks, coerce_string_to_eng
 
-log = Logger('assets.py')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('assets.py', level=LOG_LEVEL)
 
 # A set of fields that need to be converted to strings before building an
 # Asset to avoid unicode fields

catalyst/constants.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+import logbook
+
+LOG_LEVEL = logbook.INFO

catalyst/data/_equities.pyx

@@ -215,7 +215,7 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in ['open', 'high', 'low', 'close']:
+        if column_name in ['open', 'high', 'low', 'close', 'volume']:
             where_nan = (outbuf == 0)
             outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN

catalyst/data/bundles/base.py

@@ -30,8 +30,10 @@ from catalyst.utils.cli import (
 )
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
+
 logbook.StderrHandler().push_application()
-log = logbook.Logger(__name__)
+log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 

catalyst/data/bundles/quandl.py

@@ -40,7 +40,9 @@ from catalyst.utils.cli import maybe_show_progress
 
 from . import core as bundles
 
-log = Logger(__name__)
+from catalyst.constants import LOG_LEVEL
+
+log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
 class QuandlBundle(BaseEquityPricingBundle):

catalyst/data/data_portal.py

@@ -68,7 +68,9 @@ from catalyst.errors import (
     HistoryWindowStartsBeforeData,
 )
 
-log = Logger('DataPortal')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('DataPortal', level=LOG_LEVEL)
 
 BASE_FIELDS = frozenset([
     "open",

catalyst/data/loader.py

@@ -32,7 +32,9 @@ from ..utils.paths import (
     data_root,
 )
 
-logger = logbook.Logger('Loader')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
 INDEX_MAPPING = {
@@ -95,7 +97,8 @@ def has_data_for_dates(series_or_df, first_date, last_date):
 
 def load_crypto_market_data(trading_day=None, trading_days=None,
                             bm_symbol=None, bundle=None, bundle_data=None,
-                            environ=None, exchange=None):
+                            environ=None, exchange=None, start_dt=None,
+                            end_dt=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
 
@@ -104,8 +107,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
-    first_date = get_calendar('OPEN').first_trading_session
+    # if start_dt is None:
-    now = pd.Timestamp.utcnow()
+    start_dt = get_calendar('OPEN').first_trading_session
+
+    if end_dt is None:
+        end_dt = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
     # **two** full trading days prior to the most recently completed trading
@@ -131,7 +137,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 1]
+    last_date = trading_days[trading_days.get_loc(end_dt, method='ffill') - 1]
 
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
@@ -146,14 +152,14 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     br = exchange.get_history_window(
         assets=[benchmark_asset],
         end_dt=last_date,
-        bar_count=pd.Timedelta(last_date - first_date).days,
+        bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
         data_frequency='daily')
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
-    br.loc[first_date]=0
+    br.loc[start_dt] = 0
-    br=br.sort_index()
+    br = br.sort_index()
 
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
@@ -162,10 +168,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         bm_symbol,
         first_date_treasury,
         last_date,
-        now,
+        end_dt,
         environ,
     )
-    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
+    benchmark_returns = br[br.index.slice_indexer(start_dt, last_date)]
     treasury_curves = tc[
         tc.index.slice_indexer(first_date_treasury, last_date)]
     return benchmark_returns, treasury_curves

catalyst/data/minute_bars.py

@@ -44,8 +44,9 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
 
-logger = logbook.Logger('MinuteBars')
+logger = logbook.Logger('MinuteBars', level=LOG_LEVEL)
 
 US_EQUITIES_MINUTES_PER_DAY = 390
 FUTURES_MINUTES_PER_DAY = 1440
@@ -1125,7 +1126,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             else:
                 return np.nan
 
-        #if field != 'volume':
+        # if field != 'volume':
         value *= self._ohlc_ratio_inverse_for_sid(sid)
         return value
 
@@ -1206,7 +1207,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             minute_dt.value / NANOS_IN_MINUTE,
             self._minutes_per_day,
             False,
-            )
+        )
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
         """
@@ -1262,10 +1263,10 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 where = values != 0
                 # first slice down to len(where) because we might not have
                 # written data for all the minutes requested
-                #if field != 'volume':
+                # if field != 'volume':
                 out[:len(where), i][where] = (
                     values[where] * self._ohlc_ratio_inverse_for_sid(sid))
-                #else:
+                # else:
                 #    out[:len(where), i][where] = values[where]
 
             results.append(out)
@@ -1353,6 +1354,7 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
     path : str
         The path of the HDF5 file from which to source data.
     """
+
     def __init__(self, path):
         self._panel = pd.read_hdf(path)
 

catalyst/data/us_equity_pricing.py

@@ -83,7 +83,9 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-logger = logbook.Logger('UsEquityPricing')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('UsEquityPricing', level=LOG_LEVEL)
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
 OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])

catalyst/examples/buy_btc_simple.py

@@ -0,0 +1,8 @@
+from catalyst.api import order, record, symbol
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc = data.current(context.asset, 'price'))

catalyst/examples/buy_low_sell_high_live.py

@@ -1,6 +1,7 @@
 import talib
 from logbook import Logger
 
+import pandas as pd
 from catalyst.api import (
     order,
     order_target_percent,
@@ -17,10 +18,10 @@ log = Logger('buy low sell high')
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_BTC'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 300
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
     context.SLIPPAGE_ALLOWED = 0.02
 
@@ -33,31 +34,31 @@ def initialize(context):
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1d'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
+        buy_increment = 0.5
-    # elif rsi <= 70:
+    elif rsi <= 70:
-    #    buy_increment = 5
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -146,11 +147,22 @@ def analyze(context, stats):
 
 
 run_algorithm(
+    capital_base=100000,
     initialize=initialize,
     handle_data=handle_data,
     analyze=analyze,
-    exchange_name='bitfinex',
+    exchange_name='poloniex',
-    live=True,
+    start=pd.to_datetime('2017-5-01', utc=True),
-    algo_namespace=algo_namespace,
+    end=pd.to_datetime('2017-10-16', utc=True),
-    base_currency='btc'
+    base_currency='usdt',
+    data_frequency='daily'
 )
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc'
+# )

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -163,8 +163,6 @@ def analyze(context, stats):
 # Backtest
 run_algorithm(
     capital_base=250,
-    start=pd.to_datetime('2017-10-01', utc=True),
-    end=pd.to_datetime('2017-10-15', utc=True),
     data_frequency='minute',
     initialize=initialize,
     handle_data=handle_data,

catalyst/exchange/asset_finder_exchange.py

@@ -1,6 +1,8 @@
 from logbook import Logger
 
-log = Logger('AssetFinderExchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('AssetFinderExchange', level=LOG_LEVEL)
 
 
 class AssetFinderExchange(object):
@@ -41,9 +43,9 @@ class AssetFinderExchange(object):
         """
         for sid in sids:
             if sid in self._asset_cache:
-                log.info('got asset from cache: {}'.format(sid))
+                log.debug('got asset from cache: {}'.format(sid))
             else:
-                log.info('fetching asset: {}'.format(sid))
+                log.debug('fetching asset: {}'.format(sid))
         return list()
 
     def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):

catalyst/exchange/bitfinex/bitfinex.py

@@ -33,7 +33,9 @@ requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-log = Logger('Bitfinex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 
@@ -56,7 +58,7 @@ class Bitfinex(Exchange):
 
         # Max is 90 but playing it safe
         # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -665,10 +667,11 @@ class Bitfinex(Exchange):
         return time.strftime('%Y-%m-%d',
                              time.gmtime(int(response.json()[-1][0] / 1000)))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         exchange_symbol = asset.exchange_symbol
         try:
             self.ask_request()
+            # TODO: implement limit
             response = self._request(
                 'book/{}'.format(exchange_symbol), None)
             data = response.json()

catalyst/exchange/bittrex/bittrex.py

@@ -16,7 +16,9 @@ from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
 
-log = Logger('Bittrex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 
@@ -358,7 +360,7 @@ class Bittrex(Exchange):
             json.dump(symbol_map, f, sort_keys=True, indent=2,
                       separators=(',', ':'))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         if order_type == 'all':
             order_type = 'both'
         elif order_type == 'bid':
@@ -369,7 +371,11 @@ class Bittrex(Exchange):
             raise ValueError('invalid type')
 
         exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(market=exchange_symbol, type=order_type)
+        data = self.api.getorderbook(
+            market=exchange_symbol,
+            type=order_type,
+            depth=100
+        )
 
         result = dict()
         for exchange_type in data:

catalyst/exchange/bundle_utils.py

@@ -1,18 +1,15 @@
 import calendar
-import tarfile
-
-import requests
-from datetime import timedelta, datetime, date
 import os
-import pandas as pd
+import tarfile
-import numpy as np
+from datetime import timedelta, datetime, date
 
+import numpy as np
+import pandas as pd
 import pytz
 
 from catalyst.data.bundles import from_bundle_ingest_dirname
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_errors import ApiCandlesError, \
+from catalyst.exchange.exchange_errors import NoDataAvailableOnExchange
-    PricingDataBeforeTradingError, NoDataAvailableOnExchange
 from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_path
@@ -189,60 +186,6 @@ def get_df_from_arrays(arrays, periods):
     return df
 
 
-def get_df_from_candles(candles, bar_count, end_dt, data_frequency,
-                        previous_candle=None):
-    """
-    Create candles for each period of the specified range, forward-filling
-    missing candles with the previous value.
-
-    :param candles:
-    :param bar_count:
-    :param end_dt:
-    :param data_frequency:
-    :param previous_candle:
-
-    :return:
-    """
-    all_dates = []
-    all_candles = []
-
-    start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-    date = start_dt
-
-    # TODO: this works well with a small number of candles, consider using numpy as needed
-    while date <= end_dt:
-        candle = next((
-            candle for candle in candles if candle['last_traded'] == date
-        ), previous_candle)
-
-        if candle is None:
-            candle = candles[0]
-
-        all_dates.append(date)
-        all_candles.append(candle)
-
-        previous_candle = candle
-
-        date += get_delta(1, data_frequency)
-
-    return all_dates, all_candles
-
-
-def get_trailing_candles_dt(asset, start_dt, end_dt, data_frequency):
-    missing_start = None
-
-    if asset.end_minute is not None and start_dt < asset.end_minute:
-        if asset.end_minute < end_dt:
-            delta = get_delta(1, data_frequency)
-
-            missing_start = asset.end_minute + delta
-
-    else:
-        missing_start = start_dt
-
-    return missing_start
-
-
 def range_in_bundle(asset, start_dt, end_dt, reader):
     """
     Evaluate whether price data of an asset is included has been ingested in
@@ -278,6 +221,7 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
     return has_data
 
 
+@deprecated
 def find_most_recent_time(bundle_name):
     """
     Find most recent "time folder" for a given bundle.
@@ -308,83 +252,3 @@ def find_most_recent_time(bundle_name):
     else:
         return None
 
-
-@deprecated
-def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
-    """
-    History API provides OHLCV data for any of the supported exchanges up to yesterday.
-
-    :param exchange_name: string
-        Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex).
-    :param data_frequency: string
-        Required: The bar frequency (minute or daily)
-    :param symbol: string
-        Required: The trading pair symbol, using Catalyst naming convention
-    :param start: datetime
-        Optional: The start date.
-    :param end: datetime
-        Optional: The end date.
-
-    :return ohlcv: list[dict[string, float]]
-        Each row contains the following dictionary for the resulting bars:
-        'ts'     : int, the timestamp in seconds
-        'open'   : float
-        'high'   : float
-        'low'    : float
-        'close'  : float
-        'volume' : float
-
-    Notes
-    =====
-    Using seconds for the start and end dates for ease of use in the
-    function query parameters.
-
-    Sometimes, one minute goes by without completing a trade of the given
-    trading pair on the given exchange. To minimize the payload size, we
-    don't return identical sequential bars. Post-processing code will
-    forward fill missing bars outside of this function.
-    """
-
-    start_seconds = get_seconds_from_date(start) if start else None
-    end_seconds = get_seconds_from_date(end) if end else None
-
-    if exchange_name not in EXCHANGE_NAMES:
-        raise ValueError(
-            'get_history function only supports the following exchanges: {}'.format(
-                list(EXCHANGE_NAMES)))
-
-    if data_frequency != 'daily' and data_frequency != 'minute':
-        raise ValueError(
-            'get_history currently only supports daily and minute data.'
-        )
-
-    url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format(
-        api_url=API_URL,
-        exchange=exchange_name,
-        symbol=symbol,
-        data_frequency=data_frequency,
-    )
-
-    if start_seconds:
-        url += '&start={}'.format(start_seconds)
-
-    if end_seconds:
-        url += '&end={}'.format(end_seconds)
-
-    try:
-        response = requests.get(url)
-    except Exception as e:
-        raise ValueError(e)
-
-    data = response.json()
-
-    if 'error' in data:
-        raise ApiCandlesError(error=data['error'])
-
-    for candle in data:
-        last_traded = pd.Timestamp.utcfromtimestamp(candle['ts'])
-        last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-        candle['last_traded'] = last_traded
-
-    return data

catalyst/exchange/data_portal_exchange.py

@@ -12,15 +12,15 @@
 # limitations under the License.
 
 import abc
-from datetime import timedelta
 from time import sleep
 
+import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
 from catalyst.data.data_portal import DataPortal
-from catalyst.errors import HistoryWindowStartsBeforeData
+from catalyst.exchange.bundle_utils import get_start_dt
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
@@ -29,7 +29,9 @@ from catalyst.exchange.exchange_errors import (
     PricingDataNotLoadedError, InvalidHistoryFrequencyError,
     BundleNotFoundError)
 
-log = Logger('DataPortalExchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):
@@ -153,6 +155,10 @@ class DataPortalExchangeBase(DataPortal):
                 exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
                     exchange, [assets], field, dt, data_frequency)
+
+                if not spot_values:
+                    return np.nan
+
                 return spot_values[0]
 
             else:
@@ -282,109 +288,60 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange bundle.
+
+        Using a try... except approach to minimize reads most of the time,
+        when the data exists.
+
+        :param exchange:
+        :param assets:
+        :param end_dt:
+        :param bar_count:
+        :param frequency:
+        :param field:
+        :param data_frequency:
+        :param ffill:
+        :return:
+        """
+
         bundle = self.exchange_bundles[exchange.name]
-
+        series = bundle.get_history_window_series_and_load(
-        if data_frequency == 'minute':
+            assets=assets,
-            dts = self.trading_calendar.minutes_window(
+            end_dt=end_dt,
-                end_dt, -bar_count
+            bar_count=bar_count,
-            )
+            field=field,
-
+            data_frequency=data_frequency
-            self.ensure_after_first_day(dts[0], assets)
+        )
-
-        elif data_frequency == 'daily':
-            session = self.trading_calendar.minute_to_session_label(end_dt)
-            dts = self._get_days_for_window(session, bar_count)
-
-            if len(dts) == 0:
-                symbols = [asset.symbol for asset in assets]
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    symbols=symbols,
-                    exchange=exchange.name,
-                    first_trading_day= \
-                        min([asset.start_date for asset in assets]),
-                    data_frequency=data_frequency,
-                    symbol_list=','.join(symbols)
-                )
-
-            self.ensure_after_first_day(dts[0], assets)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=data_frequency)
-
-        reader = bundle.get_reader(data_frequency)
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=exchange.name.title(),
-                data_frequency=data_frequency
-            )
-
-        try:
-            values = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=dts[0],
-                end_dt=dts[-1]
-            )[0]
-
-        except Exception:
-            first_trading_day = self._get_first_trading_day(assets)
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-
-            symbol_list = ','.join(symbols)
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=first_trading_day,
-                exchange=exchange.name.title(),
-                symbols=symbols,
-                symbol_list=symbol_list,
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        for index, asset in enumerate(assets):
-            asset_values = values[:, index]
-
-            value_series = pd.Series(asset_values, index=dts)
-            series[asset] = value_series
-
         return pd.DataFrame(series)
 
-    def ensure_after_first_day(self, dt, assets):
-        first_trading_day = self._get_first_trading_day(assets)
-        if dt < first_trading_day:
-            raise PricingDataBeforeTradingError(
-                first_trading_day=first_trading_day,
-                exchange=assets[0].exchange.title(),
-                symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                dt=dt,
-            )
-
     def get_exchange_spot_value(self, exchange, assets, field, dt,
                                 data_frequency):
         bundle = self.exchange_bundles[exchange.name]
-        reader = bundle.get_reader(data_frequency)
 
-        self.ensure_after_first_day(dt, assets)
+        if data_frequency == 'daily':
+            dt = dt.floor('1D')
+        else:
+            dt = dt.floor('1 min')
 
-        values = []
+        try:
-        for asset in assets:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)
-            try:
+
-                value = reader.get_value(
+        except PricingDataNotLoadedError:
-                    sid=asset.sid,
+            log.info(
-                    dt=dt,
+                'pricing data for {symbol} not found on {dt}'
-                    field=field
+                ', updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    dt=dt
                 )
-                values.append(value)
+            )
-            except Exception:
+            bundle.ingest_assets(
-                raise PricingDataNotLoadedError(
+                assets=assets,
-                    field=field,
+                start_dt=self._first_trading_day,
-                    first_trading_day=self._get_first_trading_day(assets),
+                end_dt=self._last_available_session,
-                    exchange=exchange.name.title(),
+                data_frequency=data_frequency,
-                    symbols=[asset.symbol.encode('utf-8') for asset in assets],
+                show_progress=True
-                    symbol_list=''.join(
+            )
-                        [asset.symbol.encode('utf-8') for asset in assets]),
+            return bundle.get_spot_values(
-                    data_frequency=data_frequency
+                assets, field, dt, data_frequency, True
-                )
+            )
-
-            return values

catalyst/exchange/exchange.py

@@ -16,7 +16,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
     InvalidHistoryFrequencyError, MismatchingFrequencyError, \
-    BundleNotFoundError, NoDataAvailableOnExchange
+    BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,7 +24,9 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
-log = Logger('Exchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Exchange', level=LOG_LEVEL)
 
 
 class Exchange:
@@ -370,44 +372,6 @@ class Exchange:
 
         return value
 
-    def get_series_from_bundle(self, assets, start_dt, end_dt, data_frequency,
-                               field):
-        """
-
-        :return:
-        """
-        reader = self.bundle.get_reader(data_frequency)
-
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=self.name.title(),
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        try:
-            arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=start_dt,
-                end_dt=end_dt
-            )
-
-            periods = self.bundle.get_calendar_periods_range(
-                start_dt, end_dt, data_frequency
-            )
-
-            for asset_index, asset in enumerate(assets):
-                asset_values = arrays[asset_index]
-
-                value_series = pd.Series(asset_values[0], index=periods)
-                series[asset] = value_series
-
-        except Exception as e:
-            log.debug('unable to retrieve from bundle: {}'.format(e))
-
-        return series
-
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 field, previous_value=None):
         """
@@ -487,11 +451,6 @@ class Exchange:
                 data_frequency = 'daily'
 
         elif unit.lower() == 'm':
-            # if data_frequency != 'minute':
-            # raise MismatchingFrequencyError(
-            #     frequency=frequency,
-            #     data_frequency=data_frequency
-            # )
             if data_frequency == 'daily':
                 data_frequency = 'minute'
 
@@ -499,42 +458,15 @@ class Exchange:
             raise InvalidHistoryFrequencyError(frequency)
 
         adj_bar_count = candle_size * bar_count
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         try:
-            adj_start_dt, adj_end_dt = get_adj_dates(
+            series = self.bundle.get_history_window_series_and_load(
-                start_dt, end_dt, assets, data_frequency
-            )
-            in_bundle = True
-
-        except NoDataAvailableOnExchange:
-            in_bundle = False
-
-        if in_bundle:
-            missing_assets = self.bundle.filter_existing_assets(
                 assets=assets,
-                start_dt=adj_start_dt,
+                end_dt=end_dt,
-                end_dt=adj_end_dt,
+                bar_count=adj_bar_count,
+                field=field,
                 data_frequency=data_frequency
             )
-
+        except PricingDataNotLoadedError:
-            if missing_assets:
-                self.bundle.ingest_assets(
-                    assets=assets,
-                    start_dt=adj_start_dt,
-                    end_dt=adj_end_dt,
-                    data_frequency=data_frequency
-                )
-
-            series = self.get_series_from_bundle(
-                assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
-                data_frequency=data_frequency,
-                field=field
-            )
-
-        else:
             series = dict()
 
         for asset in assets:
@@ -542,7 +474,7 @@ class Exchange:
                 # Adding bars too recent to be contained in the consolidated
                 # exchanges bundles. We go directly against the exchange
                 # to retrieve the candles.
-
+                start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
                         if asset in series else start_dt

catalyst/exchange/exchange_algorithm.py

@@ -54,7 +54,9 @@ from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
 
-log = logbook.Logger('exchange_algorithm')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
 
 class ExchangeAlgorithmExecutor(AlgorithmSimulator):

catalyst/exchange/exchange_bcolz.py

@@ -48,9 +48,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
         # else:
         #     return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
 
-        return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
+        return self._load_raw_arrays(fields, start_dt, end_dt, sids)
 
-    def _load_daily_raw_arrays(self, fields, start_dt, end_dt, sids):
+    def _load_raw_arrays(self, fields, start_dt, end_dt, sids):
         start_idx = self._find_position_of_minute(start_dt)
         end_idx = self._find_position_of_minute(end_dt)
 

catalyst/exchange/exchange_blotter.py

@@ -6,7 +6,9 @@ from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import Transaction
 
-log = Logger('exchange_blotter')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos
 # Orders won't often close at Equity levels.

catalyst/exchange/exchange_bundle.py

@@ -10,26 +10,26 @@ from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_delta, get_adj_dates, get_month_start_end, \
-    get_year_start_end, get_periods_range, get_df_from_arrays
+    get_year_start_end, get_periods_range, get_df_from_arrays, get_start_dt
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange
+    TempBundleNotFoundError, NoDataAvailableOnExchange, \
+    PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('exchange_bundle', level=LOG_LEVEL)
+
+BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
 
 def _cachpath(symbol, type_):
     return '-'.join([symbol, type_])
 
-
-BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
-log = Logger('exchange_bundle')
-log.level = INFO
-
-
 class ExchangeBundle:
     def __init__(self, exchange):
         self.exchange = exchange
@@ -451,3 +451,152 @@ class ExchangeBundle:
         for frequency in data_frequency.split(','):
             self.ingest_assets(assets, start_dt, end_dt, frequency,
                                show_progress)
+
+    def get_history_window_series_and_load(self,
+                                           assets,
+                                           end_dt,
+                                           bar_count,
+                                           field,
+                                           data_frequency):
+        try:
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency
+            )
+            return pd.DataFrame(series)
+
+        except PricingDataNotLoadedError:
+            start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+            log.info(
+                'pricing data for {symbol} not found in range '
+                '{start} to {end}, updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    start=start_dt,
+                    end=end_dt
+                )
+            )
+            self.ingest_assets(
+                assets=assets,
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                show_progress=True
+            )
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency,
+                reset_reader=True
+            )
+            return series
+
+    def get_spot_values(self, assets, field, dt, data_frequency,
+                        reset_reader=False):
+        values = []
+        try:
+            reader = self.get_reader(data_frequency)
+            if reset_reader:
+                del self._readers[reader._rootdir]
+                reader = self.get_reader(data_frequency)
+
+            for asset in assets:
+                value = reader.get_value(
+                    sid=asset.sid,
+                    dt=dt,
+                    field=field
+                )
+                values.append(value)
+
+            return values
+
+        except Exception:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+    def get_history_window_series(self,
+                                  assets,
+                                  end_dt,
+                                  bar_count,
+                                  field,
+                                  data_frequency,
+                                  reset_reader=False):
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        start_dt, end_dt = \
+            get_adj_dates(start_dt, end_dt, assets, data_frequency)
+
+        reader = self.get_reader(data_frequency)
+        if reset_reader:
+            del self._readers[reader._rootdir]
+            reader = self.get_reader(data_frequency)
+
+        if reader is None:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        for asset in assets:
+            asset_start_dt, asset_end_dt = \
+                get_adj_dates(start_dt, end_dt, assets, data_frequency)
+
+            in_bundle = range_in_bundle(
+                asset, asset_start_dt, asset_end_dt, reader
+            )
+            if not in_bundle:
+                raise PricingDataNotLoadedError(
+                    field=field,
+                    first_trading_day=asset.start_date,
+                    exchange=self.exchange.name,
+                    symbols=asset.symbol,
+                    symbol_list=asset.symbol,
+                    data_frequency=data_frequency
+                )
+
+        series = dict()
+        try:
+            arrays = reader.load_raw_arrays(
+                sids=[asset.sid for asset in assets],
+                fields=[field],
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+
+        except Exception:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        periods = self.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+
+        for asset_index, asset in enumerate(assets):
+            asset_values = arrays[asset_index]
+
+            value_series = pd.Series(asset_values.flatten(), index=periods)
+            series[asset] = value_series
+
+        return series

catalyst/exchange/exchange_portfolio.py

@@ -3,7 +3,9 @@ from logbook import Logger
 
 from catalyst.protocol import Portfolio, Positions, Position
 
-log = Logger('ExchangePortfolio')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
 
 class ExchangePortfolio(Portfolio):

catalyst/exchange/live_graph_clock.py

@@ -22,8 +22,9 @@ from logbook import Logger
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
+from catalyst.constants import LOG_LEVEL
 
-log = Logger('LiveGraphClock')
+log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
 class LiveGraphClock(object):

catalyst/exchange/poloniex/poloniex.py

@@ -33,7 +33,9 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
 from catalyst.finance.transaction import Transaction
 
-log = Logger('Poloniex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Poloniex', level=LOG_LEVEL)
 
 
 class Poloniex(Exchange):
@@ -49,7 +51,7 @@ class Poloniex(Exchange):
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)

catalyst/finance/blotter.py

@@ -34,7 +34,9 @@ from catalyst.finance.commission import (
 from catalyst.finance.cancel_policy import NeverCancel
 from catalyst.utils.input_validation import expect_types
 
-log = Logger('Blotter')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Blotter', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/finance/controls.py

@@ -24,7 +24,9 @@ from catalyst.errors import (
     TradingControlViolation,
 )
 
-log = logbook.Logger('TradingControl')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('TradingControl', level=LOG_LEVEL)
 
 
 class TradingControl(with_metaclass(abc.ABCMeta)):

catalyst/finance/performance/period.py

@@ -88,7 +88,10 @@ from six import itervalues, iteritems
 
 import catalyst.protocol as zp
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
+
 TRADE_TYPE = zp.DATASOURCE_TYPE.TRADE
 
 

catalyst/finance/performance/position.py

@@ -40,7 +40,9 @@ import logbook
 from catalyst.assets import Future, Asset
 from catalyst.utils.input_validation import expect_types
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class Position(object):

catalyst/finance/performance/position_tracker.py

@@ -32,7 +32,9 @@ from catalyst.assets import (
 )
 from . position import positiondict
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 PositionStats = namedtuple('PositionStats',

catalyst/finance/performance/tracker.py

@@ -70,7 +70,9 @@ import catalyst.finance.risk as risk
 
 from . position_tracker import PositionTracker
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class PerformanceTracker(object):

catalyst/finance/risk/cumulative.py

@@ -38,7 +38,9 @@ from empyrical import (
     sortino_ratio,
 )
 
-log = logbook.Logger('Risk Cumulative')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
 
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',

catalyst/finance/risk/period.py

@@ -36,7 +36,9 @@ from empyrical import (
     sortino_ratio
 )
 
-log = logbook.Logger('Risk Period')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Period', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(risk.choose_treasury,
                                     risk.select_treasury_duration)

catalyst/finance/risk/report.py

@@ -63,7 +63,9 @@ from dateutil.relativedelta import relativedelta
 
 from . period import RiskMetricsPeriod
 
-log = logbook.Logger('Risk Report')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Report', level=LOG_LEVEL)
 
 
 class RiskReport(object):

catalyst/finance/risk/risk.py

@@ -61,7 +61,9 @@ Risk Report
 import logbook
 import numpy as np
 
-log = logbook.Logger('Risk')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk', level=LOG_LEVEL)
 
 
 TREASURY_DURATIONS = [

catalyst/finance/trading.py

@@ -26,7 +26,9 @@ from catalyst.data.loader import load_market_data
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.memoize import remember_last
 
-log = logbook.Logger('Trading')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Trading', level=LOG_LEVEL)
 
 
 DEFAULT_CAPITAL_BASE = 1e5

catalyst/gens/tradesimulation.py

@@ -27,7 +27,9 @@ from catalyst.gens.sim_engine import (
     BEFORE_TRADING_START_BAR
 )
 
-log = Logger('Trade Simulation')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Trade Simulation', level=LOG_LEVEL)
 
 
 class AlgorithmSimulator(object):

catalyst/sources/benchmark_source.py

@@ -72,7 +72,13 @@ class BenchmarkSource(object):
                             "benchmark_returns.")
 
     def get_value(self, dt):
-        return self._precalculated_series.loc[dt]
+        try:
+            series = self._precalculated_series
+            value = series.loc[dt]
+            return value
+        except Exception:
+            # TODO: workaround, find permanent fix
+            return 0
 
     def get_range(self, start_dt, end_dt):
         return self._precalculated_series.loc[start_dt:end_dt]

catalyst/sources/requests_csv.py

@@ -23,7 +23,9 @@ from catalyst.protocol import (
 )
 from catalyst.assets import Equity
 
-logger = Logger('Requests Source Logger')
+from catalyst.constants import LOG_LEVEL
+
+logger = Logger('Requests Source Logger', level=LOG_LEVEL)
 
 
 def roll_dts_to_midnight(dts, trading_day):

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,4 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-02-19', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/run_algo.py

@@ -43,7 +43,9 @@ from catalyst.exchange.exchange_utils import get_exchange_auth, \
     get_algo_object
 from logbook import Logger
 
-log = Logger('run_algo')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('run_algo', level=LOG_LEVEL)
 
 
 class _RunAlgoError(click.ClickException, ValueError):
@@ -191,7 +193,12 @@ def _run(handle_data,
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
-        load=partial(load_crypto_market_data, environ=environ),
+        load=partial(
+            load_crypto_market_data,
+            environ=environ,
+            start_dt=start,
+            end_dt=end
+        ),
         environ=environ,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
@@ -284,7 +291,8 @@ def _run(handle_data,
             exchanges=exchanges,
             asset_finder=None,
             trading_calendar=open_calendar,
-            first_trading_day=None,
+            first_trading_day=start,
+            last_available_session=end
         )
 
         sim_params = create_simulation_parameters(

docs/source/conf.py

@@ -41,7 +41,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC'
+copyright = u'2017, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
@@ -94,6 +94,6 @@ intersphinx_mapping = {
     'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None),
 }
 
-doctest_global_setup = "import zipline"
+doctest_global_setup = "import catalyst"
 
 todo_include_todos = True

docs/source/index.rst

@@ -1,12 +1,17 @@
-.. include:: ../../README.rst
+.. include:: welcome.rst
+|
+|
+Table of Contents
+-----------------
 
 .. toctree::
    :maxdepth: 1
 
    install
    beginner-tutorial
-   bundles
+   naming-convention
-   development-guidelines
+..   bundles
-   appendix
+..   development-guidelines
-   release-process
+..   appendix
-   releases
+..   release-process
+..   releases

docs/source/install.rst

@@ -4,16 +4,16 @@ Install
 Installing with ``pip``
 -----------------------
 
-Installing Zipline via ``pip`` is slightly more involved than the average
+Installing Catalyst via ``pip`` is slightly more involved than the average
 Python package.
 
 There are two reasons for the additional complexity:
 
-1. Zipline ships several C extensions that require access to the CPython C API.
+1. Catalyst ships several C extensions that require access to the CPython C API.
    In order to build the C extensions, ``pip`` needs access to the CPython
    header files for your Python installation.
 
-2. Zipline depends on `numpy <http://www.numpy.org/>`_, the core library for
+2. Catalyst depends on `numpy <http://www.numpy.org/>`_, the core library for
    numerical array computing in Python.  Numpy depends on having the `LAPACK
    <http://www.netlib.org/lapack>`_ linear algebra routines available.
 
@@ -28,13 +28,28 @@ your particular platform), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install zipline
+   $ pip install enigma-catalyst
 
-If you use Python for anything other than Zipline, we **strongly** recommend
+If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
 Python`_ provides an `excellent tutorial on virtualenv
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_.
+<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a summarized
+version:
+
+.. code-block:: bash
+
+   $ virtualenv catalyst-venv
+   $ source ./catalyst-venv/bin/activate
+   $ pip install enigma-
+
+Though not required by Catalyst directly, our example algorithms use matplotlib 
+to visually display the results of the trading algorithms. If you wish to run 
+any examples or use matplotlib during development, it can be installed using:
+
+.. code-block:: bash
+
+    $ pip install matplotlib
 
 GNU/Linux
 ~~~~~~~~~
@@ -60,15 +75,17 @@ On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
    $ pacman -S lapack gcc gcc-fortran pkg-config
 
-There are also AUR packages available for installing `Python 3.4
+.. Commenting it out until Catalyst fully supports Python 3.X
-<https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+..
-3.5, but Zipline only currently supports 3.4), and `ta-lib
+.. There are also AUR packages available for installing `Python 3.4
-<https://aur.archlinux.org/packages/ta-lib/>`_, an optional Zipline dependency.
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-Python 2 is also installable via:
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-.. code-block:: bash
+.. 
 
-   $ pacman -S python2
+..   $ pacman -S python2
 
 OSX
 ~~~
@@ -87,36 +104,238 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
+OSX + virtualenv + matplotlib
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A note about using matplotlib in virtual enviroments on OSX: it may be necessary to run
+
+.. code-block:: bash
+
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+
+in order to override the default ``macosx`` backend for your system, which may not 
+be accessible from inside the virtual environment. This will allow Catalyst to open 
+matplotlib charts from within a virtual environment, which is useful for displaying 
+the performance of your backtests.  To learn more about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+
 Windows
 ~~~~~~~
 
-For windows, the easiest and best supported way to install zipline is to use
+In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This package 
+contains the compiler and the set of system headers necessary for producing 
+binary wheels for Python 2.7 packages. If it's not already in your system, download
+it and install it before proceeding to the next step.
+
+For windows, the easiest and best supported way to install Catalyst is to use
 :ref:`Conda <conda>`.
 
+Amazon Linux AMI
+~~~~~~~~~~~~~~~~
+
+The packages ``pip`` and ``setuptools`` that come shipped by default are very outdated. 
+Thus, you first need to run:
+
+.. code-block:: bash
+
+   pip install --upgrade pip setuptools
+
+The default installation is also missing the C and C++ compilers, which you install by:
+
+.. code-block:: bash
+
+   sudo yum install gcc gcc-c++
+
+Then you should follow the regular installation instructions outlined at the beginning 
+of this page.
+
+
+Troubleshooting ``pip`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Issue**: 
+   Package enigma-catalyst cannot be found 
+ 
+**Solution**: 
+   Make sure you have the most up-to-date version of pip installed, by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip
+
+   On Windows, the recommended command is:
+
+   .. code-block:: bash
+
+      python -m pip install --upgrade pip
+
+----
+
+**Issue**: 
+   Package enigma-catalyst cannot still be found, even after upgrading pip (see above), with an error similar to:
+
+   .. code-block:: bash
+
+      Downloading/unpacking enigma-catalyst
+      Could not find a version that satisfies the requirement enigma-catalyst (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 0.2.dev1, 0.1.dev8, 0.1.dev6)
+      Cleaning up...
+      No distributions matching the version for enigma-catalyst
+
+**Solution**:
+   In some systems (this error has been reported in Ubuntu), pip is configured to only find stable versions by default. Since Catalyst is in alpha version, pip cannot find a matching version that satisfies the installation requirements. The solution is to include the `--pre` flag to include pre-release and development versions:
+
+   .. code-block:: bash
+
+      pip install --pre enigma-catalyst
+
+----
+
+**Issue**: 
+   Package enigma-catalyst fails to install because of outdated setuptools
+
+**Solution**: 
+   Upgrade to the most up-to-date setuptools package by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip setuptools
+
+----
+
+**Issue**:
+   Missing required packages  
+
+**Solution**:
+   Download `requirements.txt 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/requirements.txt>`_ 
+   (click on the *Raw* button and Right click -> Save As...) and use it to
+   install all the required dependencies by running:
+
+   .. code-block:: bash
+
+      pip install -r requirements.txt
+
+----
+
+**Issue**: 
+   Installation fails with error: ``fatal error: Python.h: No such file or directory``
+
+**Solution**: 
+   Some systems (this issue has been reported in Ubuntu) require `python-dev` for the proper build and installation of package dependencies. The solution is to install python-dev, which is independent of the virtual environment. In Ubuntu, you would need to run:
+
+   .. code-block:: bash
+
+      sudo apt-get install python-dev
+
+
 .. _conda:
 
 Installing with ``conda``
 -------------------------
 
-Another way to install Zipline is via the ``conda`` package manager, which
+Another way to install Catalyst is via the ``conda`` package manager, which
 comes as part of Continuum Analytics' `Anaconda
 <http://continuum.io/downloads>`_ distribution.
 
 The primary advantage of using Conda over ``pip`` is that conda natively
 understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Zipline and its dependencies
+``scipy``.  This means that ``conda`` can install Catalyst and its dependencies
-without requiring the use of a second tool to acquire Zipline's non-Python
+without requiring the use of a second tool to acquire Catalyst's non-Python
 dependencies.
 
 For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, you 
+can install MiniConda, which is a smaller footprint (fewer packages and smaller 
+size) than its big brother Anaconda, but it still contains all the main packages 
+needed. To install MiniConda, you can follow these steps:
 
-Once conda has been set up you can install Zipline from our ``Quantopian``
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 for 
-channel:
+   your Operating System.
+2. Install MiniConda. See the `Installation Instructions <https://conda.io/docs/user-guide/install/index.html>`_
+   if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal window,
+   which should print the list of packages installed with Conda.
 
-.. code-block:: bash
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you saved
+   the above ``python2.7-environment.yml`` file.
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new session
+   to run Catalyst):
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 above failed 
+for any reason, you can try setting up the environment manually with the following steps:
+
+1. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy
+
+2. Activate the environment:
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+3. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+Getting Help
+------------
+
+If after following the instructions above, and going through the *Troubleshooting* sections, 
+you still experience problems installing Catalyst, you can seek additional help through the 
+following channels:
+
+- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over the #catalyst_dev 
+  channel where many other users (as well as the project developers) hang out, and can assist 
+  you with your particular issue. The more descriptive and the more information you can provide, 
+  the easiest will be for others to help you out.
+
+- Report the problem you are experiencing on our 
+  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ following the guidelines 
+  provided therein. Before you do so, take a moment to browse through all `previous reported issues
+  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ in the likely case
+  that someone else experienced that same issue before, and you get a hint on how to solve it.
 
-    conda install -c Quantopian zipline
 
 .. _`Debian-derived`: https://www.debian.org/misc/children-distros
 .. _`RHEL-derived`: https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux_derivatives

docs/source/naming-convention.rst

@@ -0,0 +1,66 @@
+Naming Convention
+=================
+
+Catalyst introduces a standardized naming convention for all asset pairs 
+trading on any exchange in the following form:
+
+
+    **{market_currency}_{base_currency}**
+
+Where {market_currency} is the asset to be traded using {base_currency} as 
+the reference, both written in lowercase and separated with an underscore.
+
+This standardization is needed to overcome the lack of consistency in the 
+naming of assets across different exchanges, and making it easier to the user
+to refer to the asset pairs that you want to trade.
+
+Catalyst maintains a `Market Coverage Overview <https://www.enigma.co/catalyst/status>`_ 
+where you can check the mapping between Catalyst naming pairs and that of each 
+exchange. Catalyst will always expect in all its functions that you will refer to 
+the asset pairs by using the Catalyst naming convention.
+
+If at any point, you input the wrong name for an asset pair, you will get an error 
+of that pair not found in the given exchange, and a list of pairs available on that exchange:
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usd
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	Error traceback: /Volumes/Data/Users/victoris/Desktop/Enigma/user-install/catalyst-dev/catalyst/exchange/exchange.py (line 175)
+	SymbolNotFoundOnExchange:  Symbol btc_usd not found on exchange Poloniex. 
+	Choose from: ['rep_usdt', 'gno_btc', 'xvc_btc', 'pink_btc', 'sys_btc', 
+	'emc2_btc', 'rads_btc', 'note_btc', 'maid_btc', 'bch_btc', 'gnt_btc', 
+	'bcn_btc', 'rep_btc', 'bcy_btc', 'cvc_btc', 'nxt_xmr', 'zec_usdt', 
+	'fct_btc', 'gas_btc', 'pot_btc', 'eth_usdt', 'btc_usdt', 'lbc_btc', 
+	'dcr_btc', 'etc_usdt', 'omg_eth', 'amp_btc', 'xpm_btc', 'nxt_btc', 
+	'vtc_btc', 'steem_eth', 'blk_xmr', 'pasc_btc', 'zec_xmr', 'grc_btc', 
+	'nxc_btc', 'btcd_btc', 'ltc_btc', 'dash_btc', 'naut_btc', 'zec_eth', 
+	'zec_btc', 'burst_btc', 'zrx_eth', 'bela_btc', 'steem_btc', 'etc_btc', 
+	'eth_btc', 'huc_btc', 'strat_btc', 'lsk_btc', 'exp_btc', 'clam_btc', 
+	'rep_eth', 'dash_xmr', 'cvc_eth', 'bch_usdt', 'zrx_btc', 'dash_usdt', 
+	'blk_btc', 'xrp_btc', 'nxt_usdt', 'neos_btc', 'omg_btc', 'bts_btc', 
+	'doge_btc', 'gnt_eth', 'sbd_btc', 'gno_eth', 'xcp_btc', 'ltc_usdt', 
+	'btm_btc', 'xmr_usdt', 'lsk_eth', 'omni_btc', 'nav_btc', 'fldc_btc', 
+	'ppc_btc', 'xbc_btc', 'dgb_btc', 'sc_btc', 'btcd_xmr', 'vrc_btc', 
+	'ric_btc', 'str_btc', 'maid_xmr', 'xmr_btc', 'sjcx_btc', 'via_btc', 
+	'xem_btc', 'nmc_btc', 'etc_eth', 'ltc_xmr', 'ardr_btc', 'gas_eth', 
+	'flo_btc', 'xrp_usdt', 'game_btc', 'bch_eth', 'bcn_xmr', 'str_usdt']
+
+In the example above, exchange Poloniex does not use USD, but uses instead the 
+USDT cryptocurrency asset that is issued on the Bitcoin blockchain via the Omni
+Layer Protocol. Each USDT unit is backed by a U.S Dollar held in the reserves of 
+Tether Limited. USDT can be transferred, stored, and spent, just like bitcoins 
+or any other cryptocurrency. Given its 1:1 mapping to the USD, is a viable alternative.
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usdt
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	    [====================================]  Fetching poloniex daily candles: :  100%
+

docs/source/welcome.rst

@@ -0,0 +1,28 @@
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+|
+Catalyst is a data-driven crypto investment platform. It supports both 
+backtesting and live-trading in a number of different crypto-exchanges. 
+Catalyst empowers users to share and curate data and build profitable, 
+data-driven investment strategies.
+
+Features
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.

etc/requirements.txt

@@ -1,7 +1,7 @@
 # Incompatible with earlier PIP versions
 pip>=7.1.0
 # bcolz fails to install if this is not in the build_requires.
-setuptools>18.0
+setuptools>36.0
 
 # Logging
 Logbook==0.12.5

etc/requirements_docs.txt

@@ -1,4 +1,3 @@
 Sphinx>=1.3.2
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
-enigma-catalyst		# readthedocs.org

setup.py

@@ -304,7 +304,7 @@ setup(
                   if '__pycache__' not in root},
     license='Apache 2.0',
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
+        'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Programming Language :: Python',

tests/exchange/test_bundle.py

@@ -3,7 +3,8 @@ from logging import Logger
 import pandas as pd
 
 from catalyst import get_calendar
-from catalyst.exchange.bundle_utils import get_bcolz_chunk
+from catalyst.exchange.bundle_utils import get_bcolz_chunk, get_periods, \
+    get_periods_range
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
@@ -16,6 +17,25 @@ log = Logger('test_exchange_bundle')
 
 
 class ExchangeBundleTestCase:
+    def test_spot_value(self):
+        data_frequency = 'daily'
+        exchange_name = 'poloniex'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [
+            exchange.get_asset('btc_usdt')
+        ]
+        dt = pd.to_datetime('2017-10-14', utc=True)
+
+        values = exchange_bundle.get_spot_values(
+            assets=assets,
+            field='close',
+            dt=dt,
+            data_frequency=data_frequency
+        )
+        pass
+
     def test_ingest_minute(self):
         data_frequency = 'minute'
         exchange_name = 'bitfinex'
@@ -78,12 +98,13 @@ class ExchangeBundleTestCase:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
         data_frequency = 'daily'
-        include_symbols = 'etc_btc'
+        include_symbols = 'btc_usdt'
 
-        start = pd.to_datetime('2016-11-01', utc=True)
+        start = pd.to_datetime('2016-1-1', utc=True)
         end = pd.to_datetime('2017-10-16', utc=True)
+        periods = get_periods_range(start, end, data_frequency)
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)