Fixed an issue with historical data in live mode

catalyst/__main__.py

@@ -498,7 +498,7 @@ def ingest_exchange(exchange_name, data_frequency, start, end,
     exchange = get_exchange(exchange_name)
     exchange_bundle = ExchangeBundle(exchange)
 
-    click.echo('ingesting exchange bundle {}'.format(exchange_name))
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
     exchange_bundle.ingest(
         data_frequency=data_frequency,
         include_symbols=include_symbols,

catalyst/data/loader.py

@@ -95,7 +95,8 @@ def has_data_for_dates(series_or_df, first_date, last_date):
 
 def load_crypto_market_data(trading_day=None, trading_days=None,
                             bm_symbol=None, bundle=None, bundle_data=None,
-                            environ=None, exchange=None):
+                            environ=None, exchange=None, start_dt=None,
+                            end_dt=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
 
@@ -104,8 +105,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     # if trading_days is None:
     #    trading_days = get_calendar('OPEN').schedule
 
-    first_date = get_calendar('OPEN').first_trading_session
-    now = pd.Timestamp.utcnow()
+    # if start_dt is None:
+    start_dt = get_calendar('OPEN').first_trading_session
+
+    if end_dt is None:
+        end_dt = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
     # **two** full trading days prior to the most recently completed trading
@@ -131,7 +135,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     else:
         last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     '''
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 1]
+    last_date = trading_days[trading_days.get_loc(end_dt, method='ffill') - 1]
 
     if exchange is None:
         # This is exceptional, since placing the import at the module scope
@@ -146,14 +150,14 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
     br = exchange.get_history_window(
         assets=[benchmark_asset],
         end_dt=last_date,
-        bar_count=pd.Timedelta(last_date - first_date).days,
+        bar_count=pd.Timedelta(last_date - start_dt).days,
         frequency='1d',
         field='close',
         data_frequency='daily')
     br.columns = ['close']
     br = br.pct_change(1).iloc[1:]
-    br.loc[first_date]=0
-    br=br.sort_index()
+    br.loc[start_dt] = 0
+    br = br.sort_index()
 
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
@@ -162,10 +166,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         bm_symbol,
         first_date_treasury,
         last_date,
-        now,
+        end_dt,
         environ,
     )
-    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
+    benchmark_returns = br[br.index.slice_indexer(start_dt, last_date)]
     treasury_curves = tc[
         tc.index.slice_indexer(first_date_treasury, last_date)]
     return benchmark_returns, treasury_curves

catalyst/data/minute_bars.py

@@ -44,7 +44,6 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
-
 logger = logbook.Logger('MinuteBars')
 
 US_EQUITIES_MINUTES_PER_DAY = 390
@@ -1125,7 +1124,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             else:
                 return np.nan
 
-        #if field != 'volume':
+        # if field != 'volume':
         value *= self._ohlc_ratio_inverse_for_sid(sid)
         return value
 
@@ -1206,7 +1205,7 @@ class BcolzMinuteBarReader(MinuteBarReader):
             minute_dt.value / NANOS_IN_MINUTE,
             self._minutes_per_day,
             False,
-            )
+        )
 
     def load_raw_arrays(self, fields, start_dt, end_dt, sids):
         """
@@ -1262,10 +1261,10 @@ class BcolzMinuteBarReader(MinuteBarReader):
                 where = values != 0
                 # first slice down to len(where) because we might not have
                 # written data for all the minutes requested
-                #if field != 'volume':
+                # if field != 'volume':
                 out[:len(where), i][where] = (
                     values[where] * self._ohlc_ratio_inverse_for_sid(sid))
-                #else:
+                # else:
                 #    out[:len(where), i][where] = values[where]
 
             results.append(out)
@@ -1353,9 +1352,10 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
     path : str
         The path of the HDF5 file from which to source data.
     """
+
     def __init__(self, path):
         self._panel = pd.read_hdf(path)
 
     def read(self, dts, sids):
         panel = self._panel[sids, dts, :]
-        return panel.iteritems()
+        return panel.iteritems()

catalyst/examples/buy_btc_simple.py

@@ -0,0 +1,8 @@
+from catalyst.api import order, record, symbol
+
+def initialize(context):
+   context.asset = symbol('btc_usd')
+
+def handle_data(context, data):
+   order(asset, 1)
+   record(btc=data.current(context.asset, 'price'))

catalyst/examples/buy_low_sell_high_live.py

@@ -1,6 +1,7 @@
 import talib
 from logbook import Logger
 
+import pandas as pd
 from catalyst.api import (
     order,
     order_target_percent,
@@ -17,10 +18,10 @@ log = Logger('buy low sell high')
 
 def initialize(context):
     log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_BTC'
+    context.ASSET_NAME = 'btc_usdt'
     context.asset = symbol(context.ASSET_NAME)
 
-    context.TARGET_POSITIONS = 300
+    context.TARGET_POSITIONS = 30
     context.PROFIT_TARGET = 0.1
     context.SLIPPAGE_ALLOWED = 0.02
 
@@ -33,31 +34,31 @@ def initialize(context):
 
 
 def _handle_data(context, data):
+    price = data.current(context.asset, 'price')
+    log.info('got price {price}'.format(price=price))
+
     prices = data.history(
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='15m'
+        frequency='1d'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
     log.info('got rsi: {}'.format(rsi))
 
     # Buying more when RSI is low, this should lower our cost basis
     if rsi <= 30:
-        buy_increment = 50
+        buy_increment = 1
     elif rsi <= 40:
-        buy_increment = 20
-    # elif rsi <= 70:
-    #    buy_increment = 5
+        buy_increment = 0.5
+    elif rsi <= 70:
+        buy_increment = 0.2
     else:
-        buy_increment = None
+        buy_increment = 0.1
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
 
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
     record(
         price=price,
         rsi=rsi,
@@ -146,11 +147,22 @@ def analyze(context, stats):
 
 
 run_algorithm(
+    capital_base=100000,
     initialize=initialize,
     handle_data=handle_data,
     analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc'
+    exchange_name='poloniex',
+    start=pd.to_datetime('2017-5-01', utc=True),
+    end=pd.to_datetime('2017-10-16', utc=True),
+    base_currency='usdt',
+    data_frequency='daily'
 )
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='poloniex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc'
+# )

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -163,8 +163,6 @@ def analyze(context, stats):
 # Backtest
 run_algorithm(
     capital_base=250,
-    start=pd.to_datetime('2017-10-01', utc=True),
-    end=pd.to_datetime('2017-10-15', utc=True),
     data_frequency='minute',
     initialize=initialize,
     handle_data=handle_data,

catalyst/exchange/bitfinex/bitfinex.py

@@ -56,7 +56,7 @@ class Bitfinex(Exchange):
 
         # Max is 90 but playing it safe
         # https://www.bitfinex.com/posts/188
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 80
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)
@@ -665,10 +665,11 @@ class Bitfinex(Exchange):
         return time.strftime('%Y-%m-%d',
                              time.gmtime(int(response.json()[-1][0] / 1000)))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         exchange_symbol = asset.exchange_symbol
         try:
             self.ask_request()
+            # TODO: implement limit
             response = self._request(
                 'book/{}'.format(exchange_symbol), None)
             data = response.json()

catalyst/exchange/bittrex/bittrex.py

@@ -358,7 +358,7 @@ class Bittrex(Exchange):
             json.dump(symbol_map, f, sort_keys=True, indent=2,
                       separators=(',', ':'))
 
-    def get_orderbook(self, asset, order_type='all'):
+    def get_orderbook(self, asset, order_type='all', limit=100):
         if order_type == 'all':
             order_type = 'both'
         elif order_type == 'bid':
@@ -369,7 +369,11 @@ class Bittrex(Exchange):
             raise ValueError('invalid type')
 
         exchange_symbol = asset.exchange_symbol
-        data = self.api.getorderbook(market=exchange_symbol, type=order_type)
+        data = self.api.getorderbook(
+            market=exchange_symbol,
+            type=order_type,
+            depth=100
+        )
 
         result = dict()
         for exchange_type in data:

catalyst/exchange/bundle_utils.py

@@ -1,18 +1,15 @@
 import calendar
-import tarfile
-
-import requests
-from datetime import timedelta, datetime, date
 import os
-import pandas as pd
-import numpy as np
+import tarfile
+from datetime import timedelta, datetime, date
 
+import numpy as np
+import pandas as pd
 import pytz
 
 from catalyst.data.bundles import from_bundle_ingest_dirname
 from catalyst.data.bundles.core import download_without_progress
-from catalyst.exchange.exchange_errors import ApiCandlesError, \
-    PricingDataBeforeTradingError, NoDataAvailableOnExchange
+from catalyst.exchange.exchange_errors import NoDataAvailableOnExchange
 from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_path
@@ -189,60 +186,6 @@ def get_df_from_arrays(arrays, periods):
     return df
 
 
-def get_df_from_candles(candles, bar_count, end_dt, data_frequency,
-                        previous_candle=None):
-    """
-    Create candles for each period of the specified range, forward-filling
-    missing candles with the previous value.
-
-    :param candles:
-    :param bar_count:
-    :param end_dt:
-    :param data_frequency:
-    :param previous_candle:
-
-    :return:
-    """
-    all_dates = []
-    all_candles = []
-
-    start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-    date = start_dt
-
-    # TODO: this works well with a small number of candles, consider using numpy as needed
-    while date <= end_dt:
-        candle = next((
-            candle for candle in candles if candle['last_traded'] == date
-        ), previous_candle)
-
-        if candle is None:
-            candle = candles[0]
-
-        all_dates.append(date)
-        all_candles.append(candle)
-
-        previous_candle = candle
-
-        date += get_delta(1, data_frequency)
-
-    return all_dates, all_candles
-
-
-def get_trailing_candles_dt(asset, start_dt, end_dt, data_frequency):
-    missing_start = None
-
-    if asset.end_minute is not None and start_dt < asset.end_minute:
-        if asset.end_minute < end_dt:
-            delta = get_delta(1, data_frequency)
-
-            missing_start = asset.end_minute + delta
-
-    else:
-        missing_start = start_dt
-
-    return missing_start
-
-
 def range_in_bundle(asset, start_dt, end_dt, reader):
     """
     Evaluate whether price data of an asset is included has been ingested in
@@ -278,6 +221,7 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
     return has_data
 
 
+@deprecated
 def find_most_recent_time(bundle_name):
     """
     Find most recent "time folder" for a given bundle.
@@ -308,83 +252,3 @@ def find_most_recent_time(bundle_name):
     else:
         return None
 
-
-@deprecated
-def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
-    """
-    History API provides OHLCV data for any of the supported exchanges up to yesterday.
-
-    :param exchange_name: string
-        Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex).
-    :param data_frequency: string
-        Required: The bar frequency (minute or daily)
-    :param symbol: string
-        Required: The trading pair symbol, using Catalyst naming convention
-    :param start: datetime
-        Optional: The start date.
-    :param end: datetime
-        Optional: The end date.
-
-    :return ohlcv: list[dict[string, float]]
-        Each row contains the following dictionary for the resulting bars:
-        'ts'     : int, the timestamp in seconds
-        'open'   : float
-        'high'   : float
-        'low'    : float
-        'close'  : float
-        'volume' : float
-
-    Notes
-    =====
-    Using seconds for the start and end dates for ease of use in the
-    function query parameters.
-
-    Sometimes, one minute goes by without completing a trade of the given
-    trading pair on the given exchange. To minimize the payload size, we
-    don't return identical sequential bars. Post-processing code will
-    forward fill missing bars outside of this function.
-    """
-
-    start_seconds = get_seconds_from_date(start) if start else None
-    end_seconds = get_seconds_from_date(end) if end else None
-
-    if exchange_name not in EXCHANGE_NAMES:
-        raise ValueError(
-            'get_history function only supports the following exchanges: {}'.format(
-                list(EXCHANGE_NAMES)))
-
-    if data_frequency != 'daily' and data_frequency != 'minute':
-        raise ValueError(
-            'get_history currently only supports daily and minute data.'
-        )
-
-    url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format(
-        api_url=API_URL,
-        exchange=exchange_name,
-        symbol=symbol,
-        data_frequency=data_frequency,
-    )
-
-    if start_seconds:
-        url += '&start={}'.format(start_seconds)
-
-    if end_seconds:
-        url += '&end={}'.format(end_seconds)
-
-    try:
-        response = requests.get(url)
-    except Exception as e:
-        raise ValueError(e)
-
-    data = response.json()
-
-    if 'error' in data:
-        raise ApiCandlesError(error=data['error'])
-
-    for candle in data:
-        last_traded = pd.Timestamp.utcfromtimestamp(candle['ts'])
-        last_traded = last_traded.replace(tzinfo=pytz.UTC)
-
-        candle['last_traded'] = last_traded
-
-    return data

catalyst/exchange/data_portal_exchange.py

@@ -12,15 +12,15 @@
 # limitations under the License.
 
 import abc
-from datetime import timedelta
 from time import sleep
 
+import numpy as np
 import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
 from catalyst.data.data_portal import DataPortal
-from catalyst.errors import HistoryWindowStartsBeforeData
+from catalyst.exchange.bundle_utils import get_start_dt
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
@@ -153,6 +153,10 @@ class DataPortalExchangeBase(DataPortal):
                 exchange = self.exchanges[assets.exchange]
                 spot_values = self.get_exchange_spot_value(
                     exchange, [assets], field, dt, data_frequency)
+
+                if not spot_values:
+                    return np.nan
+
                 return spot_values[0]
 
             else:
@@ -282,109 +286,60 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
                                     field,
                                     data_frequency,
                                     ffill=True):
+        """
+        Fetching price history window from the exchange bundle.
+
+        Using a try... except approach to minimize reads most of the time,
+        when the data exists.
+
+        :param exchange:
+        :param assets:
+        :param end_dt:
+        :param bar_count:
+        :param frequency:
+        :param field:
+        :param data_frequency:
+        :param ffill:
+        :return:
+        """
+
         bundle = self.exchange_bundles[exchange.name]
-
-        if data_frequency == 'minute':
-            dts = self.trading_calendar.minutes_window(
-                end_dt, -bar_count
-            )
-
-            self.ensure_after_first_day(dts[0], assets)
-
-        elif data_frequency == 'daily':
-            session = self.trading_calendar.minute_to_session_label(end_dt)
-            dts = self._get_days_for_window(session, bar_count)
-
-            if len(dts) == 0:
-                symbols = [asset.symbol for asset in assets]
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    symbols=symbols,
-                    exchange=exchange.name,
-                    first_trading_day= \
-                        min([asset.start_date for asset in assets]),
-                    data_frequency=data_frequency,
-                    symbol_list=','.join(symbols)
-                )
-
-            self.ensure_after_first_day(dts[0], assets)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency=data_frequency)
-
-        reader = bundle.get_reader(data_frequency)
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=exchange.name.title(),
-                data_frequency=data_frequency
-            )
-
-        try:
-            values = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=dts[0],
-                end_dt=dts[-1]
-            )[0]
-
-        except Exception:
-            first_trading_day = self._get_first_trading_day(assets)
-            symbols = [asset.symbol.encode('utf-8') for asset in assets]
-
-            symbol_list = ','.join(symbols)
-            raise PricingDataNotLoadedError(
-                field=field,
-                first_trading_day=first_trading_day,
-                exchange=exchange.name.title(),
-                symbols=symbols,
-                symbol_list=symbol_list,
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        for index, asset in enumerate(assets):
-            asset_values = values[:, index]
-
-            value_series = pd.Series(asset_values, index=dts)
-            series[asset] = value_series
-
+        series = bundle.get_history_window_series_and_load(
+            assets=assets,
+            end_dt=end_dt,
+            bar_count=bar_count,
+            field=field,
+            data_frequency=data_frequency
+        )
         return pd.DataFrame(series)
 
-    def ensure_after_first_day(self, dt, assets):
-        first_trading_day = self._get_first_trading_day(assets)
-        if dt < first_trading_day:
-            raise PricingDataBeforeTradingError(
-                first_trading_day=first_trading_day,
-                exchange=assets[0].exchange.title(),
-                symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                dt=dt,
-            )
-
     def get_exchange_spot_value(self, exchange, assets, field, dt,
                                 data_frequency):
         bundle = self.exchange_bundles[exchange.name]
-        reader = bundle.get_reader(data_frequency)
 
-        self.ensure_after_first_day(dt, assets)
+        if data_frequency == 'daily':
+            dt = dt.floor('1D')
+        else:
+            dt = dt.floor('1 min')
 
-        values = []
-        for asset in assets:
-            try:
-                value = reader.get_value(
-                    sid=asset.sid,
-                    dt=dt,
-                    field=field
+        try:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)
+
+        except PricingDataNotLoadedError:
+            log.info(
+                'pricing data for {symbol} not found on {dt}'
+                ', updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    dt=dt
                 )
-                values.append(value)
-            except Exception:
-                raise PricingDataNotLoadedError(
-                    field=field,
-                    first_trading_day=self._get_first_trading_day(assets),
-                    exchange=exchange.name.title(),
-                    symbols=[asset.symbol.encode('utf-8') for asset in assets],
-                    symbol_list=''.join(
-                        [asset.symbol.encode('utf-8') for asset in assets]),
-                    data_frequency=data_frequency
-                )
-
-            return values
+            )
+            bundle.ingest_assets(
+                assets=assets,
+                start_dt=self._first_trading_day,
+                end_dt=self._last_available_session,
+                data_frequency=data_frequency,
+                show_progress=True
+            )
+            return bundle.get_spot_values(
+                assets, field, dt, data_frequency, True
+            )

catalyst/exchange/exchange.py

@@ -16,7 +16,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
     InvalidHistoryFrequencyError, MismatchingFrequencyError, \
-    BundleNotFoundError, NoDataAvailableOnExchange
+    BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -370,44 +370,6 @@ class Exchange:
 
         return value
 
-    def get_series_from_bundle(self, assets, start_dt, end_dt, data_frequency,
-                               field):
-        """
-
-        :return:
-        """
-        reader = self.bundle.get_reader(data_frequency)
-
-        if reader is None:
-            raise BundleNotFoundError(
-                exchange=self.name.title(),
-                data_frequency=data_frequency
-            )
-
-        series = dict()
-        try:
-            arrays = reader.load_raw_arrays(
-                sids=[asset.sid for asset in assets],
-                fields=[field],
-                start_dt=start_dt,
-                end_dt=end_dt
-            )
-
-            periods = self.bundle.get_calendar_periods_range(
-                start_dt, end_dt, data_frequency
-            )
-
-            for asset_index, asset in enumerate(assets):
-                asset_values = arrays[asset_index]
-
-                value_series = pd.Series(asset_values[0], index=periods)
-                series[asset] = value_series
-
-        except Exception as e:
-            log.debug('unable to retrieve from bundle: {}'.format(e))
-
-        return series
-
     def get_series_from_candles(self, candles, start_dt, end_dt,
                                 field, previous_value=None):
         """
@@ -487,11 +449,6 @@ class Exchange:
                 data_frequency = 'daily'
 
         elif unit.lower() == 'm':
-            # if data_frequency != 'minute':
-            # raise MismatchingFrequencyError(
-            #     frequency=frequency,
-            #     data_frequency=data_frequency
-            # )
             if data_frequency == 'daily':
                 data_frequency = 'minute'
 
@@ -499,42 +456,15 @@ class Exchange:
             raise InvalidHistoryFrequencyError(frequency)
 
         adj_bar_count = candle_size * bar_count
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
-
         try:
-            adj_start_dt, adj_end_dt = get_adj_dates(
-                start_dt, end_dt, assets, data_frequency
-            )
-            in_bundle = True
-
-        except NoDataAvailableOnExchange:
-            in_bundle = False
-
-        if in_bundle:
-            missing_assets = self.bundle.filter_existing_assets(
+            series = self.bundle.get_history_window_series_and_load(
                 assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
+                end_dt=end_dt,
+                bar_count=adj_bar_count,
+                field=field,
                 data_frequency=data_frequency
             )
-
-            if missing_assets:
-                self.bundle.ingest_assets(
-                    assets=assets,
-                    start_dt=adj_start_dt,
-                    end_dt=adj_end_dt,
-                    data_frequency=data_frequency
-                )
-
-            series = self.get_series_from_bundle(
-                assets=assets,
-                start_dt=adj_start_dt,
-                end_dt=adj_end_dt,
-                data_frequency=data_frequency,
-                field=field
-            )
-
-        else:
+        except PricingDataNotLoadedError:
             series = dict()
 
         for asset in assets:
@@ -542,7 +472,7 @@ class Exchange:
                 # Adding bars too recent to be contained in the consolidated
                 # exchanges bundles. We go directly against the exchange
                 # to retrieve the candles.
-
+                start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
                         if asset in series else start_dt

catalyst/exchange/exchange_bcolz.py

@@ -48,9 +48,9 @@ class BcolzExchangeBarReader(BcolzMinuteBarReader):
         # else:
         #     return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
 
-        return self._load_daily_raw_arrays(fields, start_dt, end_dt, sids)
+        return self._load_raw_arrays(fields, start_dt, end_dt, sids)
 
-    def _load_daily_raw_arrays(self, fields, start_dt, end_dt, sids):
+    def _load_raw_arrays(self, fields, start_dt, end_dt, sids):
         start_idx = self._find_position_of_minute(start_dt)
         end_idx = self._find_position_of_minute(end_dt)
 

catalyst/exchange/exchange_bundle.py

@@ -10,12 +10,13 @@ from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_delta, get_adj_dates, get_month_start_end, \
-    get_year_start_end, get_periods_range, get_df_from_arrays
+    get_year_start_end, get_periods_range, get_df_from_arrays, get_start_dt
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange
+    TempBundleNotFoundError, NoDataAvailableOnExchange, \
+    PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
@@ -451,3 +452,152 @@ class ExchangeBundle:
         for frequency in data_frequency.split(','):
             self.ingest_assets(assets, start_dt, end_dt, frequency,
                                show_progress)
+
+    def get_history_window_series_and_load(self,
+                                           assets,
+                                           end_dt,
+                                           bar_count,
+                                           field,
+                                           data_frequency):
+        try:
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency
+            )
+            return pd.DataFrame(series)
+
+        except PricingDataNotLoadedError:
+            start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+            log.info(
+                'pricing data for {symbol} not found in range '
+                '{start} to {end}, updating the bundles.'.format(
+                    symbol=[asset.symbol for asset in assets],
+                    start=start_dt,
+                    end=end_dt
+                )
+            )
+            self.ingest_assets(
+                assets=assets,
+                start_dt=start_dt,
+                end_dt=end_dt,
+                data_frequency=data_frequency,
+                show_progress=True
+            )
+            series = self.get_history_window_series(
+                assets=assets,
+                end_dt=end_dt,
+                bar_count=bar_count,
+                field=field,
+                data_frequency=data_frequency,
+                reset_reader=True
+            )
+            return series
+
+    def get_spot_values(self, assets, field, dt, data_frequency,
+                        reset_reader=False):
+        values = []
+        try:
+            reader = self.get_reader(data_frequency)
+            if reset_reader:
+                del self._readers[reader._rootdir]
+                reader = self.get_reader(data_frequency)
+
+            for asset in assets:
+                value = reader.get_value(
+                    sid=asset.sid,
+                    dt=dt,
+                    field=field
+                )
+                values.append(value)
+
+            return values
+
+        except Exception:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+    def get_history_window_series(self,
+                                  assets,
+                                  end_dt,
+                                  bar_count,
+                                  field,
+                                  data_frequency,
+                                  reset_reader=False):
+        start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+        start_dt, end_dt = \
+            get_adj_dates(start_dt, end_dt, assets, data_frequency)
+
+        reader = self.get_reader(data_frequency)
+        if reset_reader:
+            del self._readers[reader._rootdir]
+            reader = self.get_reader(data_frequency)
+
+        if reader is None:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        for asset in assets:
+            asset_start_dt, asset_end_dt = \
+                get_adj_dates(start_dt, end_dt, assets, data_frequency)
+
+            in_bundle = range_in_bundle(
+                asset, asset_start_dt, asset_end_dt, reader
+            )
+            if not in_bundle:
+                raise PricingDataNotLoadedError(
+                    field=field,
+                    first_trading_day=asset.start_date,
+                    exchange=self.exchange.name,
+                    symbols=asset.symbol,
+                    symbol_list=asset.symbol,
+                    data_frequency=data_frequency
+                )
+
+        series = dict()
+        try:
+            arrays = reader.load_raw_arrays(
+                sids=[asset.sid for asset in assets],
+                fields=[field],
+                start_dt=start_dt,
+                end_dt=end_dt
+            )
+
+        except Exception:
+            symbols = [asset.symbol.encode('utf-8') for asset in assets]
+            raise PricingDataNotLoadedError(
+                field=field,
+                first_trading_day=min([asset.start_date for asset in assets]),
+                exchange=self.exchange.name,
+                symbols=symbols,
+                symbol_list=','.join(symbols),
+                data_frequency=data_frequency
+            )
+
+        periods = self.get_calendar_periods_range(
+            start_dt, end_dt, data_frequency
+        )
+
+        for asset_index, asset in enumerate(assets):
+            asset_values = arrays[asset_index]
+
+            value_series = pd.Series(asset_values.flatten(), index=periods)
+            series[asset] = value_series
+
+        return series

catalyst/exchange/poloniex/poloniex.py

@@ -49,7 +49,7 @@ class Poloniex(Exchange):
         self.transactions = defaultdict(list)
 
         self.num_candles_limit = 2000
-        self.max_requests_per_minute = 20
+        self.max_requests_per_minute = 60
         self.request_cpt = dict()
 
         self.bundle = ExchangeBundle(self)

catalyst/sources/benchmark_source.py

@@ -72,7 +72,13 @@ class BenchmarkSource(object):
                             "benchmark_returns.")
 
     def get_value(self, dt):
-        return self._precalculated_series.loc[dt]
+        try:
+            series = self._precalculated_series
+            value = series.loc[dt]
+            return value
+        except Exception:
+            # TODO: workaround, find permanent fix
+            return 0
 
     def get_range(self, start_dt, end_dt):
         return self._precalculated_series.loc[start_dt:end_dt]

catalyst/utils/calendars/exchange_calendar_open.py

@@ -31,4 +31,4 @@ class OpenExchangeCalendar(TradingCalendar):
         return DateOffset(days=1)
 
     def __init__(self, *args, **kwargs):
-        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-02-19', tz='UTC'), **kwargs)
+        super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)

catalyst/utils/run_algo.py

@@ -191,7 +191,12 @@ def _run(handle_data,
     open_calendar = get_calendar('OPEN')
 
     env = TradingEnvironment(
-        load=partial(load_crypto_market_data, environ=environ),
+        load=partial(
+            load_crypto_market_data,
+            environ=environ,
+            start_dt=start,
+            end_dt=end
+        ),
         environ=environ,
         exchange_tz='UTC',
         asset_db_path=None  # We don't need an asset db, we have exchanges
@@ -284,7 +289,8 @@ def _run(handle_data,
             exchanges=exchanges,
             asset_finder=None,
             trading_calendar=open_calendar,
-            first_trading_day=None,
+            first_trading_day=start,
+            last_available_session=end
         )
 
         sim_params = create_simulation_parameters(

docs/source/beginner-tutorial.rst

@@ -1,608 +1,281 @@
-Zipline Beginner Tutorial
--------------------------
+Catalyst Beginner Tutorial
+--------------------------
 
 Basics
 ~~~~~~
 
-Zipline is an open-source algorithmic trading simulator written in
-Python.
+Catalyst is an open-source algorithmic trading simulator for crypto
+assets written in Python.
 
-The source can be found at: https://github.com/quantopian/zipline
+The source can be found at: https://github.com/enigmampc/catalyst
 
 Some benefits include:
 
+-  Support for several of the top crypto-exchanges by trading volume.
 -  Realistic: slippage, transaction costs, order delays.
 -  Stream-based: Process each event individually, avoids look-ahead
    bias.
 -  Batteries included: Common transforms (moving average) as well as
    common risk calculations (Sharpe).
 -  Developed and continuously updated by
-   `Quantopian <https://www.quantopian.com>`__ which provides an
-   easy-to-use web-interface to Zipline, 10 years of minute-resolution
-   historical US stock data, and live-trading capabilities. This
-   tutorial is directed at users wishing to use Zipline without using
-   Quantopian. If you instead want to get started on Quantopian, see
-   `here <https://www.quantopian.com/faq#get-started>`__.
+   `Enigma MPC <https://www.enigma.co>`__ which is building the Enigma 
+   data marketplace protocol as well as Catalyst, the first application
+   that will run on our protocol. Powered by our financial data 
+   marketplace, Catalyst empowers users to share and curate data and 
+   build profitable, data-driven investment strategies.
 
-This tutorial assumes that you have zipline correctly installed, see the
-`installation
-instructions <https://github.com/quantopian/zipline#installation>`__ if
-you haven't set up zipline yet.
+This tutorial assumes that you have Catalyst correctly installed, see the
+:doc:`installation instructions <install>` if you haven't set up 
+Catalyst yet.
 
-Every ``zipline`` algorithm consists of two functions you have to
+Every ``catalyst`` algorithm consists of at least two functions you have to
 define:
 
 * ``initialize(context)``
 * ``handle_data(context, data)``
 
-Before the start of the algorithm, ``zipline`` calls the
+Before the start of the algorithm, ``catalyst`` calls the
 ``initialize()`` function and passes in a ``context`` variable.
 ``context`` is a persistent namespace for you to store variables you
 need to access from one algorithm iteration to the next.
 
-After the algorithm has been initialized, ``zipline`` calls the
+After the algorithm has been initialized, ``catalyst`` calls the
 ``handle_data()`` function once for each event. At every call, it passes
 the same ``context`` variable and an event-frame called ``data``
 containing the current trading bar with open, high, low, and close
-(OHLC) prices as well as volume for each stock in your universe. For
-more information on these functions, see the `relevant part of the
-Quantopian docs <https://www.quantopian.com/help#api-toplevel>`__.
+(OHLC) prices as well as volume for each crypto asset in your universe. 
+
+.. For more information on these functions, see the `relevant part of the
+.. Quantopian docs <https://www.quantopian.com/help#api-toplevel>`.
 
 My first algorithm
 ~~~~~~~~~~~~~~~~~~
 
 Lets take a look at a very simple algorithm from the ``examples``
-directory, ``buyapple.py``:
+directory, ``buy_btc.py``:
 
 .. code-block:: python
 
-   from zipline.examples import buyapple
-   buyapple??
-
-
-.. code-block:: python
-
-   from zipline.api import order, record, symbol
+   from catalyst.api import order, record, symbol
 
 
    def initialize(context):
-       pass
+       context.asset = symbol('btc_usd')
 
 
    def handle_data(context, data):
-       order(symbol('AAPL'), 10)
-       record(AAPL=data.current(symbol('AAPL'), 'price'))
+       order(context.asset, 1)
+       record(btc = data.current(context.asset, 'price'))
 
 
 As you can see, we first have to import some functions we would like to
 use. All functions commonly used in your algorithm can be found in
-``zipline.api``. Here we are using :func:`~zipline.api.order()` which takes two
-arguments: a security object, and a number specifying how many stocks you would
-like to order (if negative, :func:`~zipline.api.order()` will sell/short
-stocks). In this case we want to order 10 shares of Apple at each iteration. For
-more documentation on ``order()``, see the `Quantopian docs
-<https://www.quantopian.com/help#api-order>`__.
+``catalyst.api``. Here we are using :func:`~catalyst.api.order()` which takes two
+arguments: a cryptoasset object, and a number specifying how many assets you would
+like to order (if negative, :func:`~catalyst.api.order()` will sell/short
+assets). In this case we want to order 1 bitcoin at each iteration. 
 
-Finally, the :func:`~zipline.api.record` function allows you to save the value
+.. For more documentation on ``order()``, see the `Quantopian docs
+.. <https://www.quantopian.com/help#api-order>`__.
+
+Finally, the :func:`~catalyst.api.record` function allows you to save the value
 of a variable at each iteration. You provide it with a name for the variable
 together with the variable itself: ``varname=var``. After the algorithm
 finished running you will have access to each variable value you tracked
-with :func:`~zipline.api.record` under the name you provided (we will see this
-further below). You also see how we can access the current price data of the
-AAPL stock in the ``data`` event frame (for more information see
-`here <https://www.quantopian.com/help#api-event-properties>`__.
+with :func:`~catalyst.api.record` under the name you provided (we will see this
+further below). You also see how we can access the current price data of
+a bitcoin in the ``data`` event frame.
+
+.. (for more information see `here <https://www.quantopian.com/help#api-event-properties>`__.
 
 Running the algorithm
 ~~~~~~~~~~~~~~~~~~~~~
 
-To now test this algorithm on financial data, ``zipline`` provides three
-interfaces: A command-line interface, ``IPython Notebook`` magic, and
-:func:`~zipline.run_algorithm`.
+To can now test this algorithm on crypto data, ``catalyst`` provides three
+interfaces: 
 
-Ingesting Data
+-  A command-line interface,
+-  ``IPython Notebook`` magic, 
+-  and :func:`~catalyst.run_algorithm`.
+
+Ingesting data
 ^^^^^^^^^^^^^^
-If you haven't ingested the data, run:
 
-.. code-block:: bash
+In previous versions of Catalyst you needed to manually ingest data before running
+your algorithm to make it available at runtime. Starting with version 0.3, the
+algorithm will automagically ingest the data it needs the first time that encounters 
+a data request for data that it doesn't have.
 
-   $ zipline ingest [-b <bundle>]
+Still, we believe it is important for you to have a high-level understanding
+of how data is managed:
 
-where ``<bundle>`` is the name of the bundle to ingest, defaulting to
-:ref:`quantopian-quandl <quantopian-quandl-mirror>`.
+-  Pricing data is split and packaged into ``bundles``: chunks of data organized 
+   as time series that are kept up to date daily on Enigma's servers. Catalyst 
+   downloads the bundles that needs at any given time, and reconstructs the whole
+   dataset in your hard drive.
 
-you can check out the :ref:`ingesting data <ingesting-data>` section for
-more detail.
+-  Pricing data is provided in ``daily`` and ``minute`` resolution. Those are different
+   bundle datasets, and are managed separately.
+
+-  Bundles are exchange-specific, as the pricing data is specific to the trades that
+   happen in each exchange. You can optionally specify which exchange you want pricing
+   data from.
+
+-  Catalyst keeps track of all the downloaded bundles, so that it only has to download
+   them once, and will do incremental updates as needed.
+
+-  When running in ``live trading`` mode, Catalyst will first look for historical 
+   pricing data in the locally stored bundles. If there is anything missing, Catalyst will
+   hit the exchange for the most recent data, and merge it with the local bundle to make
+   it available for future iterations.
+
+If you want to learn more, check out the :ref:`ingesting data <ingesting-data>` section
+for more detail.
 
 Command line interface
 ^^^^^^^^^^^^^^^^^^^^^^
 
-After you installed zipline you should be able to execute the following
+After you installed Catalyst you should be able to execute the following
 from your command line (e.g. ``cmd.exe`` on Windows, or the Terminal app
-on OSX):
+on OSX). Displaying here a simplified output for eductional purposes:
 
 .. code-block:: bash
 
-   $ zipline run --help
+   $ catalyst --help
 
 .. parsed-literal::
 
-   Usage: zipline run [OPTIONS]
+     Usage: catalyst [OPTIONS] COMMAND [ARGS]...
 
-     Run a backtest for the given algorithm.
+       Top level catalyst entry point.
+
+     Options:
+       --version               Show the version and exit.
+       --help                  Show this message and exit.
+
+     Commands:
+       ingest-exchange  Ingest data for the given exchange.
+       live             Trade live with the given algorithm.
+       run              Run a backtest for the given algorithm.
+
+There are three main modes you can run on Catalyst. The first being ``ingest-exchange`` 
+for data ingestion, which we have summarized in the previous section. The second 
+is ``live`` to use your algorithm to trade live against a given exchange, and the 
+third mode ``run`` is to backtest your algorithm before trading live with it.
+
+Let's start with backtesting, so run this other command to learn more about 
+the available options:
+
+.. code-block:: bash
+
+   $ catalyst run --help
+
+.. parsed-literal::
+
+      Usage: catalyst run [OPTIONS]
+
+        Run a backtest for the given algorithm.
+
+      Options:
+        -f, --algofile FILENAME         The file that contains the algorithm to run.
+        -t, --algotext TEXT             The algorithm script to run.
+        -D, --define TEXT               Define a name to be bound in the namespace
+                                        before executing the algotext. For example
+                                        '-Dname=value'. The value may be any python
+                                        expression. These are evaluated in order so
+                                        they may refer to previously defined names.
+        --data-frequency [daily|minute]
+                                        The data frequency of the simulation.
+                                        [default: daily]
+        --capital-base FLOAT            The starting capital for the simulation.
+                                        [default: 10000000.0]
+        -b, --bundle BUNDLE-NAME        The data bundle to use for the simulation.
+                                        [default: poloniex]
+        --bundle-timestamp TIMESTAMP    The date to lookup data on or before.
+                                        [default: <current-time>]
+        -s, --start DATE                The start date of the simulation.
+        -e, --end DATE                  The end date of the simulation.
+        -o, --output FILENAME           The location to write the perf data. If this
+                                        is '-' the perf will be written to stdout.
+                                        [default: -]
+        --print-algo / --no-print-algo  Print the algorithm to stdout.
+        -x, --exchange-name [poloniex|bitfinex|bittrex]
+                                        The name of the targeted exchange
+                                        (supported: bitfinex, bittrex, poloniex).
+        -n, --algo-namespace TEXT       A label assigned to the algorithm for data
+                                        storage purposes.
+        -c, --base-currency TEXT        The base currency used to calculate
+                                        statistics (e.g. usd, btc, eth).
+        --help                          Show this message and exit.
 
-   Options:
-     -f, --algofile FILENAME         The file that contains the algorithm to run.
-     -t, --algotext TEXT             The algorithm script to run.
-     -D, --define TEXT               Define a name to be bound in the namespace
-                                     before executing the algotext. For example
-                                     '-Dname=value'. The value may be any python
-                                     expression. These are evaluated in order so
-                                     they may refer to previously defined names.
-     --data-frequency [minute|daily]
-                                     The data frequency of the simulation.
-                                     [default: daily]
-     --capital-base FLOAT            The starting capital for the simulation.
-                                     [default: 10000000.0]
-     -b, --bundle BUNDLE-NAME        The data bundle to use for the simulation.
-                                     [default: quantopian-quandl]
-     --bundle-timestamp TIMESTAMP    The date to lookup data on or before.
-                                     [default: <current-time>]
-     -s, --start DATE                The start date of the simulation.
-     -e, --end DATE                  The end date of the simulation.
-     -o, --output FILENAME           The location to write the perf data. If this
-                                     is '-' the perf will be written to stdout.
-                                     [default: -]
-     --print-algo / --no-print-algo  Print the algorithm to stdout.
-     --help                          Show this message and exit.
 
 As you can see there are a couple of flags that specify where to find your
-algorithm (``-f``) as well as parameters specifying which data to use,
-defaulting to the :ref:`quantopian-quandl-mirror`. There are also arguments for
-the date range to run the algorithm over (``--start`` and ``--end``). Finally,
-you'll want to save the performance metrics of your algorithm so that you can
-analyze how it performed. This is done via the ``--output`` flag and will cause
-it to write the performance ``DataFrame`` in the pickle Python file format.
-Note that you can also define a configuration file with these parameters that
-you can then conveniently pass to the ``-c`` option so that you don't have to
-supply the command line args all the time (see the .conf files in the examples
-directory).
+algorithm (``-f``) as well as a parameter to specify which exchange to use. 
+There are also arguments for the date range to run the algorithm over 
+(``--start`` and ``--end``). Finally, you'll want to save the performance 
+metrics of your algorithm so that you can analyze how it performed. This is 
+done via the ``--output`` flag and will cause it to write the performance 
+``DataFrame`` in the pickle Python file format. Note that you can also define 
+a configuration file with these parameters that you can then conveniently pass 
+to the ``-c`` option so that you don't have to supply the command line args 
+all the time (see the .conf files in the examples directory).
 
 Thus, to execute our algorithm from above and save the results to
-``buyapple_out.pickle`` we would call ``zipline run`` as follows:
+``buy_btc_simple_out.pickle`` we would call ``catalyst run`` as follows:
 
 .. code-block:: python
 
-    zipline run -f ../../zipline/examples/buyapple.py --start 2000-1-1 --end 2014-1-1 -o buyapple_out.pickle
+    catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2016-9-29 -o buy_simple_btc_out.pickle
 
 
-.. parsed-literal::
+.. 
+.. parsed-literal
 
-    AAPL
-    [2015-11-04 22:45:32.820166] INFO: Performance: Simulated 3521 trading days out of 3521.
-    [2015-11-04 22:45:32.820314] INFO: Performance: first open: 2000-01-03 14:31:00+00:00
-    [2015-11-04 22:45:32.820401] INFO: Performance: last close: 2013-12-31 21:00:00+00:00
+..    AAPL
+..    [2015-11-04 22:45:32.820166] INFO: Performance: Simulated 3521 trading days out of 3521.
+..    [2015-11-04 22:45:32.820314] INFO: Performance: first open: 2000-01-03 14:31:00+00:00
+..    [2015-11-04 22:45:32.820401] INFO: Performance: last close: 2013-12-31 21:00:00+00:00
 
 
 ``run`` first calls the ``initialize()`` function, and then
-streams the historical stock price day-by-day through ``handle_data()``.
-After each call to ``handle_data()`` we instruct ``zipline`` to order 10
-stocks of AAPL. After the call of the ``order()`` function, ``zipline``
+streams the historical asset price day-by-day through ``handle_data()``.
+After each call to ``handle_data()`` we instruct ``catalyst`` to order 1
+bitcoin. After the call of the ``order()`` function, ``catalyst``
 enters the ordered stock and amount in the order book. After the
-``handle_data()`` function has finished, ``zipline`` looks for any open
+``handle_data()`` function has finished, ``catalyst`` looks for any open
 orders and tries to fill them. If the trading volume is high enough for
-this stock, the order is executed after adding the commission and
+this asset, the order is executed after adding the commission and
 applying the slippage model which models the influence of your order on
 the stock price, so your algorithm will be charged more than just the
-stock price \* 10. (Note, that you can also change the commission and
-slippage model that ``zipline`` uses, see the `Quantopian
-docs <https://www.quantopian.com/help#ide-slippage>`__ for more
-information).
+asset price. (Note, that you can also change the commission and
+slippage model that ``catalyst`` uses).
 
-Lets take a quick look at the performance ``DataFrame``. For this, we
+.. see the `Quantopian docs <https://www.quantopian.com/help#ide-slippage>`__
+.. for more information).
+
+Let's take a quick look at the performance ``DataFrame``. For this, we
 use ``pandas`` from inside the IPython Notebook and print the first ten
-rows. Note that ``zipline`` makes heavy usage of ``pandas``, especially
-for data input and outputting so it's worth spending some time to learn
-it.
+rows. Note that ``catalyst`` makes heavy usage of 
+`pandas <http://pandas.pydata.org/>`_, especially for data input and 
+outputting so it's worth spending some time to learn it.
 
 .. code-block:: python
 
     import pandas as pd
-    perf = pd.read_pickle('buyapple_out.pickle') # read in perf DataFrame
+    perf = pd.read_pickle('buy_btc_simple_out.pickle') # read in perf DataFrame
     perf.head()
 
-.. raw:: html
-
-    <div style="max-height:1000px;max-width:1500px;overflow:auto;">
-    <table border="1" class="dataframe">
-      <thead>
-        <tr style="text-align: right;">
-          <th></th>
-          <th>AAPL</th>
-          <th>algo_volatility</th>
-          <th>algorithm_period_return</th>
-          <th>alpha</th>
-          <th>benchmark_period_return</th>
-          <th>benchmark_volatility</th>
-          <th>beta</th>
-          <th>capital_used</th>
-          <th>ending_cash</th>
-          <th>ending_exposure</th>
-          <th>...</th>
-          <th>short_exposure</th>
-          <th>short_value</th>
-          <th>shorts_count</th>
-          <th>sortino</th>
-          <th>starting_cash</th>
-          <th>starting_exposure</th>
-          <th>starting_value</th>
-          <th>trading_days</th>
-          <th>transactions</th>
-          <th>treasury_period_return</th>
-        </tr>
-      </thead>
-      <tbody>
-        <tr>
-          <th>2000-01-03 21:00:00</th>
-          <td>3.738314</td>
-          <td>0.000000e+00</td>
-          <td>0.000000e+00</td>
-          <td>-0.065800</td>
-          <td>-0.009549</td>
-          <td>0.000000</td>
-          <td>0.000000</td>
-          <td>0.00000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>0.00000</td>
-          <td>1</td>
-          <td>[]</td>
-          <td>0.0658</td>
-        </tr>
-        <tr>
-          <th>2000-01-04 21:00:00</th>
-          <td>3.423135</td>
-          <td>3.367492e-07</td>
-          <td>-3.000000e-08</td>
-          <td>-0.064897</td>
-          <td>-0.047528</td>
-          <td>0.323229</td>
-          <td>0.000001</td>
-          <td>-34.53135</td>
-          <td>9999965.46865</td>
-          <td>34.23135</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>0.00000</td>
-          <td>2</td>
-          <td>[{u'order_id': u'513357725cb64a539e3dd02b47da7...</td>
-          <td>0.0649</td>
-        </tr>
-        <tr>
-          <th>2000-01-05 21:00:00</th>
-          <td>3.473229</td>
-          <td>4.001918e-07</td>
-          <td>-9.906000e-09</td>
-          <td>-0.066196</td>
-          <td>-0.045697</td>
-          <td>0.329321</td>
-          <td>0.000001</td>
-          <td>-35.03229</td>
-          <td>9999930.43636</td>
-          <td>69.46458</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>9999965.46865</td>
-          <td>34.23135</td>
-          <td>34.23135</td>
-          <td>3</td>
-          <td>[{u'order_id': u'd7d4ad03cfec4d578c0d817dc3829...</td>
-          <td>0.0662</td>
-        </tr>
-        <tr>
-          <th>2000-01-06 21:00:00</th>
-          <td>3.172661</td>
-          <td>4.993979e-06</td>
-          <td>-6.410420e-07</td>
-          <td>-0.065758</td>
-          <td>-0.044785</td>
-          <td>0.298325</td>
-          <td>-0.000006</td>
-          <td>-32.02661</td>
-          <td>9999898.40975</td>
-          <td>95.17983</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>-12731.780516</td>
-          <td>9999930.43636</td>
-          <td>69.46458</td>
-          <td>69.46458</td>
-          <td>4</td>
-          <td>[{u'order_id': u'1fbf5e9bfd7c4d9cb2e8383e1085e...</td>
-          <td>0.0657</td>
-        </tr>
-        <tr>
-          <th>2000-01-07 21:00:00</th>
-          <td>3.322945</td>
-          <td>5.977002e-06</td>
-          <td>-2.201900e-07</td>
-          <td>-0.065206</td>
-          <td>-0.018908</td>
-          <td>0.375301</td>
-          <td>0.000005</td>
-          <td>-33.52945</td>
-          <td>9999864.88030</td>
-          <td>132.91780</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>-12629.274583</td>
-          <td>9999898.40975</td>
-          <td>95.17983</td>
-          <td>95.17983</td>
-          <td>5</td>
-          <td>[{u'order_id': u'9ea6b142ff09466b9113331a37437...</td>
-          <td>0.0652</td>
-        </tr>
-      </tbody>
-    </table>
-    <p>5 rows × 39 columns</p>
-    </div>
-
-
-
-As you can see, there is a row for each trading day, starting on the
-first business day of 2000. In the columns you can find various
+There is a row for each trading day, starting on the first day of our 
+simulation Jan 1st, 2016. In the columns you can find various
 information about the state of your algorithm. The very first column
-``AAPL`` was placed there by the ``record()`` function mentioned earlier
-and allows us to plot the price of apple. For example, we could easily
+``btc`` was placed there by the ``record()`` function mentioned earlier
+and allows us to plot the price of bitcoin. For example, we could easily
 examine now how our portfolio value changed over time compared to the
-AAPL stock price.
+bitcoin price.
 
-.. code-block:: python
-
-    %pylab inline
-    figsize(12, 12)
-    import matplotlib.pyplot as plt
-
-    ax1 = plt.subplot(211)
-    perf.portfolio_value.plot(ax=ax1)
-    ax1.set_ylabel('portfolio value')
-    ax2 = plt.subplot(212, sharex=ax1)
-    perf.AAPL.plot(ax=ax2)
-    ax2.set_ylabel('AAPL stock price')
-
-.. parsed-literal::
-
-    Populating the interactive namespace from numpy and matplotlib
-
-.. parsed-literal::
-
-    <matplotlib.text.Text at 0x7ff5c6147f90>
-
-.. image:: tutorial_files/tutorial_11_2.png
-
-
-As you can see, our algorithm performance as assessed by the
-``portfolio_value`` closely matches that of the AAPL stock price. This
-is not surprising as our algorithm only bought AAPL every chance it got.
-
-IPython Notebook
-~~~~~~~~~~~~~~~~
-
-The `IPython Notebook <http://ipython.org/notebook.html>`__ is a very
-powerful browser-based interface to a Python interpreter (this tutorial
-was written in it). As it is already the de-facto interface for most
-quantitative researchers ``zipline`` provides an easy way to run your
-algorithm inside the Notebook without requiring you to use the CLI.
-
-To use it you have to write your algorithm in a cell and let ``zipline``
-know that it is supposed to run this algorithm. This is done via the
-``%%zipline`` IPython magic command that is available after you
-``import zipline`` from within the IPython Notebook. This magic takes
-the same arguments as the command line interface described above. Thus
-to run the algorithm from above with the same parameters we just have to
-execute the following cell after importing ``zipline`` to register the
-magic.
-
-.. code-block:: python
-
-   %load_ext zipline
-
-.. code-block:: python
-
-   %%zipline --start 2000-1-1 --end 2014-1-1
-   from zipline.api import symbol, order, record
-
-   def initialize(context):
-       pass
-
-   def handle_data(context, data):
-       order(symbol('AAPL'), 10)
-       record(AAPL=data[symbol('AAPL')].price)
-
-Note that we did not have to specify an input file as above since the
-magic will use the contents of the cell and look for your algorithm
-functions there. Also, instead of defining an output file we are
-specifying a variable name with ``-o`` that will be created in the name
-space and contain the performance ``DataFrame`` we looked at above.
-
-.. code-block:: python
-
-   _.head()
-
-.. raw:: html
-
-    <div style="max-height:1000px;max-width:1500px;overflow:auto;">
-    <table border="1" class="dataframe">
-      <thead>
-        <tr style="text-align: right;">
-          <th></th>
-          <th>AAPL</th>
-          <th>algo_volatility</th>
-          <th>algorithm_period_return</th>
-          <th>alpha</th>
-          <th>benchmark_period_return</th>
-          <th>benchmark_volatility</th>
-          <th>beta</th>
-          <th>capital_used</th>
-          <th>ending_cash</th>
-          <th>ending_exposure</th>
-          <th>...</th>
-          <th>short_exposure</th>
-          <th>short_value</th>
-          <th>shorts_count</th>
-          <th>sortino</th>
-          <th>starting_cash</th>
-          <th>starting_exposure</th>
-          <th>starting_value</th>
-          <th>trading_days</th>
-          <th>transactions</th>
-          <th>treasury_period_return</th>
-        </tr>
-      </thead>
-      <tbody>
-        <tr>
-          <th>2000-01-03 21:00:00</th>
-          <td>3.738314</td>
-          <td>0.000000e+00</td>
-          <td>0.000000e+00</td>
-          <td>-0.065800</td>
-          <td>-0.009549</td>
-          <td>0.000000</td>
-          <td>0.000000</td>
-          <td>0.00000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>0.00000</td>
-          <td>1</td>
-          <td>[]</td>
-          <td>0.0658</td>
-        </tr>
-        <tr>
-          <th>2000-01-04 21:00:00</th>
-          <td>3.423135</td>
-          <td>3.367492e-07</td>
-          <td>-3.000000e-08</td>
-          <td>-0.064897</td>
-          <td>-0.047528</td>
-          <td>0.323229</td>
-          <td>0.000001</td>
-          <td>-34.53135</td>
-          <td>9999965.46865</td>
-          <td>34.23135</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>10000000.00000</td>
-          <td>0.00000</td>
-          <td>0.00000</td>
-          <td>2</td>
-          <td>[{u'commission': 0.3, u'amount': 10, u'sid': 0...</td>
-          <td>0.0649</td>
-        </tr>
-        <tr>
-          <th>2000-01-05 21:00:00</th>
-          <td>3.473229</td>
-          <td>4.001918e-07</td>
-          <td>-9.906000e-09</td>
-          <td>-0.066196</td>
-          <td>-0.045697</td>
-          <td>0.329321</td>
-          <td>0.000001</td>
-          <td>-35.03229</td>
-          <td>9999930.43636</td>
-          <td>69.46458</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0.000000</td>
-          <td>9999965.46865</td>
-          <td>34.23135</td>
-          <td>34.23135</td>
-          <td>3</td>
-          <td>[{u'commission': 0.3, u'amount': 10, u'sid': 0...</td>
-          <td>0.0662</td>
-        </tr>
-        <tr>
-          <th>2000-01-06 21:00:00</th>
-          <td>3.172661</td>
-          <td>4.993979e-06</td>
-          <td>-6.410420e-07</td>
-          <td>-0.065758</td>
-          <td>-0.044785</td>
-          <td>0.298325</td>
-          <td>-0.000006</td>
-          <td>-32.02661</td>
-          <td>9999898.40975</td>
-          <td>95.17983</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>-12731.780516</td>
-          <td>9999930.43636</td>
-          <td>69.46458</td>
-          <td>69.46458</td>
-          <td>4</td>
-          <td>[{u'commission': 0.3, u'amount': 10, u'sid': 0...</td>
-          <td>0.0657</td>
-        </tr>
-        <tr>
-          <th>2000-01-07 21:00:00</th>
-          <td>3.322945</td>
-          <td>5.977002e-06</td>
-          <td>-2.201900e-07</td>
-          <td>-0.065206</td>
-          <td>-0.018908</td>
-          <td>0.375301</td>
-          <td>0.000005</td>
-          <td>-33.52945</td>
-          <td>9999864.88030</td>
-          <td>132.91780</td>
-          <td>...</td>
-          <td>0</td>
-          <td>0</td>
-          <td>0</td>
-          <td>-12629.274583</td>
-          <td>9999898.40975</td>
-          <td>95.17983</td>
-          <td>95.17983</td>
-          <td>5</td>
-          <td>[{u'commission': 0.3, u'amount': 10, u'sid': 0...</td>
-          <td>0.0652</td>
-        </tr>
-      </tbody>
-    </table>
-    <p>5 rows × 39 columns</p>
-    </div>
+Our algorithm performance as assessed by the
+``portfolio_value`` closely matches that of the bitcoin price. This
+is not surprising as our algorithm only bought bitcoin every chance it got.
 
 
 Access to previous prices using ``history``
@@ -627,22 +300,16 @@ we need a new concept: History
 ``data.history()`` is a convenience function that keeps a rolling window of
 data for you. The first argument is the number of bars you want to
 collect, the second argument is the unit (either ``'1d'`` for ``'1m'``
-but note that you need to have minute-level data for using ``1m``). For
-a more detailed description ``history()``'s features, see the
-`Quantopian docs <https://www.quantopian.com/help#ide-history>`__.
-Let's look at the strategy which should make this clear:
+but note that you need to have minute-level data for using ``1m``). This is
+a function we use in the ``handle_data()`` section:
 
 .. code-block:: python
 
-   %%zipline --start 2000-1-1 --end 2012-1-1 -o dma.pickle
+    from catalyst.api import order, record, symbol
 
-
-   from zipline.api import order_target, record, symbol
-
-   def initialize(context):
+    def initialize(context):
        context.i = 0
-       context.asset = symbol('AAPL')
-
+       context.asset = symbol('btc_usd')
 
    def handle_data(context, data):
        # Skip first 300 days to get full windows
@@ -665,67 +332,22 @@ Let's look at the strategy which should make this clear:
            order_target(context.asset, 0)
 
        # Save values for later inspection
-       record(AAPL=data.current(context.asset, 'price'),
+       record(btc=data.current(context.asset, 'price'),
               short_mavg=short_mavg,
               long_mavg=long_mavg)
 
 
-   def analyze(context, perf):
-       fig = plt.figure()
-       ax1 = fig.add_subplot(211)
-       perf.portfolio_value.plot(ax=ax1)
-       ax1.set_ylabel('portfolio value in $')
-
-       ax2 = fig.add_subplot(212)
-       perf['AAPL'].plot(ax=ax2)
-       perf[['short_mavg', 'long_mavg']].plot(ax=ax2)
-
-       perf_trans = perf.ix[[t != [] for t in perf.transactions]]
-       buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]]
-       sells = perf_trans.ix[
-           [t[0]['amount'] < 0 for t in perf_trans.transactions]]
-       ax2.plot(buys.index, perf.short_mavg.ix[buys.index],
-                '^', markersize=10, color='m')
-       ax2.plot(sells.index, perf.short_mavg.ix[sells.index],
-                'v', markersize=10, color='k')
-       ax2.set_ylabel('price in $')
-       plt.legend(loc=0)
-       plt.show()
-
-.. image:: tutorial_files/tutorial_22_1.png
-
-Here we are explicitly defining an ``analyze()`` function that gets
-automatically called once the backtest is done (this is not possible on
-Quantopian currently).
-
-Although it might not be directly apparent, the power of ``history()``
-(pun intended) can not be under-estimated as most algorithms make use of
-prior market developments in one form or another. You could easily
-devise a strategy that trains a classifier with
-`scikit-learn <http://scikit-learn.org/stable/>`__ which tries to
-predict future market movements based on past prices (note, that most of
-the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
-``pandas.DataFrame``\ s, so you can simply pass the underlying
-``ndarray`` of a ``DataFrame`` via ``.values``).
-
-We also used the ``order_target()`` function above. This and other
-functions like it can make order management and portfolio rebalancing
-much easier. See the `Quantopian documentation on order
-functions <https://www.quantopian.com/help#api-order-methods>`__ fore
-more details.
-
 Conclusions
 ~~~~~~~~~~~
 
 We hope that this tutorial gave you a little insight into the
-architecture, API, and features of ``zipline``. For next steps, check
+architecture, API, and features of ``catalyst``. For next steps, check
 out some of the
-`examples <https://github.com/quantopian/zipline/tree/master/zipline/examples>`__.
+`examples <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`__.
+The natural next step would be too look into the 
+`buy_and_hodl <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_ 
+example, which is a more elaborated and realistic version of the ``buy_btc_simple`` example presented in this tutorial.
 
-Feel free to ask questions on `our mailing
-list <https://groups.google.com/forum/#!forum/zipline>`__, report
-problems on our `GitHub issue
-tracker <https://github.com/quantopian/zipline/issues?state=open>`__,
-`get
-involved <https://github.com/quantopian/zipline/wiki/Contribution-Requests>`__,
-and `checkout Quantopian <https://quantopian.com>`__.
+Feel free to ask questions on the ``#catalyst_dev`` channel of our 
+`Discord group <https://discord.gg/SJK32GY>`__ and report
+problems on our `GitHub issue tracker <https://github.com/enigmampc/catalyst/issues>`__.

docs/source/conf.py

@@ -41,7 +41,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Catalyst'
-copyright = u'2017, Enigma MPC'
+copyright = u'2017, Enigma MPC, Inc.'
 
 # The full version, including alpha/beta/rc tags, but excluding the commit hash
 #release = version.split('+', 1)[0]
@@ -94,6 +94,6 @@ intersphinx_mapping = {
     'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None),
 }
 
-doctest_global_setup = "import zipline"
+doctest_global_setup = "import catalyst"
 
 todo_include_todos = True

docs/source/index.rst

@@ -1,12 +1,17 @@
-.. include:: ../../README.rst
+.. include:: welcome.rst
+|
+|
+Table of Contents
+-----------------
 
 .. toctree::
    :maxdepth: 1
 
    install
    beginner-tutorial
-   bundles
-   development-guidelines
-   appendix
-   release-process
-   releases
+   naming-convention
+..   bundles
+..   development-guidelines
+..   appendix
+..   release-process
+..   releases

docs/source/install.rst

@@ -4,16 +4,16 @@ Install
 Installing with ``pip``
 -----------------------
 
-Installing Zipline via ``pip`` is slightly more involved than the average
+Installing Catalyst via ``pip`` is slightly more involved than the average
 Python package.
 
 There are two reasons for the additional complexity:
 
-1. Zipline ships several C extensions that require access to the CPython C API.
+1. Catalyst ships several C extensions that require access to the CPython C API.
    In order to build the C extensions, ``pip`` needs access to the CPython
    header files for your Python installation.
 
-2. Zipline depends on `numpy <http://www.numpy.org/>`_, the core library for
+2. Catalyst depends on `numpy <http://www.numpy.org/>`_, the core library for
    numerical array computing in Python.  Numpy depends on having the `LAPACK
    <http://www.netlib.org/lapack>`_ linear algebra routines available.
 
@@ -28,13 +28,28 @@ your particular platform), you should be able to simply run
 
 .. code-block:: bash
 
-   $ pip install zipline
+   $ pip install enigma-catalyst
 
-If you use Python for anything other than Zipline, we **strongly** recommend
+If you use Python for anything other than Catalyst, we **strongly** recommend
 that you install in a `virtualenv
 <https://virtualenv.readthedocs.org/en/latest>`_.  The `Hitchhiker's Guide to
 Python`_ provides an `excellent tutorial on virtualenv
-<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_.
+<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_. Here's a summarized
+version:
+
+.. code-block:: bash
+
+   $ virtualenv catalyst-venv
+   $ source ./catalyst-venv/bin/activate
+   $ pip install enigma-
+
+Though not required by Catalyst directly, our example algorithms use matplotlib 
+to visually display the results of the trading algorithms. If you wish to run 
+any examples or use matplotlib during development, it can be installed using:
+
+.. code-block:: bash
+
+    $ pip install matplotlib
 
 GNU/Linux
 ~~~~~~~~~
@@ -60,15 +75,17 @@ On `Arch Linux`_, you can acquire the additional dependencies via ``pacman``:
 
    $ pacman -S lapack gcc gcc-fortran pkg-config
 
-There are also AUR packages available for installing `Python 3.4
-<https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
-3.5, but Zipline only currently supports 3.4), and `ta-lib
-<https://aur.archlinux.org/packages/ta-lib/>`_, an optional Zipline dependency.
-Python 2 is also installable via:
+.. Commenting it out until Catalyst fully supports Python 3.X
+..
+.. There are also AUR packages available for installing `Python 3.4
+.. <https://aur.archlinux.org/packages/python34/>`_ (Arch's default python is now
+.. 3.5, but Catalyst only currently supports 3.4), and `ta-lib
+.. <https://aur.archlinux.org/packages/ta-lib/>`_, an optional Catalyst dependency.
+.. Python 2 is also installable via:
 
-.. code-block:: bash
+.. 
 
-   $ pacman -S python2
+..   $ pacman -S python2
 
 OSX
 ~~~
@@ -87,36 +104,238 @@ following brew packages:
 
    $ brew install freetype pkg-config gcc openssl
 
+OSX + virtualenv + matplotlib
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A note about using matplotlib in virtual enviroments on OSX: it may be necessary to run
+
+.. code-block:: bash
+
+   echo "backend: TkAgg" > ~/.matplotlib/matplotlibrc
+
+in order to override the default ``macosx`` backend for your system, which may not 
+be accessible from inside the virtual environment. This will allow Catalyst to open 
+matplotlib charts from within a virtual environment, which is useful for displaying 
+the performance of your backtests.  To learn more about matplotlib backends, please refer to the
+`matplotlib backend documentation <https://matplotlib.org/faq/usage_faq.html#what-is-a-backend>`_.
+
+
 Windows
 ~~~~~~~
 
-For windows, the easiest and best supported way to install zipline is to use
+In Windows, you will need the `Microsoft Visual C++ Compiler for Python 2.7 
+<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This package 
+contains the compiler and the set of system headers necessary for producing 
+binary wheels for Python 2.7 packages. If it's not already in your system, download
+it and install it before proceeding to the next step.
+
+For windows, the easiest and best supported way to install Catalyst is to use
 :ref:`Conda <conda>`.
 
+Amazon Linux AMI
+~~~~~~~~~~~~~~~~
+
+The packages ``pip`` and ``setuptools`` that come shipped by default are very outdated. 
+Thus, you first need to run:
+
+.. code-block:: bash
+
+   pip install --upgrade pip setuptools
+
+The default installation is also missing the C and C++ compilers, which you install by:
+
+.. code-block:: bash
+
+   sudo yum install gcc gcc-c++
+
+Then you should follow the regular installation instructions outlined at the beginning 
+of this page.
+
+
+Troubleshooting ``pip`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Issue**: 
+   Package enigma-catalyst cannot be found 
+ 
+**Solution**: 
+   Make sure you have the most up-to-date version of pip installed, by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip
+
+   On Windows, the recommended command is:
+
+   .. code-block:: bash
+
+      python -m pip install --upgrade pip
+
+----
+
+**Issue**: 
+   Package enigma-catalyst cannot still be found, even after upgrading pip (see above), with an error similar to:
+
+   .. code-block:: bash
+
+      Downloading/unpacking enigma-catalyst
+      Could not find a version that satisfies the requirement enigma-catalyst (from versions: 0.1.dev9, 0.2.dev2, 0.1.dev4, 0.1.dev5, 0.1.dev3, 0.2.dev1, 0.1.dev8, 0.1.dev6)
+      Cleaning up...
+      No distributions matching the version for enigma-catalyst
+
+**Solution**:
+   In some systems (this error has been reported in Ubuntu), pip is configured to only find stable versions by default. Since Catalyst is in alpha version, pip cannot find a matching version that satisfies the installation requirements. The solution is to include the `--pre` flag to include pre-release and development versions:
+
+   .. code-block:: bash
+
+      pip install --pre enigma-catalyst
+
+----
+
+**Issue**: 
+   Package enigma-catalyst fails to install because of outdated setuptools
+
+**Solution**: 
+   Upgrade to the most up-to-date setuptools package by running: 
+
+   .. code-block:: bash
+
+      pip install --upgrade pip setuptools
+
+----
+
+**Issue**:
+   Missing required packages  
+
+**Solution**:
+   Download `requirements.txt 
+   <https://github.com/enigmampc/catalyst/blob/master/etc/requirements.txt>`_ 
+   (click on the *Raw* button and Right click -> Save As...) and use it to
+   install all the required dependencies by running:
+
+   .. code-block:: bash
+
+      pip install -r requirements.txt
+
+----
+
+**Issue**: 
+   Installation fails with error: ``fatal error: Python.h: No such file or directory``
+
+**Solution**: 
+   Some systems (this issue has been reported in Ubuntu) require `python-dev` for the proper build and installation of package dependencies. The solution is to install python-dev, which is independent of the virtual environment. In Ubuntu, you would need to run:
+
+   .. code-block:: bash
+
+      sudo apt-get install python-dev
+
+
 .. _conda:
 
 Installing with ``conda``
 -------------------------
 
-Another way to install Zipline is via the ``conda`` package manager, which
+Another way to install Catalyst is via the ``conda`` package manager, which
 comes as part of Continuum Analytics' `Anaconda
 <http://continuum.io/downloads>`_ distribution.
 
 The primary advantage of using Conda over ``pip`` is that conda natively
 understands the complex binary dependencies of packages like ``numpy`` and
-``scipy``.  This means that ``conda`` can install Zipline and its dependencies
-without requiring the use of a second tool to acquire Zipline's non-Python
+``scipy``.  This means that ``conda`` can install Catalyst and its dependencies
+without requiring the use of a second tool to acquire Catalyst's non-Python
 dependencies.
 
 For instructions on how to install ``conda``, see the `Conda Installation
-Documentation <http://conda.pydata.org/docs/download.html>`_
+Documentation <http://conda.pydata.org/docs/download.html>`_. Alternatively, you 
+can install MiniConda, which is a smaller footprint (fewer packages and smaller 
+size) than its big brother Anaconda, but it still contains all the main packages 
+needed. To install MiniConda, you can follow these steps:
 
-Once conda has been set up you can install Zipline from our ``Quantopian``
-channel:
+1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7 for 
+   your Operating System.
+2. Install MiniConda. See the `Installation Instructions <https://conda.io/docs/user-guide/install/index.html>`_
+   if you need help.
+3. Ensure the correct installation by running ``conda list`` in a Terminal window,
+   which should print the list of packages installed with Conda.
 
-.. code-block:: bash
+Once either Conda or MiniConda has been set up you can install Catalyst:
+
+1. Download the file `python2.7-environment.yml <https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
+2. Open a Terminal window and enter [``cd/dir``] into the directory where you saved
+   the above ``python2.7-environment.yml`` file.
+3. Install using this file. This step can take about 5-10 minutes to install.
+
+   .. code-block:: bash
+
+      conda env create -f python2.7-environment.yml
+
+4. Activate the environment (which you need to do every time you start a new session
+   to run Catalyst):
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+Congratulations! You now have Catalyst installed.
+
+Troubleshooting ``conda`` Install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the command  ``conda env create -f python2.7-environment.yml`` in step 3 above failed 
+for any reason, you can try setting up the environment manually with the following steps:
+
+1. Create the environment:
+
+   .. code-block:: bash
+
+      conda create --name catalyst python=2.7 scipy
+
+2. Activate the environment:
+
+   **Linux or OSX:**
+
+   .. code-block:: bash
+
+      source activate catalyst
+
+   **Windows:**
+
+   .. code-block:: bash
+
+      activate catalyst
+
+3. Install the Catalyst inside the environment:
+
+   .. code-block:: bash
+
+      pip install enigma-catalyst matplotlib
+
+Getting Help
+------------
+
+If after following the instructions above, and going through the *Troubleshooting* sections, 
+you still experience problems installing Catalyst, you can seek additional help through the 
+following channels:
+
+- Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over the #catalyst_dev 
+  channel where many other users (as well as the project developers) hang out, and can assist 
+  you with your particular issue. The more descriptive and the more information you can provide, 
+  the easiest will be for others to help you out.
+
+- Report the problem you are experiencing on our 
+  `GitHub repository <https://github.com/enigmampc/catalyst/issues>`_ following the guidelines 
+  provided therein. Before you do so, take a moment to browse through all `previous reported issues
+  <https://github.com/enigmampc/catalyst/issues?utf8=%E2%9C%93&q=is%3Aissue>`_ in the likely case
+  that someone else experienced that same issue before, and you get a hint on how to solve it.
 
-    conda install -c Quantopian zipline
 
 .. _`Debian-derived`: https://www.debian.org/misc/children-distros
 .. _`RHEL-derived`: https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux_derivatives

docs/source/naming-convention.rst

@@ -0,0 +1,66 @@
+Naming Convention
+=================
+
+Catalyst introduces a standardized naming convention for all asset pairs 
+trading on any exchange in the following form:
+
+
+    **{market_currency}_{base_currency}**
+
+Where {market_currency} is the asset to be traded using {base_currency} as 
+the reference, both written in lowercase and separated with an underscore.
+
+This standardization is needed to overcome the lack of consistency in the 
+naming of assets across different exchanges, and making it easier to the user
+to refer to the asset pairs that you want to trade.
+
+Catalyst maintains a `Market Coverage Overview <https://www.enigma.co/catalyst/status>`_ 
+where you can check the mapping between Catalyst naming pairs and that of each 
+exchange. Catalyst will always expect in all its functions that you will refer to 
+the asset pairs by using the Catalyst naming convention.
+
+If at any point, you input the wrong name for an asset pair, you will get an error 
+of that pair not found in the given exchange, and a list of pairs available on that exchange:
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usd
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	Error traceback: /Volumes/Data/Users/victoris/Desktop/Enigma/user-install/catalyst-dev/catalyst/exchange/exchange.py (line 175)
+	SymbolNotFoundOnExchange:  Symbol btc_usd not found on exchange Poloniex. 
+	Choose from: ['rep_usdt', 'gno_btc', 'xvc_btc', 'pink_btc', 'sys_btc', 
+	'emc2_btc', 'rads_btc', 'note_btc', 'maid_btc', 'bch_btc', 'gnt_btc', 
+	'bcn_btc', 'rep_btc', 'bcy_btc', 'cvc_btc', 'nxt_xmr', 'zec_usdt', 
+	'fct_btc', 'gas_btc', 'pot_btc', 'eth_usdt', 'btc_usdt', 'lbc_btc', 
+	'dcr_btc', 'etc_usdt', 'omg_eth', 'amp_btc', 'xpm_btc', 'nxt_btc', 
+	'vtc_btc', 'steem_eth', 'blk_xmr', 'pasc_btc', 'zec_xmr', 'grc_btc', 
+	'nxc_btc', 'btcd_btc', 'ltc_btc', 'dash_btc', 'naut_btc', 'zec_eth', 
+	'zec_btc', 'burst_btc', 'zrx_eth', 'bela_btc', 'steem_btc', 'etc_btc', 
+	'eth_btc', 'huc_btc', 'strat_btc', 'lsk_btc', 'exp_btc', 'clam_btc', 
+	'rep_eth', 'dash_xmr', 'cvc_eth', 'bch_usdt', 'zrx_btc', 'dash_usdt', 
+	'blk_btc', 'xrp_btc', 'nxt_usdt', 'neos_btc', 'omg_btc', 'bts_btc', 
+	'doge_btc', 'gnt_eth', 'sbd_btc', 'gno_eth', 'xcp_btc', 'ltc_usdt', 
+	'btm_btc', 'xmr_usdt', 'lsk_eth', 'omni_btc', 'nav_btc', 'fldc_btc', 
+	'ppc_btc', 'xbc_btc', 'dgb_btc', 'sc_btc', 'btcd_xmr', 'vrc_btc', 
+	'ric_btc', 'str_btc', 'maid_xmr', 'xmr_btc', 'sjcx_btc', 'via_btc', 
+	'xem_btc', 'nmc_btc', 'etc_eth', 'ltc_xmr', 'ardr_btc', 'gas_eth', 
+	'flo_btc', 'xrp_usdt', 'game_btc', 'bch_eth', 'bcn_xmr', 'str_usdt']
+
+In the example above, exchange Poloniex does not use USD, but uses instead the 
+USDT cryptocurrency asset that is issued on the Bitcoin blockchain via the Omni
+Layer Protocol. Each USDT unit is backed by a U.S Dollar held in the reserves of 
+Tether Limited. USDT can be transferred, stored, and spent, just like bitcoins 
+or any other cryptocurrency. Given its 1:1 mapping to the USD, is a viable alternative.
+
+.. code-block:: bash
+
+   $ catalyst ingest-exchange -x poloniex -i btc_usdt
+
+.. parsed-literal::
+
+	Ingesting exchange bundle poloniex...
+	    [====================================]  Fetching poloniex daily candles: :  100%
+

docs/source/welcome.rst

@@ -0,0 +1,28 @@
+.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
+|
+Catalyst is a data-driven crypto investment platform. It supports both 
+backtesting and live-trading in a number of different crypto-exchanges. 
+Catalyst empowers users to share and curate data and build profitable, 
+data-driven investment strategies.
+
+Features
+========
+
+-  Ease of use: Catalyst tries to get out of your way so that you can 
+   focus on algorithm development. See 
+   `examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_ 
+   provided.
+-  Support for several of the top crypto-exchanges by trading volume:
+   `Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
+   and `Poloniex <https://www.poloniex.com>`_. 
+-  Secure: You and only you have access to each exchange API keys for your accounts.
+-  Input of historical pricing data of all crypto-assets by exchange, 
+   with daily and minute resolution. See 
+   `Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
+-  Backtesting and live-trading functionality, with a seamless transition
+   between the two modes.
+-  Output of performance statistics are based on Pandas DataFrames to 
+   integrate nicely into the existing PyData eco-system.
+-  Statistic and machine learning libraries like matplotlib, scipy, 
+   statsmodels, and sklearn support development, analysis, and 
+   visualization of state-of-the-art trading systems.

etc/requirements_docs.txt

@@ -1,4 +1,3 @@
 Sphinx>=1.3.2
 numpydoc>=0.5.0
 sphinx-autobuild==0.6.0
-enigma-catalyst		# readthedocs.org

setup.py

@@ -304,7 +304,7 @@ setup(
                   if '__pycache__' not in root},
     license='Apache 2.0',
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
+        'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Programming Language :: Python',

tests/exchange/test_bundle.py

@@ -3,7 +3,8 @@ from logging import Logger
 import pandas as pd
 
 from catalyst import get_calendar
-from catalyst.exchange.bundle_utils import get_bcolz_chunk
+from catalyst.exchange.bundle_utils import get_bcolz_chunk, get_periods, \
+    get_periods_range
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_bundle import ExchangeBundle, \
@@ -16,6 +17,25 @@ log = Logger('test_exchange_bundle')
 
 
 class ExchangeBundleTestCase:
+    def test_spot_value(self):
+        data_frequency = 'daily'
+        exchange_name = 'poloniex'
+
+        exchange = get_exchange(exchange_name)
+        exchange_bundle = ExchangeBundle(exchange)
+        assets = [
+            exchange.get_asset('btc_usdt')
+        ]
+        dt = pd.to_datetime('2017-10-14', utc=True)
+
+        values = exchange_bundle.get_spot_values(
+            assets=assets,
+            field='close',
+            dt=dt,
+            data_frequency=data_frequency
+        )
+        pass
+
     def test_ingest_minute(self):
         data_frequency = 'minute'
         exchange_name = 'bitfinex'
@@ -78,12 +98,13 @@ class ExchangeBundleTestCase:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bitfinex'
+        exchange_name = 'poloniex'
         data_frequency = 'daily'
-        include_symbols = 'etc_btc'
+        include_symbols = 'btc_usdt'
 
-        start = pd.to_datetime('2016-11-01', utc=True)
+        start = pd.to_datetime('2016-1-1', utc=True)
         end = pd.to_datetime('2017-10-16', utc=True)
+        periods = get_periods_range(start, end, data_frequency)
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)