Changed zipline -> catalyst import paths

* Updated cython build scripts
 * Updated setup.py to to install catalyst package
 * Updated momentum example to use catalyst package
 * catalyst executable now supports loading pipelines from multiple bundles

catalyst/data/__init__.py

@@ -0,0 +1,16 @@
+from . import loader
+from .loader import (
+    load_from_yahoo,
+    load_bars_from_yahoo,
+    load_prices_from_csv,
+    load_prices_from_csv_folder,
+)
+
+
+__all__ = [
+    'load_bars_from_yahoo',
+    'load_from_yahoo',
+    'load_prices_from_csv',
+    'load_prices_from_csv_folder',
+    'loader',
+]

catalyst/data/_adjustments.pyx

@@ -0,0 +1,303 @@
+#
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from cpython cimport (
+    PyDict_Contains,
+    PySet_Add,
+)
+
+from numpy import (
+    int64,
+    uint32,
+    zeros,
+)
+from numpy cimport int64_t, ndarray
+from pandas import Timestamp
+
+ctypedef object Timestamp_t
+ctypedef object DatetimeIndex_t
+ctypedef object Int64Index_t
+
+from catalyst.lib.adjustment import Float64Multiply
+from catalyst.assets.asset_writer import (
+    SQLITE_MAX_VARIABLE_NUMBER as SQLITE_MAX_IN_STATEMENT,
+)
+from catalyst.utils.pandas_utils import timedelta_to_integral_seconds
+
+
+_SID_QUERY_TEMPLATE = """
+SELECT DISTINCT sid FROM {0}
+WHERE effective_date >= ? AND effective_date <= ?
+"""
+cdef dict SID_QUERIES = {
+    tablename: _SID_QUERY_TEMPLATE.format(tablename)
+    for tablename in ('splits', 'dividends', 'mergers')
+}
+
+ADJ_QUERY_TEMPLATE = """
+SELECT sid, ratio, effective_date
+FROM {0}
+WHERE sid IN ({1}) AND effective_date >= {2} AND effective_date <= {3}
+"""
+
+EPOCH = Timestamp(0, tz='UTC')
+
+cdef set _get_sids_from_table(object db,
+                              str tablename,
+                              int start_date,
+                              int end_date):
+    """
+    Get the unique sids for all adjustments between start_date and end_date
+    from table `tablename`.
+
+    Parameters
+    ----------
+    db : sqlite3.connection
+    tablename : str
+    start_date : int (seconds since epoch)
+    end_date : int (seconds since epoch)
+
+    Returns
+    -------
+    sids : set
+        Set of sets
+    """
+
+    cdef object cursor = db.execute(
+        SID_QUERIES[tablename],
+        (start_date, end_date),
+    )
+    cdef set out = set()
+    cdef tuple result
+    for result in cursor.fetchall():
+        PySet_Add(out, result[0])
+    return out
+
+
+cdef set _get_split_sids(object db, int start_date, int end_date):
+    return _get_sids_from_table(db, 'splits', start_date, end_date)
+
+
+cdef set _get_merger_sids(object db, int start_date, int end_date):
+    return _get_sids_from_table(db, 'mergers', start_date, end_date)
+
+
+cdef set _get_dividend_sids(object db, int start_date, int end_date):
+    return _get_sids_from_table(db, 'dividends', start_date, end_date)
+
+
+cdef _adjustments(object adjustments_db,
+                  set split_sids,
+                  set merger_sids,
+                  set dividends_sids,
+                  int start_date,
+                  int end_date,
+                  Int64Index_t assets):
+
+    c = adjustments_db.cursor()
+
+    splits_to_query = [str(a) for a in assets if a in split_sids]
+    splits_results = []
+    while splits_to_query:
+        query_len = min(len(splits_to_query), SQLITE_MAX_IN_STATEMENT)
+        query_assets = splits_to_query[:query_len]
+        t= [str(a) for a in query_assets]
+        statement = ADJ_QUERY_TEMPLATE.format('splits',
+            ",".join(['?' for _ in query_assets]), start_date, end_date)
+        c.execute(statement, t)
+        splits_to_query = splits_to_query[query_len:]
+        splits_results.extend(c.fetchall())
+
+    mergers_to_query = [str(a) for a in assets if a in merger_sids]
+    mergers_results = []
+    while mergers_to_query:
+        query_len = min(len(mergers_to_query), SQLITE_MAX_IN_STATEMENT)
+        query_assets = mergers_to_query[:query_len]
+        t= [str(a) for a in query_assets]
+        statement = ADJ_QUERY_TEMPLATE.format('mergers',
+            ",".join(['?' for _ in query_assets]), start_date, end_date)
+        c.execute(statement, t)
+        mergers_to_query = mergers_to_query[query_len:]
+        mergers_results.extend(c.fetchall())
+
+    dividends_to_query = [str(a) for a in assets if a in dividends_sids]
+    dividends_results = []
+    while dividends_to_query:
+        query_len = min(len(dividends_to_query), SQLITE_MAX_IN_STATEMENT)
+        query_assets = dividends_to_query[:query_len]
+        t= [str(a) for a in query_assets]
+        statement = ADJ_QUERY_TEMPLATE.format('dividends',
+            ",".join(['?' for _ in query_assets]), start_date, end_date)
+        c.execute(statement, t)
+        dividends_to_query = dividends_to_query[query_len:]
+        dividends_results.extend(c.fetchall())
+
+    return splits_results, mergers_results, dividends_results
+
+
+cpdef load_adjustments_from_sqlite(object adjustments_db,  # sqlite3.Connection
+                                   list columns,
+                                   DatetimeIndex_t dates,
+                                   Int64Index_t assets):
+    """
+    Load a dictionary of Adjustment objects from adjustments_db
+
+    Parameters
+    ----------
+    adjustments_db : sqlite3.Connection
+        Connection to a sqlite3 table in the format written by
+        SQLiteAdjustmentWriter.
+    columns : list[str]
+        List of column names for which adjustments are needed.
+    dates : pd.DatetimeIndex
+        Dates for which adjustments are needed
+    assets : pd.Int64Index
+        Assets for which adjustments are needed.
+
+    Returns
+    -------
+    adjustments : list[dict[int -> Adjustment]]
+        A list of mappings from index to adjustment objects to apply at that
+        index.
+    """
+
+    cdef int start_date = timedelta_to_integral_seconds(dates[0] - EPOCH)
+    cdef int end_date = timedelta_to_integral_seconds(dates[-1] - EPOCH)
+
+    cdef set split_sids = _get_split_sids(
+        adjustments_db,
+        start_date,
+        end_date,
+    )
+    cdef set merger_sids = _get_merger_sids(
+        adjustments_db,
+        start_date,
+        end_date,
+    )
+    cdef set dividend_sids = _get_dividend_sids(
+        adjustments_db,
+        start_date,
+        end_date,
+    )
+
+    cdef:
+        list splits, mergers, dividends
+    splits, mergers, dividends = _adjustments(
+        adjustments_db,
+        split_sids,
+        merger_sids,
+        dividend_sids,
+        start_date,
+        end_date,
+        assets,
+    )
+
+    cdef list results = [{} for column in columns]
+    cdef dict asset_ixs = {}  # Cache sid lookups here.
+    cdef dict date_ixs = {}
+    cdef:
+        int i
+        int dt
+        int sid
+        double ratio
+        int eff_date
+        int date_loc
+        Py_ssize_t asset_ix
+        dict col_adjustments
+
+    cdef ndarray[int64_t, ndim=1] _dates_seconds = \
+        dates.values.astype('datetime64[s]').view(int64)
+
+    # Pre-populate date index cache.
+    for i, dt in enumerate(_dates_seconds):
+        date_ixs[dt] = i
+
+    # splits affect prices and volumes, volumes is the inverse
+    for sid, ratio, eff_date in splits:
+        if eff_date < start_date:
+            continue
+
+        date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
+
+        if not PyDict_Contains(asset_ixs, sid):
+            asset_ixs[sid] = assets.get_loc(sid)
+        asset_ix = asset_ixs[sid]
+
+        price_adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
+        for i, column in enumerate(columns):
+            col_adjustments = results[i]
+            if column != 'volume':
+                try:
+                    col_adjustments[date_loc].append(price_adj)
+                except KeyError:
+                    col_adjustments[date_loc] = [price_adj]
+            else:
+                volume_adj = Float64Multiply(
+                    0, date_loc, asset_ix, asset_ix, 1.0 / ratio
+                )
+                try:
+                    col_adjustments[date_loc].append(volume_adj)
+                except KeyError:
+                    col_adjustments[date_loc] = [volume_adj]
+
+    # mergers affect prices only
+    for sid, ratio, eff_date in mergers:
+        if eff_date < start_date:
+            continue
+
+        date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
+
+        if not PyDict_Contains(asset_ixs, sid):
+            asset_ixs[sid] = assets.get_loc(sid)
+        asset_ix = asset_ixs[sid]
+
+        adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
+        for i, column in enumerate(columns):
+            col_adjustments = results[i]
+            if column != 'volume':
+                try:
+                    col_adjustments[date_loc].append(adj)
+                except KeyError:
+                    col_adjustments[date_loc] = [adj]
+
+    # dividends affect prices only
+    for sid, ratio, eff_date in dividends:
+        if eff_date < start_date:
+            continue
+
+        date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
+
+        if not PyDict_Contains(asset_ixs, sid):
+            asset_ixs[sid] = assets.get_loc(sid)
+        asset_ix = asset_ixs[sid]
+
+        adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
+        for i, column in enumerate(columns):
+            col_adjustments = results[i]
+            if column != 'volume':
+                try:
+                    col_adjustments[date_loc].append(adj)
+                except KeyError:
+                    col_adjustments[date_loc] = [adj]
+
+    return results
+
+
+cdef _lookup_dt(dict dt_cache,
+                int dt,
+                ndarray[int64_t, ndim=1] fallback):
+
+    if not PyDict_Contains(dt_cache, dt):
+        dt_cache[dt] = fallback.searchsorted(dt, side='right')
+    return dt_cache[dt]

catalyst/data/_equities.pyx

@@ -0,0 +1,223 @@
+#
+# Copyright 2015 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import bcolz
+cimport cython
+from cpython cimport bool
+
+from numpy import (
+    array,
+    float64,
+    intp,
+    uint32,
+    zeros,
+)
+from numpy cimport (
+    float64_t,
+    intp_t,
+    ndarray,
+    uint32_t,
+    uint8_t,
+)
+from numpy.math cimport NAN
+
+ctypedef object carray_t
+ctypedef object ctable_t
+ctypedef object Timestamp_t
+ctypedef object DatetimeIndex_t
+ctypedef object Int64Index_t
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+cpdef _compute_row_slices(dict asset_starts_absolute,
+                          dict asset_ends_absolute,
+                          dict asset_starts_calendar,
+                          intp_t query_start,
+                          intp_t query_end,
+                          Int64Index_t requested_assets):
+    """
+    Core indexing functionality for loading raw data from bcolz.
+
+    Parameters
+    ----------
+    asset_starts_absolute : dict
+        Dictionary containing the index of the first row of each asset in the
+        bcolz file from which we will query.
+
+    asset_ends_absolute : dict
+        Dictionary containing the index of the last row of each asset in the
+        bcolz file from which we will query.
+
+    asset_starts_calendar : dict
+        Dictionary containing the index of in our calendar corresponding to the
+        start date of each asset
+
+    query_start : intp
+    query_end : intp
+        Start and end indices in our calendar of the dates for which we're
+        querying.
+
+    requested_assets : pandas.Int64Index
+        The assets for which we want to load data.
+
+    For each asset in requested assets, computes three values:
+    1.) The index in the raw bcolz data of first row to load.
+    2.) The index in the raw bcolz data of the last row to load.
+    3.) The index in the dates of our query corresponding to the first row for
+        each asset. This is non-zero iff the asset's lifetime begins partway
+        through the requested query dates.
+
+    Returns
+    -------
+    first_rows, last_rows, offsets : 3-tuple of ndarrays
+    """
+    cdef:
+        intp_t nassets = len(requested_assets)
+
+        # For each sid, we need to compute the following:
+        ndarray[dtype=intp_t, ndim=1] first_row_a = zeros(nassets, dtype=intp)
+        ndarray[dtype=intp_t, ndim=1] last_row_a = zeros(nassets, dtype=intp)
+        ndarray[dtype=intp_t, ndim=1] offset_a = zeros(nassets, dtype=intp)
+
+        # Loop variables.
+        intp_t i
+        intp_t asset
+        intp_t asset_start_data
+        intp_t asset_end_data
+        intp_t asset_start_calendar
+        intp_t asset_end_calendar
+
+    for i, asset in enumerate(requested_assets):
+        asset_start_data = asset_starts_absolute[asset]
+        asset_end_data = asset_ends_absolute[asset]
+        asset_start_calendar = asset_starts_calendar[asset]
+        asset_end_calendar = (
+            asset_start_calendar + (asset_end_data - asset_start_data)
+        )
+
+        # If the asset started during the query, then start with the asset's
+        # first row.
+        # Otherwise start with the asset's first row + the number of rows
+        # before the query on which the asset existed.
+        first_row_a[i] = (
+            asset_start_data + max(0, (query_start - asset_start_calendar))
+        )
+        # If the asset ended during the query, the end with the asset's last
+        # row.
+        # Otherwise, end with the asset's last row minus the number of rows
+        # after the query for which the asset
+        last_row_a[i] = (
+            asset_end_data - max(0, asset_end_calendar - query_end)
+        )
+        # If the asset existed on or before the query, no offset.
+        # Otherwise, offset by the number of rows in the query in which the
+        # asset did not yet exist.
+        offset_a[i] = max(0, asset_start_calendar - query_start)
+
+    return first_row_a, last_row_a, offset_a
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+cpdef _read_bcolz_data(ctable_t table,
+                       tuple shape,
+                       list columns,
+                       intp_t[:] first_rows,
+                       intp_t[:] last_rows,
+                       intp_t[:] offsets,
+                       bool read_all):
+    """
+    Load raw bcolz data for the given columns and indices.
+
+    Parameters
+    ----------
+    table : bcolz.ctable
+        The table from which to read.
+    shape : tuple (length 2)
+        The shape of the expected output arrays.
+    columns : list[str]
+        List of column names to read.
+
+    first_rows : ndarray[intp]
+    last_rows : ndarray[intp]
+    offsets : ndarray[intp
+        Arrays in the format returned by _compute_row_slices.
+    read_all : bool
+        Whether to read_all sid data at once, or to read a silce from the
+        carray for each sid.
+
+    Returns
+    -------
+    results : list of ndarray
+        A 2D array of shape `shape` for each column in `columns`.
+    """
+    cdef:
+        int nassets
+        str column_name
+        carray_t carray
+        ndarray[dtype=uint32_t, ndim=1] raw_data
+        ndarray[dtype=uint32_t, ndim=2] outbuf
+        ndarray[dtype=uint8_t, ndim=2, cast=True] where_nan
+        ndarray[dtype=float64_t, ndim=2] outbuf_as_float
+        intp_t asset
+        intp_t out_idx
+        intp_t raw_idx
+        intp_t first_row
+        intp_t last_row
+        intp_t offset
+        list results = []
+
+    ndays = shape[0]
+    nassets = shape[1]
+    if not nassets== len(first_rows) == len(last_rows) == len(offsets):
+        raise ValueError("Incompatible index arrays.")
+
+    for column_name in columns:
+        outbuf = zeros(shape=shape, dtype=uint32)
+        if read_all:
+            raw_data = table[column_name][:]
+
+            for asset in range(nassets):
+                first_row = first_rows[asset]
+                last_row = last_rows[asset]
+                offset = offsets[asset]
+                if first_row <= last_row:
+                    outbuf[offset:offset + (last_row + 1 - first_row), asset] =\
+                        raw_data[first_row:last_row + 1]
+                else:
+                    continue
+        else:
+            carray = table[column_name]
+
+            for asset in range(nassets):
+                first_row = first_rows[asset]
+                last_row = last_rows[asset]
+                offset = offsets[asset]
+                out_start = offset
+                out_end = (last_row - first_row) + offset + 1
+                if first_row <= last_row:
+                    outbuf[offset:offset + (last_row + 1 - first_row), asset] =\
+                        carray[first_row:last_row + 1]
+                else:
+                    continue
+
+        if column_name in {'open', 'high', 'low', 'close'}:
+            where_nan = (outbuf == 0)
+            outbuf_as_float = outbuf.astype(float64) * .001
+            outbuf_as_float[where_nan] = NAN
+            results.append(outbuf_as_float)
+        else:
+            results.append(outbuf)
+    return results

catalyst/data/_minute_bar_internal.pyx

@@ -0,0 +1,159 @@
+from numpy cimport ndarray, long_t
+from numpy import searchsorted
+from cpython cimport bool
+cimport cython
+
+cdef inline int int_min(int a, int b): return a if a <= b else b
+
+@cython.cdivision(True)
+def minute_value(ndarray[long_t, ndim=1] market_opens,
+                 Py_ssize_t pos,
+                 short minutes_per_day):
+    """
+    Finds the value of the minute represented by `pos` in the given array of
+    market opens.
+
+    Parameters
+    ----------
+    market_opens: numpy array of ints
+        Market opens, in minute epoch values.
+
+    pos: int
+        The index of the desired minute.
+
+    minutes_per_day: int
+        The number of minutes per day (e.g. 390 for NYSE).
+
+    Returns
+    -------
+    int: The minute epoch value of the desired minute.
+    """
+    cdef short q, r
+
+    q = cython.cdiv(pos, minutes_per_day)
+    r = cython.cmod(pos, minutes_per_day)
+
+    return market_opens[q] + r
+
+def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
+                            ndarray[long_t, ndim=1] market_closes,
+                            long_t minute_val,
+                            short minutes_per_day,
+                            bool forward_fill):
+    """
+    Finds the position of a given minute in the given array of market opens.
+    If not a market minute, adjusts to the last market minute.
+
+    Parameters
+    ----------
+    market_opens: numpy array of ints
+        Market opens, in minute epoch values.
+
+    market_closes: numpy array of ints
+        Market closes, in minute epoch values.
+
+    minute_val: int
+        The desired minute, as a minute epoch.
+
+    minutes_per_day: int
+        The number of minutes per day (e.g. 390 for NYSE).
+
+    forward_fill: bool
+        Whether to use the previous market minute if the given minute does
+        not fall within an open/close pair.
+
+    Returns
+    -------
+    int: The position of the given minute in the market opens array.
+
+    Raises
+    ------
+    ValueError
+        If the given minute is not between a single open/close pair AND
+        forward_fill is False.  For example, if minute_val is 17:00 Eastern
+        for a given day whose normal hours are 9:30 to 16:00, and we are not
+        forward filling, ValueError is raised.
+    """
+    cdef Py_ssize_t market_open_loc, market_open, delta
+
+    market_open_loc = \
+        searchsorted(market_opens, minute_val, side='right') - 1
+    market_open = market_opens[market_open_loc]
+    market_close = market_closes[market_open_loc]
+
+    if not forward_fill and ((minute_val - market_open) >= minutes_per_day):
+        raise ValueError("Given minute is not between an open and a close")
+
+    delta = int_min(minute_val - market_open, market_close - market_open)
+
+    return (market_open_loc * minutes_per_day) + delta
+
+def find_last_traded_position_internal(
+        ndarray[long_t, ndim=1] market_opens,
+        ndarray[long_t, ndim=1] market_closes,
+        long_t end_minute,
+        long_t start_minute,
+        volumes,
+        short minutes_per_day):
+
+    """
+    Finds the position of the last traded minute for the given volumes array.
+
+    Parameters
+    ----------
+    market_opens: numpy array of ints
+        Market opens, in minute epoch values.
+
+    market_closes: numpy array of ints
+        Market closes, in minute epoch values.
+
+    end_minute: int
+        The minute from which to start looking backwards, as a minute epoch.
+
+    start_minute: int
+        The asset's start date, as a minute epoch.  Acts as the bottom limit of
+        how far we can look backwards.
+
+    volumes: bcolz carray
+        The volume history for the given asset.
+
+    minutes_per_day: int
+        The number of minutes per day (e.g. 390 for NYSE).
+
+    Returns
+    -------
+    int: The position of the last traded minute, starting from `minute_val`
+    """
+    cdef Py_ssize_t minute_pos, current_minute, q
+
+    minute_pos = int_min(
+        find_position_of_minute(market_opens, market_closes, end_minute,
+                                minutes_per_day, True),
+        len(volumes) - 1
+    )
+
+    while minute_pos >= 0:
+        current_minute = minute_value(
+            market_opens, minute_pos, minutes_per_day
+        )
+
+        q = cython.cdiv(minute_pos, minutes_per_day)
+        if current_minute > market_closes[q]:
+            minute_pos = find_position_of_minute(market_opens,
+                                                 market_closes,
+                                                 market_closes[q],
+                                                 minutes_per_day,
+                                                 False)
+            continue
+
+        if current_minute < start_minute:
+            return -1
+
+        if volumes[minute_pos] != 0:
+            return minute_pos
+
+        minute_pos -= 1
+
+    # we've gone to the beginning of this asset's range, and still haven't
+    # found a trade event
+    return -1

catalyst/data/_resample.pyx

@@ -0,0 +1,116 @@
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from cython cimport boundscheck, wraparound
+from numpy import finfo, float64, nan, isnan
+from numpy cimport intp_t, float64_t, uint32_t
+
+@boundscheck(False)
+@wraparound(False)
+cpdef void _minute_to_session_open(intp_t[:] close_locs,
+                                   float64_t[:] data,
+                                   float64_t[:] out):
+    cdef intp_t i, close_loc, loc = 0
+    cdef float64_t val
+    for i, close_loc in enumerate(close_locs):
+        val = nan
+        # Start by getting the price value at the opening minute of each day.
+        # If the value is NaN, continue looking forward until we either find a
+        # valid value or reach the closing minute, at which point the value is
+        # just kept as a NaN. We increment 'loc' after obtaining the value to
+        # ensure we do not reach an out of bounds index.
+        while isnan(val) and loc <= close_loc:
+            val = data[loc]
+            loc += 1
+        out[i] = val
+        loc = close_loc + 1
+
+
+@boundscheck(False)
+@wraparound(False)
+cpdef void _minute_to_session_high(intp_t[:] close_locs,
+                                   float64_t[:] data,
+                                   float64_t[:] out):
+    cdef intp_t i, close_loc, loc = 0
+    cdef float64_t val
+    for i, close_loc in enumerate(close_locs):
+        val = -1
+        while loc <= close_loc:
+            val = max(val, data[loc])
+            loc += 1
+        if val == -1:
+            val = nan
+        out[i] = val
+        loc = close_loc + 1
+
+
+@boundscheck(False)
+@wraparound(False)
+cpdef void _minute_to_session_low(intp_t[:] close_locs,
+                                   float64_t[:] data,
+                                   float64_t[:] out):
+    cdef intp_t i, close_loc, loc = 0
+    cdef float64_t val
+    cdef float64_t max_float = finfo(float64).max
+    for i, close_loc in enumerate(close_locs):
+        val = max_float
+        while loc <= close_loc:
+            val = min(val, data[loc])
+            loc += 1
+        if val == max_float:
+            val = nan
+        out[i] = val
+        loc = close_loc + 1
+
+
+@boundscheck(False)
+@wraparound(False)
+cpdef void _minute_to_session_close(intp_t[:] close_locs,
+                                    float64_t[:] data,
+                                    float64_t[:] out):
+    cdef intp_t i, prev_close_loc, loc = 0
+    cdef float64_t val
+    num_out = len(out)
+    for i in range(num_out - 1, -1, -1):
+        if i > 0:
+            prev_close_loc = close_locs[i - 1]
+        else:
+            prev_close_loc = -1
+        loc = close_locs[i]
+        val = nan
+        # Start by getting the price value at the closing minute of each day.
+        # If the value is NaN, continue looking back until we either find a
+        # valid value or reach the closing minute of the previous day, at which
+        # point the value is just kept as a NaN. We decrement 'loc' after
+        # obtaining the value to ensure we do not reach a negative index.
+        while isnan(val) and loc > prev_close_loc:
+            val = data[loc]
+            loc -= 1
+        out[i] = val
+
+
+@boundscheck(False)
+@wraparound(False)
+cpdef void _minute_to_session_volume(intp_t[:] close_locs,
+                                     uint32_t[:] data,
+                                     uint32_t[:] out):
+    cdef intp_t i, close_loc, loc = 0
+    cdef uint32_t val
+    loc = 0
+    for i, close_loc in enumerate(close_locs):
+        val = 0
+        while loc <= close_loc:
+            val += data[loc]
+            loc += 1
+        out[i] = val
+        loc = close_loc + 1

catalyst/data/bar_reader.py

@@ -0,0 +1,138 @@
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from abc import ABCMeta, abstractmethod, abstractproperty
+from six import with_metaclass
+
+
+class NoDataOnDate(Exception):
+    """
+    Raised when a spot price cannot be found for the sid and date.
+    """
+    pass
+
+
+class NoDataBeforeDate(NoDataOnDate):
+    pass
+
+
+class NoDataAfterDate(NoDataOnDate):
+    pass
+
+
+class BarReader(with_metaclass(ABCMeta, object)):
+    @abstractproperty
+    def data_frequency(self):
+        pass
+
+    @abstractmethod
+    def load_raw_arrays(self, columns, start_date, end_date, assets):
+        """
+        Parameters
+        ----------
+        fields : list of str
+           'open', 'high', 'low', 'close', or 'volume'
+        start_dt: Timestamp
+           Beginning of the window range.
+        end_dt: Timestamp
+           End of the window range.
+        sids : list of int
+           The asset identifiers in the window.
+
+        Returns
+        -------
+        list of np.ndarray
+            A list with an entry per field of ndarrays with shape
+            (minutes in range, sids) with a dtype of float64, containing the
+            values for the respective field over start and end dt range.
+        """
+        pass
+
+    @abstractproperty
+    def last_available_dt(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The last session for which the reader can provide data.
+        """
+        pass
+
+    @abstractproperty
+    def trading_calendar(self):
+        """
+        Returns the catalyst.utils.calendar.trading_calendar used to read
+        the data.  Can be None (if the writer didn't specify it).
+        """
+        pass
+
+    @abstractproperty
+    def first_trading_day(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The first trading day (session) for which the reader can provide
+            data.
+        """
+        pass
+
+    @abstractmethod
+    def get_value(self, sid, dt, field):
+        """
+        Retrieve the value at the given coordinates.
+
+        Parameters
+        ----------
+        sid : int
+            The asset identifier.
+        dt : pd.Timestamp
+            The timestamp for the desired data point.
+        field : string
+            The OHLVC name for the desired data point.
+
+        Returns
+        -------
+        value : float|int
+            The value at the given coordinates, ``float`` for OHLC, ``int``
+            for 'volume'.
+
+        Raises
+        ------
+        NoDataOnDate
+            If the given dt is not a valid market minute (in minute mode) or
+            session (in daily mode) according to this reader's tradingcalendar.
+        """
+        pass
+
+    @abstractmethod
+    def get_last_traded_dt(self, asset, dt):
+        """
+        Get the latest minute on or before ``dt`` in which ``asset`` traded.
+
+        If there are no trades on or before ``dt``, returns ``pd.NaT``.
+
+        Parameters
+        ----------
+        asset : catalyst.asset.Asset
+            The asset for which to get the last traded minute.
+        dt : pd.Timestamp
+            The minute at which to start searching for the last traded minute.
+
+        Returns
+        -------
+        last_traded : pd.Timestamp
+            The dt of the last trade for the given asset, using the input
+            dt as a vantage point.
+        """
+        pass

catalyst/data/benchmarks.py

@@ -0,0 +1,61 @@
+#
+# Copyright 2013 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+import pandas as pd
+
+import pandas_datareader.data as pd_reader
+
+
+def get_benchmark_returns(symbol, first_date, last_date):
+    """
+    Get a Series of benchmark returns from Google associated with `symbol`.
+    Default is `SPY`.
+
+    Parameters
+    ----------
+    symbol : str
+        Benchmark symbol for which we're getting the returns.
+    first_date : pd.Timestamp
+        First date for which we want to get data.
+    last_date : pd.Timestamp
+        Last date for which we want to get data.
+
+    The furthest date that Google goes back to is 1993-02-01. It has missing
+    data for 2008-12-15, 2009-08-11, and 2012-02-02, so we add data for the
+    dates for which Google is missing data.
+
+    We're also limited to 4000 days worth of data per request. If we make a
+    request for data that extends past 4000 trading days, we'll still only
+    receive 4000 days of data.
+
+    first_date is **not** included because we need the close from day N - 1 to
+    compute the returns for day N.
+    """
+    data = pd_reader.DataReader(
+        symbol,
+        'google',
+        first_date,
+        last_date
+    )
+
+    data = data['Close']
+
+    data[pd.Timestamp('2008-12-15')] = np.nan
+    data[pd.Timestamp('2009-08-11')] = np.nan
+    data[pd.Timestamp('2012-02-02')] = np.nan
+
+    data = data.fillna(method='ffill')
+
+    return data.sort_index().tz_localize('UTC').pct_change(1).iloc[1:]

catalyst/data/bundles/__init__.py

@@ -0,0 +1,29 @@
+# These imports are necessary to force module-scope register calls to happen.
+from . import quandl  # noqa
+from .core import (
+    UnknownBundle,
+    bundles,
+    clean,
+    from_bundle_ingest_dirname,
+    ingest,
+    ingestions_for_bundle,
+    load,
+    register,
+    to_bundle_ingest_dirname,
+    unregister,
+)
+from .yahoo import yahoo_equities
+
+__all__ = [
+    'UnknownBundle',
+    'bundles',
+    'clean',
+    'from_bundle_ingest_dirname',
+    'ingest',
+    'ingestions_for_bundle',
+    'load',
+    'register',
+    'to_bundle_ingest_dirname',
+    'unregister',
+    'yahoo_equities',
+]

catalyst/data/bundles/core.py

@@ -0,0 +1,622 @@
+from collections import namedtuple
+import errno
+import os
+import shutil
+import warnings
+
+from contextlib2 import ExitStack
+import click
+import pandas as pd
+from toolz import curry, complement, take
+
+from ..us_equity_pricing import (
+    BcolzDailyBarReader,
+    BcolzDailyBarWriter,
+    SQLiteAdjustmentReader,
+    SQLiteAdjustmentWriter,
+)
+from ..minute_bars import (
+    BcolzMinuteBarReader,
+    BcolzMinuteBarWriter,
+)
+from catalyst.assets import AssetDBWriter, AssetFinder, ASSET_DB_VERSION
+from catalyst.assets.asset_db_migrations import downgrade
+from catalyst.utils.cache import (
+    dataframe_cache,
+    working_dir,
+    working_file,
+)
+from catalyst.utils.compat import mappingproxy
+from catalyst.utils.input_validation import ensure_timestamp, optionally
+import catalyst.utils.paths as pth
+from catalyst.utils.preprocess import preprocess
+from catalyst.utils.calendars import get_calendar
+
+
+def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
+    return pth.data_path(
+        asset_db_relative(bundle_name, timestr, environ, db_version),
+        environ=environ,
+    )
+
+
+def minute_equity_path(bundle_name, timestr, environ=None):
+    return pth.data_path(
+        minute_equity_relative(bundle_name, timestr, environ),
+        environ=environ,
+    )
+
+
+def daily_equity_path(bundle_name, timestr, environ=None):
+    return pth.data_path(
+        daily_equity_relative(bundle_name, timestr, environ),
+        environ=environ,
+    )
+
+
+def adjustment_db_path(bundle_name, timestr, environ=None):
+    return pth.data_path(
+        adjustment_db_relative(bundle_name, timestr, environ),
+        environ=environ,
+    )
+
+
+def cache_path(bundle_name, environ=None):
+    return pth.data_path(
+        cache_relative(bundle_name, environ),
+        environ=environ,
+    )
+
+
+def adjustment_db_relative(bundle_name, timestr, environ=None):
+    return bundle_name, timestr, 'adjustments.sqlite'
+
+
+def cache_relative(bundle_name, timestr, environ=None):
+    return bundle_name, '.cache'
+
+
+def daily_equity_relative(bundle_name, timestr, environ=None):
+    return bundle_name, timestr, 'daily_equities.bcolz'
+
+
+def minute_equity_relative(bundle_name, timestr, environ=None):
+    return bundle_name, timestr, 'minute_equities.bcolz'
+
+
+def asset_db_relative(bundle_name, timestr, environ=None, db_version=None):
+    db_version = ASSET_DB_VERSION if db_version is None else db_version
+
+    return bundle_name, timestr, 'assets-%d.sqlite' % db_version
+
+
+def to_bundle_ingest_dirname(ts):
+    """Convert a pandas Timestamp into the name of the directory for the
+    ingestion.
+
+    Parameters
+    ----------
+    ts : pandas.Timestamp
+        The time of the ingestions
+
+    Returns
+    -------
+    name : str
+        The name of the directory for this ingestion.
+    """
+    return ts.isoformat().replace(':', ';')
+
+
+def from_bundle_ingest_dirname(cs):
+    """Read a bundle ingestion directory name into a pandas Timestamp.
+
+    Parameters
+    ----------
+    cs : str
+        The name of the directory.
+
+    Returns
+    -------
+    ts : pandas.Timestamp
+        The time when this ingestion happened.
+    """
+    return pd.Timestamp(cs.replace(';', ':'))
+
+
+def ingestions_for_bundle(bundle, environ=None):
+    return sorted(
+        (from_bundle_ingest_dirname(ing)
+         for ing in os.listdir(pth.data_path([bundle], environ))
+         if not pth.hidden(ing)),
+        reverse=True,
+    )
+
+
+RegisteredBundle = namedtuple(
+    'RegisteredBundle',
+    ['calendar_name',
+     'start_session',
+     'end_session',
+     'minutes_per_day',
+     'ingest',
+     'create_writers']
+)
+
+BundleData = namedtuple(
+    'BundleData',
+    'asset_finder equity_minute_bar_reader equity_daily_bar_reader '
+    'adjustment_reader',
+)
+
+BundleCore = namedtuple(
+    'BundleCore',
+    'bundles register unregister ingest load clean',
+)
+
+
+class UnknownBundle(click.ClickException, LookupError):
+    """Raised if no bundle with the given name was registered.
+    """
+    exit_code = 1
+
+    def __init__(self, name):
+        super(UnknownBundle, self).__init__(
+            'No bundle registered with the name %r' % name,
+        )
+        self.name = name
+
+    def __str__(self):
+        return self.message
+
+
+class BadClean(click.ClickException, ValueError):
+    """Exception indicating that an invalid argument set was passed to
+    ``clean``.
+
+    Parameters
+    ----------
+    before, after, keep_last : any
+        The bad arguments to ``clean``.
+
+    See Also
+    --------
+    clean
+    """
+    def __init__(self, before, after, keep_last):
+        super(BadClean, self).__init__(
+            'Cannot pass a combination of `before` and `after` with'
+            '`keep_last`. Got: before=%r, after=%r, keep_n=%r\n' % (
+                before,
+                after,
+                keep_last,
+            ),
+        )
+
+    def __str__(self):
+        return self.message
+
+
+def _make_bundle_core():
+    """Create a family of data bundle functions that read from the same
+    bundle mapping.
+
+    Returns
+    -------
+    bundles : mappingproxy
+        The mapping of bundles to bundle payloads.
+    register : callable
+        The function which registers new bundles in the ``bundles`` mapping.
+    unregister : callable
+        The function which deregisters bundles from the ``bundles`` mapping.
+    ingest : callable
+        The function which downloads and write data for a given data bundle.
+    load : callable
+        The function which loads the ingested bundles back into memory.
+    clean : callable
+        The function which cleans up data written with ``ingest``.
+    """
+    _bundles = {}  # the registered bundles
+    # Expose _bundles through a proxy so that users cannot mutate this
+    # accidentally. Users may go through `register` to update this which will
+    # warn when trampling another bundle.
+    bundles = mappingproxy(_bundles)
+
+    @curry
+    def register(name,
+                 f,
+                 calendar_name='NYSE',
+                 start_session=None,
+                 end_session=None,
+                 minutes_per_day=390,
+                 create_writers=True):
+        """Register a data bundle ingest function.
+
+        Parameters
+        ----------
+        name : str
+            The name of the bundle.
+        f : callable
+            The ingest function. This function will be passed:
+
+              environ : mapping
+                  The environment this is being run with.
+              asset_db_writer : AssetDBWriter
+                  The asset db writer to write into.
+              minute_bar_writer : BcolzMinuteBarWriter
+                  The minute bar writer to write into.
+              daily_bar_writer : BcolzDailyBarWriter
+                  The daily bar writer to write into.
+              adjustment_writer : SQLiteAdjustmentWriter
+                  The adjustment db writer to write into.
+              calendar : catalyst.utils.calendars.TradingCalendar
+                  The trading calendar to ingest for.
+              start_session : pd.Timestamp
+                  The first session of data to ingest.
+              end_session : pd.Timestamp
+                  The last session of data to ingest.
+              cache : DataFrameCache
+                  A mapping object to temporarily store dataframes.
+                  This should be used to cache intermediates in case the load
+                  fails. This will be automatically cleaned up after a
+                  successful load.
+              show_progress : bool
+                  Show the progress for the current load where possible.
+        calendar_name : str, optional
+            The name of a calendar used to align bundle data.
+            Default is 'NYSE'.
+        start_session : pd.Timestamp, optional
+            The first session for which we want data. If not provided,
+            or if the date lies outside the range supported by the
+            calendar, the first_session of the calendar is used.
+        end_session : pd.Timestamp, optional
+            The last session for which we want data. If not provided,
+            or if the date lies outside the range supported by the
+            calendar, the last_session of the calendar is used.
+        minutes_per_day : int, optional
+            The number of minutes in each normal trading day.
+        create_writers : bool, optional
+            Should the ingest machinery create the writers for the ingest
+            function. This can be disabled as an optimization for cases where
+            they are not needed, like the ``quantopian-quandl`` bundle.
+
+        Notes
+        -----
+        This function my be used as a decorator, for example:
+
+        .. code-block:: python
+
+           @register('quandl')
+           def quandl_ingest_function(...):
+               ...
+
+        See Also
+        --------
+        catalyst.data.bundles.bundles
+        """
+        if name in bundles:
+            warnings.warn(
+                'Overwriting bundle with name %r' % name,
+                stacklevel=3,
+            )
+
+        # NOTE: We don't eagerly compute calendar values here because
+        # `register` is called at module scope in catalyst, and creating a
+        # calendar currently takes between 0.5 and 1 seconds, which causes a
+        # noticeable delay on the catalyst CLI.
+        _bundles[name] = RegisteredBundle(
+            calendar_name=calendar_name,
+            start_session=start_session,
+            end_session=end_session,
+            minutes_per_day=minutes_per_day,
+            ingest=f,
+            create_writers=create_writers,
+        )
+        return f
+
+    def unregister(name):
+        """Unregister a bundle.
+
+        Parameters
+        ----------
+        name : str
+            The name of the bundle to unregister.
+
+        Raises
+        ------
+        UnknownBundle
+            Raised when no bundle has been registered with the given name.
+
+        See Also
+        --------
+        catalyst.data.bundles.bundles
+        """
+        try:
+            del _bundles[name]
+        except KeyError:
+            raise UnknownBundle(name)
+
+    def ingest(name,
+               environ=os.environ,
+               timestamp=None,
+               assets_versions=(),
+               show_progress=False):
+        """Ingest data for a given bundle.
+
+        Parameters
+        ----------
+        name : str
+            The name of the bundle.
+        environ : mapping, optional
+            The environment variables. By default this is os.environ.
+        timestamp : datetime, optional
+            The timestamp to use for the load.
+            By default this is the current time.
+        assets_versions : Iterable[int], optional
+            Versions of the assets db to which to downgrade.
+        show_progress : bool, optional
+            Tell the ingest function to display the progress where possible.
+        """
+        try:
+            bundle = bundles[name]
+        except KeyError:
+            raise UnknownBundle(name)
+
+        calendar = get_calendar(bundle.calendar_name)
+
+        start_session = bundle.start_session
+        end_session = bundle.end_session
+
+        if start_session is None or start_session < calendar.first_session:
+            start_session = calendar.first_session
+
+        if end_session is None or end_session > calendar.last_session:
+            end_session = calendar.last_session
+
+        if timestamp is None:
+            timestamp = pd.Timestamp.utcnow()
+        timestamp = timestamp.tz_convert('utc').tz_localize(None)
+
+        timestr = to_bundle_ingest_dirname(timestamp)
+        cachepath = cache_path(name, environ=environ)
+        pth.ensure_directory(pth.data_path([name, timestr], environ=environ))
+        pth.ensure_directory(cachepath)
+        with dataframe_cache(cachepath, clean_on_failure=False) as cache, \
+                ExitStack() as stack:
+            # we use `cleanup_on_failure=False` so that we don't purge the
+            # cache directory if the load fails in the middle
+            if bundle.create_writers:
+                wd = stack.enter_context(working_dir(
+                    pth.data_path([], environ=environ))
+                )
+                daily_bars_path = wd.ensure_dir(
+                    *daily_equity_relative(
+                        name, timestr, environ=environ,
+                    )
+                )
+                daily_bar_writer = BcolzDailyBarWriter(
+                    daily_bars_path,
+                    calendar,
+                    start_session,
+                    end_session,
+                )
+                # Do an empty write to ensure that the daily ctables exist
+                # when we create the SQLiteAdjustmentWriter below. The
+                # SQLiteAdjustmentWriter needs to open the daily ctables so
+                # that it can compute the adjustment ratios for the dividends.
+
+                daily_bar_writer.write(())
+                minute_bar_writer = BcolzMinuteBarWriter(
+                    wd.ensure_dir(*minute_equity_relative(
+                        name, timestr, environ=environ)
+                    ),
+                    calendar,
+                    start_session,
+                    end_session,
+                    minutes_per_day=bundle.minutes_per_day,
+                )
+                assets_db_path = wd.getpath(*asset_db_relative(
+                    name, timestr, environ=environ,
+                ))
+                asset_db_writer = AssetDBWriter(assets_db_path)
+
+                adjustment_db_writer = stack.enter_context(
+                    SQLiteAdjustmentWriter(
+                        wd.getpath(*adjustment_db_relative(
+                            name, timestr, environ=environ)),
+                        BcolzDailyBarReader(daily_bars_path),
+                        calendar.all_sessions,
+                        overwrite=True,
+                    )
+                )
+            else:
+                daily_bar_writer = None
+                minute_bar_writer = None
+                asset_db_writer = None
+                adjustment_db_writer = None
+                if assets_versions:
+                    raise ValueError('Need to ingest a bundle that creates '
+                                     'writers in order to downgrade the assets'
+                                     ' db.')
+            bundle.ingest(
+                environ,
+                asset_db_writer,
+                minute_bar_writer,
+                daily_bar_writer,
+                adjustment_db_writer,
+                calendar,
+                start_session,
+                end_session,
+                cache,
+                show_progress,
+                pth.data_path([name, timestr], environ=environ),
+            )
+
+            for version in sorted(set(assets_versions), reverse=True):
+                version_path = wd.getpath(*asset_db_relative(
+                    name, timestr, environ=environ, db_version=version,
+                ))
+                with working_file(version_path) as wf:
+                    shutil.copy2(assets_db_path, wf.path)
+                    downgrade(wf.path, version)
+
+    def most_recent_data(bundle_name, timestamp, environ=None):
+        """Get the path to the most recent data after ``date``for the
+        given bundle.
+
+        Parameters
+        ----------
+        bundle_name : str
+            The name of the bundle to lookup.
+        timestamp : datetime
+            The timestamp to begin searching on or before.
+        environ : dict, optional
+            An environment dict to forward to catalyst_root.
+        """
+        if bundle_name not in bundles:
+            raise UnknownBundle(bundle_name)
+
+        try:
+            candidates = os.listdir(
+                pth.data_path([bundle_name], environ=environ),
+            )
+            return pth.data_path(
+                [bundle_name,
+                 max(
+                     filter(complement(pth.hidden), candidates),
+                     key=from_bundle_ingest_dirname,
+                 )],
+                environ=environ,
+            )
+        except (ValueError, OSError) as e:
+            if getattr(e, 'errno', errno.ENOENT) != errno.ENOENT:
+                raise
+            raise ValueError(
+                'no data for bundle {bundle!r} on or before {timestamp}\n'
+                'maybe you need to run: $ catalyst ingest -b {bundle}'.format(
+                    bundle=bundle_name,
+                    timestamp=timestamp,
+                ),
+            )
+
+    def load(name, environ=os.environ, timestamp=None):
+        """Loads a previously ingested bundle.
+
+        Parameters
+        ----------
+        name : str
+            The name of the bundle.
+        environ : mapping, optional
+            The environment variables. Defaults of os.environ.
+        timestamp : datetime, optional
+            The timestamp of the data to lookup.
+            Defaults to the current time.
+
+        Returns
+        -------
+        bundle_data : BundleData
+            The raw data readers for this bundle.
+        """
+        if timestamp is None:
+            timestamp = pd.Timestamp.utcnow()
+        timestr = most_recent_data(name, timestamp, environ=environ)
+        return BundleData(
+            asset_finder=AssetFinder(
+                asset_db_path(name, timestr, environ=environ),
+            ),
+            equity_minute_bar_reader=BcolzMinuteBarReader(
+                minute_equity_path(name, timestr, environ=environ),
+            ),
+            equity_daily_bar_reader=BcolzDailyBarReader(
+                daily_equity_path(name, timestr, environ=environ),
+            ),
+            adjustment_reader=SQLiteAdjustmentReader(
+                adjustment_db_path(name, timestr, environ=environ),
+            ),
+        )
+
+    @preprocess(
+        before=optionally(ensure_timestamp),
+        after=optionally(ensure_timestamp),
+    )
+    def clean(name,
+              before=None,
+              after=None,
+              keep_last=None,
+              environ=os.environ):
+        """Clean up data that was created with ``ingest`` or
+        ``$ python -m catalyst ingest``
+
+        Parameters
+        ----------
+        name : str
+            The name of the bundle to remove data for.
+        before : datetime, optional
+            Remove data ingested before this date.
+            This argument is mutually exclusive with: keep_last
+        after : datetime, optional
+            Remove data ingested after this date.
+            This argument is mutually exclusive with: keep_last
+        keep_last : int, optional
+            Remove all but the last ``keep_last`` ingestions.
+            This argument is mutually exclusive with:
+              before
+              after
+        environ : mapping, optional
+            The environment variables. Defaults of os.environ.
+
+        Returns
+        -------
+        cleaned : set[str]
+            The names of the runs that were removed.
+
+        Raises
+        ------
+        BadClean
+            Raised when ``before`` and or ``after`` are passed with
+            ``keep_last``. This is a subclass of ``ValueError``.
+        """
+        try:
+            all_runs = sorted(
+                filter(
+                    complement(pth.hidden),
+                    os.listdir(pth.data_path([name], environ=environ)),
+                ),
+                key=from_bundle_ingest_dirname,
+            )
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            raise UnknownBundle(name)
+        if ((before is not None or after is not None) and
+                keep_last is not None):
+            raise BadClean(before, after, keep_last)
+
+        if keep_last is None:
+            def should_clean(name):
+                dt = from_bundle_ingest_dirname(name)
+                return (
+                    (before is not None and dt < before) or
+                    (after is not None and dt > after)
+                )
+
+        elif keep_last >= 0:
+            last_n_dts = set(take(keep_last, reversed(all_runs)))
+
+            def should_clean(name):
+                return name not in last_n_dts
+        else:
+            raise BadClean(before, after, keep_last)
+
+        cleaned = set()
+        for run in all_runs:
+            if should_clean(run):
+                path = pth.data_path([name, run], environ=environ)
+                shutil.rmtree(path)
+                cleaned.add(path)
+
+        return cleaned
+
+    return BundleCore(bundles, register, unregister, ingest, load, clean)
+
+
+bundles, register, unregister, ingest, load, clean = _make_bundle_core()

catalyst/data/bundles/quandl.py

@@ -0,0 +1,444 @@
+"""
+Module for building a complete daily dataset from Quandl's WIKI dataset.
+"""
+from io import BytesIO
+from itertools import count
+import tarfile
+from time import time, sleep
+
+from click import progressbar
+from logbook import Logger
+import pandas as pd
+import requests
+from six.moves.urllib.parse import urlencode
+
+from catalyst.utils.calendars import register_calendar_alias
+from catalyst.utils.cli import maybe_show_progress
+
+from . import core as bundles
+
+log = Logger(__name__)
+seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
+# Invalid symbols that quandl has had in its metadata:
+excluded_symbols = frozenset({'TEST123456789'})
+
+
+def _fetch_raw_metadata(api_key, cache, retries, environ):
+    """Generator that yields each page of data from the metadata endpoint
+    as a dataframe.
+    """
+    for page_number in count(1):
+        key = 'metadata-page-%d' % page_number
+        try:
+            raw = cache[key]
+        except KeyError:
+            for _ in range(retries):
+                try:
+                    raw = pd.read_csv(
+                        format_metadata_url(api_key, page_number),
+                        parse_dates=[
+                            'oldest_available_date',
+                            'newest_available_date',
+                        ],
+                        usecols=[
+                            'dataset_code',
+                            'name',
+                            'oldest_available_date',
+                            'newest_available_date',
+                        ],
+                    )
+                    break
+                except ValueError:
+                    # when we are past the last page we will get a value
+                    # error because there will be no columns
+                    raw = pd.DataFrame([])
+                    break
+                except Exception:
+                    pass
+            else:
+                raise ValueError(
+                    'Failed to download metadata page %d after %d'
+                    ' attempts.' % (page_number, retries),
+                )
+
+            cache[key] = raw
+
+        if raw.empty:
+            # use the empty dataframe to signal completion
+            break
+        yield raw
+
+
+def fetch_symbol_metadata_frame(api_key,
+                                cache,
+                                retries=5,
+                                environ=None,
+                                show_progress=False):
+    """
+    Download Quandl symbol metadata.
+
+    Parameters
+    ----------
+    api_key : str
+        The quandl api key to use. If this is None then no api key will be
+        sent.
+    cache : DataFrameCache
+        The cache to use for persisting the intermediate data.
+    retries : int, optional
+        The number of times to retry each request before failing.
+    environ : mapping[str -> str], optional
+        The environment to use to find the catalyst home. By default this
+        is ``os.environ``.
+    show_progress : bool, optional
+        Show a progress bar for the download of this data.
+
+    Returns
+    -------
+    metadata_frame : pd.DataFrame
+        A dataframe with the following columns:
+          symbol: the asset's symbol
+          name: the full name of the asset
+          start_date: the first date of data for this asset
+          end_date: the last date of data for this asset
+          auto_close_date: end_date + one day
+          exchange: the exchange for the asset; this is always 'quandl'
+        The index of the dataframe will be used for symbol->sid mappings but
+        otherwise does not have specific meaning.
+    """
+    raw_iter = _fetch_raw_metadata(api_key, cache, retries, environ)
+
+    def item_show_func(_, _it=iter(count())):
+        'Downloading page: %d' % next(_it)
+
+    with maybe_show_progress(raw_iter,
+                             show_progress,
+                             item_show_func=item_show_func,
+                             label='Downloading WIKI metadata: ') as blocks:
+        data = pd.concat(blocks, ignore_index=True).rename(columns={
+            'dataset_code': 'symbol',
+            'name': 'asset_name',
+            'oldest_available_date': 'start_date',
+            'newest_available_date': 'end_date',
+        }).sort_values('symbol')
+
+    data = data[~data.symbol.isin(excluded_symbols)]
+    # cut out all the other stuff in the name column
+    # we need to escape the paren because it is actually splitting on a regex
+    data.asset_name = data.asset_name.str.split(r' \(', 1).str.get(0)
+    data['exchange'] = 'QUANDL'
+    data['auto_close_date'] = data['end_date'] + pd.Timedelta(days=1)
+    return data
+
+
+def format_metadata_url(api_key, page_number):
+    """Build the query RL for the quandl WIKI metadata.
+    """
+    query_params = [
+        ('per_page', '100'),
+        ('sort_by', 'id'),
+        ('page', str(page_number)),
+        ('database_code', 'WIKI'),
+    ]
+    if api_key is not None:
+        query_params = [('api_key', api_key)] + query_params
+    return (
+        'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
+    )
+
+
+def format_wiki_url(api_key, symbol, start_date, end_date):
+    """
+    Build a query URL for a quandl WIKI dataset.
+    """
+    query_params = [
+        ('start_date', start_date.strftime('%Y-%m-%d')),
+        ('end_date', end_date.strftime('%Y-%m-%d')),
+        ('order', 'asc'),
+    ]
+    if api_key is not None:
+        query_params = [('api_key', api_key)] + query_params
+
+    return (
+        "https://www.quandl.com/api/v3/datasets/WIKI/"
+        "{symbol}.csv?{query}".format(
+            symbol=symbol,
+            query=urlencode(query_params),
+        )
+    )
+
+
+def fetch_single_equity(api_key,
+                        symbol,
+                        start_date,
+                        end_date,
+                        retries=5):
+    """
+    Download data for a single equity.
+    """
+    for _ in range(retries):
+        try:
+            return pd.read_csv(
+                format_wiki_url(api_key, symbol, start_date, end_date),
+                parse_dates=['Date'],
+                index_col='Date',
+                usecols=[
+                    'Open',
+                    'High',
+                    'Low',
+                    'Close',
+                    'Volume',
+                    'Date',
+                    'Ex-Dividend',
+                    'Split Ratio',
+                ],
+                na_values=['NA'],
+            ).rename(columns={
+                'Open': 'open',
+                'High': 'high',
+                'Low': 'low',
+                'Close': 'close',
+                'Volume': 'volume',
+                'Date': 'date',
+                'Ex-Dividend': 'ex_dividend',
+                'Split Ratio': 'split_ratio',
+            })
+        except Exception:
+            log.exception("Exception raised reading Quandl data. Retrying.")
+    else:
+        raise ValueError(
+            "Failed to download data for %r after %d attempts." % (
+                symbol, retries
+            )
+        )
+
+
+def _update_splits(splits, asset_id, raw_data):
+    split_ratios = raw_data.split_ratio
+    df = pd.DataFrame({'ratio': 1 / split_ratios[split_ratios != 1]})
+    df.index.name = 'effective_date'
+    df.reset_index(inplace=True)
+    df['sid'] = asset_id
+    splits.append(df)
+
+
+def _update_dividends(dividends, asset_id, raw_data):
+    divs = raw_data.ex_dividend
+    df = pd.DataFrame({'amount': divs[divs != 0]})
+    df.index.name = 'ex_date'
+    df.reset_index(inplace=True)
+    df['sid'] = asset_id
+    # we do not have this data in the WIKI dataset
+    df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
+    dividends.append(df)
+
+
+def gen_symbol_data(api_key,
+                    cache,
+                    symbol_map,
+                    calendar,
+                    start_session,
+                    end_session,
+                    splits,
+                    dividends,
+                    retries):
+    for asset_id, symbol in symbol_map.iteritems():
+        start_time = time()
+        try:
+            # see if we have this data cached.
+            raw_data = cache[symbol]
+            should_sleep = False
+        except KeyError:
+            # we need to fetch the data and then write it to our cache
+            raw_data = cache[symbol] = fetch_single_equity(
+                api_key,
+                symbol,
+                start_date=start_session,
+                end_date=end_session,
+            )
+            should_sleep = True
+
+        _update_splits(splits, asset_id, raw_data)
+        _update_dividends(dividends, asset_id, raw_data)
+
+        sessions = calendar.sessions_in_range(start_session, end_session)
+
+        raw_data = raw_data.reindex(
+            sessions.tz_localize(None),
+            copy=False,
+        ).fillna(0.0)
+        yield asset_id, raw_data
+
+        if should_sleep:
+            remaining = seconds_per_call - time() - start_time
+            if remaining > 0:
+                sleep(remaining)
+
+
+@bundles.register('quandl')
+def quandl_bundle(environ,
+                  asset_db_writer,
+                  minute_bar_writer,
+                  daily_bar_writer,
+                  adjustment_writer,
+                  calendar,
+                  start_session,
+                  end_session,
+                  cache,
+                  show_progress,
+                  output_dir):
+    """Build a catalyst data bundle from the Quandl WIKI dataset.
+    """
+    api_key = environ.get('QUANDL_API_KEY')
+    metadata = fetch_symbol_metadata_frame(
+        api_key,
+        cache=cache,
+        show_progress=show_progress,
+    )
+    symbol_map = metadata.symbol
+
+    # data we will collect in `gen_symbol_data`
+    splits = []
+    dividends = []
+
+    asset_db_writer.write(metadata)
+    daily_bar_writer.write(
+        gen_symbol_data(
+            api_key,
+            cache,
+            symbol_map,
+            calendar,
+            start_session,
+            end_session,
+            splits,
+            dividends,
+            environ.get('QUANDL_DOWNLOAD_ATTEMPTS', 5),
+        ),
+        show_progress=show_progress,
+    )
+    adjustment_writer.write(
+        splits=pd.concat(splits, ignore_index=True),
+        dividends=pd.concat(dividends, ignore_index=True),
+    )
+
+
+def download_with_progress(url, chunk_size, **progress_kwargs):
+    """
+    Download streaming data from a URL, printing progress information to the
+    terminal.
+
+    Parameters
+    ----------
+    url : str
+        A URL that can be understood by ``requests.get``.
+    chunk_size : int
+        Number of bytes to read at a time from requests.
+    **progress_kwargs
+        Forwarded to click.progressbar.
+
+    Returns
+    -------
+    data : BytesIO
+        A BytesIO containing the downloaded data.
+    """
+    resp = requests.get(url, stream=True)
+    resp.raise_for_status()
+
+    total_size = int(resp.headers['content-length'])
+    data = BytesIO()
+    with progressbar(length=total_size, **progress_kwargs) as pbar:
+        for chunk in resp.iter_content(chunk_size=chunk_size):
+            data.write(chunk)
+            pbar.update(len(chunk))
+
+    data.seek(0)
+    return data
+
+
+def download_without_progress(url):
+    """
+    Download data from a URL, returning a BytesIO containing the loaded data.
+
+    Parameters
+    ----------
+    url : str
+        A URL that can be understood by ``requests.get``.
+
+    Returns
+    -------
+    data : BytesIO
+        A BytesIO containing the downloaded data.
+    """
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return BytesIO(resp.content)
+
+
+QUANTOPIAN_QUANDL_URL = (
+    'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl'
+)
+ONE_MEGABYTE = 1024 * 1024
+
+
+@bundles.register('quantopian-quandl', create_writers=False)
+def quantopian_quandl_bundle(environ,
+                             asset_db_writer,
+                             minute_bar_writer,
+                             daily_bar_writer,
+                             adjustment_writer,
+                             calendar,
+                             start_session,
+                             end_session,
+                             cache,
+                             show_progress,
+                             output_dir):
+    if show_progress:
+        data = download_with_progress(
+            QUANTOPIAN_QUANDL_URL,
+            chunk_size=ONE_MEGABYTE,
+            label="Downloading Bundle: quantopian-quandl",
+        )
+    else:
+        data = download_without_progress(QUANTOPIAN_QUANDL_URL)
+
+    with tarfile.open('r', fileobj=data) as tar:
+        if show_progress:
+            print("Writing data to %s." % output_dir)
+        tar.extractall(output_dir)
+
+
+register_calendar_alias("QUANDL", "NYSE")
+
+CATALYST_URL = (
+    'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl'
+)
+
+@bundles.register(
+    'catalyst',
+    calendar_name='NYSE',
+    minutes_per_day=390,
+    create_writers=False,
+)
+def catalyst_bundle(environ,
+                    asset_db_writer,
+                    minute_bar_writer,
+                    daily_bar_writer,
+                    adjustment_writer,
+                    calendar,
+                    start_session,
+                    end_session,
+                    cache,
+                    show_progress,
+                    output_dir):
+    if show_progress:
+        data = download_with_progress(
+            CATALYST_URL,
+            chunk_size=ONE_MEGABYTE,
+            label="Downloading Bundle: catalyst",
+        )
+    else:
+        data = download_without_progress(CATALYST_URL)
+
+    with tarfile.open('r', fileobj=data) as tar:
+        if show_progress:
+            print("Writing data to %s." % output_dir)
+        tar.extractall(output_dir)

catalyst/data/bundles/yahoo.py

@@ -0,0 +1,203 @@
+import os
+
+import numpy as np
+import pandas as pd
+from pandas_datareader.data import DataReader
+import requests
+
+from catalyst.utils.calendars import register_calendar_alias
+from catalyst.utils.cli import maybe_show_progress
+from .core import register
+
+
+def _cachpath(symbol, type_):
+    return '-'.join((symbol.replace(os.path.sep, '_'), type_))
+
+
+def yahoo_equities(symbols, start=None, end=None):
+    """Create a data bundle ingest function from a set of symbols loaded from
+    yahoo.
+
+    Parameters
+    ----------
+    symbols : iterable[str]
+        The ticker symbols to load data for.
+    start : datetime, optional
+        The start date to query for. By default this pulls the full history
+        for the calendar.
+    end : datetime, optional
+        The end date to query for. By default this pulls the full history
+        for the calendar.
+
+    Returns
+    -------
+    ingest : callable
+        The bundle ingest function for the given set of symbols.
+
+    Examples
+    --------
+    This code should be added to ~/.catalyst/extension.py
+
+    .. code-block:: python
+
+       from catalyst.data.bundles import yahoo_equities, register
+
+       symbols = (
+           'AAPL',
+           'IBM',
+           'MSFT',
+       )
+       register('my_bundle', yahoo_equities(symbols))
+
+    Notes
+    -----
+    The sids for each symbol will be the index into the symbols sequence.
+    """
+    # strict this in memory so that we can reiterate over it
+    symbols = tuple(symbols)
+
+    def ingest(environ,
+               asset_db_writer,
+               minute_bar_writer,  # unused
+               daily_bar_writer,
+               adjustment_writer,
+               calendar,
+               start_session,
+               end_session,
+               cache,
+               show_progress,
+               output_dir,
+               # pass these as defaults to make them 'nonlocal' in py2
+               start=start,
+               end=end):
+        if start is None:
+            start = start_session
+        if end is None:
+            end = None
+
+        metadata = pd.DataFrame(np.empty(len(symbols), dtype=[
+            ('start_date', 'datetime64[ns]'),
+            ('end_date', 'datetime64[ns]'),
+            ('auto_close_date', 'datetime64[ns]'),
+            ('symbol', 'object'),
+        ]))
+
+        def _pricing_iter():
+            sid = 0
+            with maybe_show_progress(
+                    symbols,
+                    show_progress,
+                    label='Downloading Yahoo pricing data: ') as it, \
+                    requests.Session() as session:
+                for symbol in it:
+                    path = _cachpath(symbol, 'ohlcv')
+                    try:
+                        df = cache[path]
+                    except KeyError:
+                        df = cache[path] = DataReader(
+                            symbol,
+                            'yahoo',
+                            start,
+                            end,
+                            session=session,
+                        ).sort_index()
+
+                    # the start date is the date of the first trade and
+                    # the end date is the date of the last trade
+                    start_date = df.index[0]
+                    end_date = df.index[-1]
+                    # The auto_close date is the day after the last trade.
+                    ac_date = end_date + pd.Timedelta(days=1)
+                    metadata.iloc[sid] = start_date, end_date, ac_date, symbol
+
+                    df.rename(
+                        columns={
+                            'Open': 'open',
+                            'High': 'high',
+                            'Low': 'low',
+                            'Close': 'close',
+                            'Volume': 'volume',
+                        },
+                        inplace=True,
+                    )
+                    yield sid, df
+                    sid += 1
+
+        daily_bar_writer.write(_pricing_iter(), show_progress=show_progress)
+
+        symbol_map = pd.Series(metadata.symbol.index, metadata.symbol)
+
+        # Hardcode the exchange to "YAHOO" for all assets and (elsewhere)
+        # register "YAHOO" to resolve to the NYSE calendar, because these are
+        # all equities and thus can use the NYSE calendar.
+        metadata['exchange'] = "YAHOO"
+        asset_db_writer.write(equities=metadata)
+
+        adjustments = []
+        with maybe_show_progress(
+                symbols,
+                show_progress,
+                label='Downloading Yahoo adjustment data: ') as it, \
+                requests.Session() as session:
+            for symbol in it:
+                path = _cachpath(symbol, 'adjustment')
+                try:
+                    df = cache[path]
+                except KeyError:
+                    df = cache[path] = DataReader(
+                        symbol,
+                        'yahoo-actions',
+                        start,
+                        end,
+                        session=session,
+                    ).sort_index()
+
+                df['sid'] = symbol_map[symbol]
+                adjustments.append(df)
+
+        adj_df = pd.concat(adjustments)
+        adj_df.index.name = 'date'
+        adj_df.reset_index(inplace=True)
+
+        splits = adj_df[adj_df.action == 'SPLIT']
+        splits = splits.rename(
+            columns={'value': 'ratio', 'date': 'effective_date'},
+        )
+        splits.drop('action', axis=1, inplace=True)
+
+        dividends = adj_df[adj_df.action == 'DIVIDEND']
+        dividends = dividends.rename(
+            columns={'value': 'amount', 'date': 'ex_date'},
+        )
+        dividends.drop('action', axis=1, inplace=True)
+        # we do not have this data in the yahoo dataset
+        dividends['record_date'] = pd.NaT
+        dividends['declared_date'] = pd.NaT
+        dividends['pay_date'] = pd.NaT
+
+        adjustment_writer.write(splits=splits, dividends=dividends)
+
+    return ingest
+
+
+# bundle used when creating test data
+register(
+    '.test',
+    yahoo_equities(
+        (
+            'AMD',
+            'CERN',
+            'COST',
+            'DELL',
+            'GPS',
+            'INTC',
+            'MMM',
+            'AAPL',
+            'MSFT',
+        ),
+        pd.Timestamp('2004-01-02', tz='utc'),
+        pd.Timestamp('2015-01-01', tz='utc'),
+    ),
+)
+
+register_calendar_alias("YAHOO", "NYSE")

catalyst/data/continuous_future_reader.py

@@ -0,0 +1,359 @@
+import numpy as np
+import pandas as pd
+from catalyst.data.session_bars import SessionBarReader
+
+
+class ContinuousFutureSessionBarReader(SessionBarReader):
+
+    def __init__(self, bar_reader, roll_finders):
+        self._bar_reader = bar_reader
+        self._roll_finders = roll_finders
+
+    def load_raw_arrays(self, columns, start_date, end_date, assets):
+        """
+        Parameters
+        ----------
+        fields : list of str
+            'sid'
+        start_dt: Timestamp
+           Beginning of the window range.
+        end_dt: Timestamp
+           End of the window range.
+        sids : list of int
+           The asset identifiers in the window.
+
+        Returns
+        -------
+        list of np.ndarray
+            A list with an entry per field of ndarrays with shape
+            (minutes in range, sids) with a dtype of float64, containing the
+            values for the respective field over start and end dt range.
+        """
+        rolls_by_asset = {}
+        for asset in assets:
+            rf = self._roll_finders[asset.roll_style]
+            rolls_by_asset[asset] = rf.get_rolls(
+                asset.root_symbol, start_date, end_date, asset.offset)
+        num_sessions = len(
+            self.trading_calendar.sessions_in_range(start_date, end_date))
+        shape = num_sessions, len(assets)
+
+        results = []
+
+        tc = self._bar_reader.trading_calendar
+        sessions = tc.sessions_in_range(start_date, end_date)
+
+        # Get partitions
+        partitions_by_asset = {}
+        for asset in assets:
+            partitions = []
+            partitions_by_asset[asset] = partitions
+            rolls = rolls_by_asset[asset]
+            start = start_date
+            for roll in rolls:
+                sid, roll_date = roll
+                start_loc = sessions.get_loc(start)
+                if roll_date is not None:
+                    end = roll_date - sessions.freq
+                    end_loc = sessions.get_loc(end)
+                else:
+                    end = end_date
+                    end_loc = len(sessions) - 1
+                partitions.append((sid, start, end, start_loc, end_loc))
+                if roll[-1] is not None:
+                    start = sessions[end_loc + 1]
+
+        for column in columns:
+            if column != 'volume' and column != 'sid':
+                out = np.full(shape, np.nan)
+            else:
+                out = np.zeros(shape, dtype=np.int64)
+            for i, asset in enumerate(assets):
+                partitions = partitions_by_asset[asset]
+                for sid, start, end, start_loc, end_loc in partitions:
+                    if column != 'sid':
+                        result = self._bar_reader.load_raw_arrays(
+                            [column], start, end, [sid])[0][:, 0]
+                    else:
+                        result = int(sid)
+                    out[start_loc:end_loc + 1, i] = result
+            results.append(out)
+        return results
+
+    @property
+    def last_available_dt(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The last session for which the reader can provide data.
+        """
+        return self._bar_reader.last_available_dt
+
+    @property
+    def trading_calendar(self):
+        """
+        Returns the catalyst.utils.calendar.trading_calendar used to read
+        the data.  Can be None (if the writer didn't specify it).
+        """
+        return self._bar_reader.trading_calendar
+
+    @property
+    def first_trading_day(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The first trading day (session) for which the reader can provide
+            data.
+        """
+        return self._bar_reader.first_trading_day
+
+    def get_value(self, continuous_future, dt, field):
+        """
+        Retrieve the value at the given coordinates.
+
+        Parameters
+        ----------
+        sid : int
+            The asset identifier.
+        dt : pd.Timestamp
+            The timestamp for the desired data point.
+        field : string
+            The OHLVC name for the desired data point.
+
+        Returns
+        -------
+        value : float|int
+            The value at the given coordinates, ``float`` for OHLC, ``int``
+            for 'volume'.
+
+        Raises
+        ------
+        NoDataOnDate
+            If the given dt is not a valid market minute (in minute mode) or
+            session (in daily mode) according to this reader's tradingcalendar.
+        """
+        rf = self._roll_finders[continuous_future.roll_style]
+        sid = (rf.get_contract_center(continuous_future.root_symbol,
+                                      dt,
+                                      continuous_future.offset))
+        return self._bar_reader.get_value(sid, dt, field)
+
+    def get_last_traded_dt(self, asset, dt):
+        """
+        Get the latest minute on or before ``dt`` in which ``asset`` traded.
+
+        If there are no trades on or before ``dt``, returns ``pd.NaT``.
+
+        Parameters
+        ----------
+        asset : catalyst.asset.Asset
+            The asset for which to get the last traded minute.
+        dt : pd.Timestamp
+            The minute at which to start searching for the last traded minute.
+
+        Returns
+        -------
+        last_traded : pd.Timestamp
+            The dt of the last trade for the given asset, using the input
+            dt as a vantage point.
+        """
+        rf = self._roll_finders[asset.roll_style]
+        sid = (rf.get_contract_center(asset.root_symbol,
+                                      dt,
+                                      asset.offset))
+        if sid is None:
+            return pd.NaT
+        contract = rf.asset_finder.retrieve_asset(sid)
+        return self._bar_reader.get_last_traded_dt(contract, dt)
+
+    @property
+    def sessions(self):
+        """
+        Returns
+        -------
+        sessions : DatetimeIndex
+           All session labels (unionining the range for all assets) which the
+           reader can provide.
+        """
+        return self._bar_reader.sessions
+
+
+class ContinuousFutureMinuteBarReader(SessionBarReader):
+
+    def __init__(self, bar_reader, roll_finders):
+        self._bar_reader = bar_reader
+        self._roll_finders = roll_finders
+
+    def load_raw_arrays(self, columns, start_date, end_date, assets):
+        """
+        Parameters
+        ----------
+        fields : list of str
+           'open', 'high', 'low', 'close', or 'volume'
+        start_dt: Timestamp
+           Beginning of the window range.
+        end_dt: Timestamp
+           End of the window range.
+        sids : list of int
+           The asset identifiers in the window.
+
+        Returns
+        -------
+        list of np.ndarray
+            A list with an entry per field of ndarrays with shape
+            (minutes in range, sids) with a dtype of float64, containing the
+            values for the respective field over start and end dt range.
+        """
+        rolls_by_asset = {}
+
+        tc = self.trading_calendar
+        start_session = tc.minute_to_session_label(start_date)
+        end_session = tc.minute_to_session_label(end_date)
+
+        for asset in assets:
+            rf = self._roll_finders[asset.roll_style]
+            rolls_by_asset[asset] = rf.get_rolls(
+                asset.root_symbol,
+                start_session,
+                end_session, asset.offset)
+
+        sessions = tc.sessions_in_range(start_date, end_date)
+
+        minutes = tc.minutes_in_range(start_date, end_date)
+        num_minutes = len(minutes)
+        shape = num_minutes, len(assets)
+
+        results = []
+
+        # Get partitions
+        partitions_by_asset = {}
+        for asset in assets:
+            partitions = []
+            partitions_by_asset[asset] = partitions
+            rolls = rolls_by_asset[asset]
+            start = start_date
+            for roll in rolls:
+                sid, roll_date = roll
+                start_loc = minutes.searchsorted(start)
+                if roll_date is not None:
+                    _, end = tc.open_and_close_for_session(
+                        roll_date - sessions.freq)
+                    end_loc = minutes.searchsorted(end)
+                else:
+                    end = end_date
+                    end_loc = len(minutes) - 1
+                partitions.append((sid, start, end, start_loc, end_loc))
+                if roll[-1] is not None:
+                    start, _ = tc.open_and_close_for_session(
+                        tc.minute_to_session_label(minutes[end_loc + 1]))
+
+        for column in columns:
+            if column != 'volume':
+                out = np.full(shape, np.nan)
+            else:
+                out = np.zeros(shape, dtype=np.uint32)
+            for i, asset in enumerate(assets):
+                partitions = partitions_by_asset[asset]
+                for sid, start, end, start_loc, end_loc in partitions:
+                    if column != 'sid':
+                        result = self._bar_reader.load_raw_arrays(
+                            [column], start, end, [sid])[0][:, 0]
+                    else:
+                        result = int(sid)
+                    out[start_loc:end_loc + 1, i] = result
+            results.append(out)
+        return results
+
+    @property
+    def last_available_dt(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The last session for which the reader can provide data.
+        """
+        return self._bar_reader.last_available_dt
+
+    @property
+    def trading_calendar(self):
+        """
+        Returns the catalyst.utils.calendar.trading_calendar used to read
+        the data.  Can be None (if the writer didn't specify it).
+        """
+        return self._bar_reader.trading_calendar
+
+    @property
+    def first_trading_day(self):
+        """
+        Returns
+        -------
+        dt : pd.Timestamp
+            The first trading day (session) for which the reader can provide
+            data.
+        """
+        return self._bar_reader.first_trading_day
+
+    def get_value(self, continuous_future, dt, field):
+        """
+        Retrieve the value at the given coordinates.
+
+        Parameters
+        ----------
+        sid : int
+            The asset identifier.
+        dt : pd.Timestamp
+            The timestamp for the desired data point.
+        field : string
+            The OHLVC name for the desired data point.
+
+        Returns
+        -------
+        value : float|int
+            The value at the given coordinates, ``float`` for OHLC, ``int``
+            for 'volume'.
+
+        Raises
+        ------
+        NoDataOnDate
+            If the given dt is not a valid market minute (in minute mode) or
+            session (in daily mode) according to this reader's tradingcalendar.
+        """
+        rf = self._roll_finders[continuous_future.roll_style]
+        sid = (rf.get_contract_center(continuous_future.root_symbol,
+                                      dt,
+                                      continuous_future.offset))
+        return self._bar_reader.get_value(sid, dt, field)
+
+    def get_last_traded_dt(self, asset, dt):
+        """
+        Get the latest minute on or before ``dt`` in which ``asset`` traded.
+
+        If there are no trades on or before ``dt``, returns ``pd.NaT``.
+
+        Parameters
+        ----------
+        asset : catalyst.asset.Asset
+            The asset for which to get the last traded minute.
+        dt : pd.Timestamp
+            The minute at which to start searching for the last traded minute.
+
+        Returns
+        -------
+        last_traded : pd.Timestamp
+            The dt of the last trade for the given asset, using the input
+            dt as a vantage point.
+        """
+        rf = self._roll_finders[asset.roll_style]
+        sid = (rf.get_contract_center(asset.root_symbol,
+                                      dt,
+                                      asset.offset))
+        if sid is None:
+            return pd.NaT
+        contract = rf.asset_finder.retrieve_asset(sid)
+        return self._bar_reader.get_last_traded_dt(contract, dt)
+
+    @property
+    def sessions(self):
+        return self._bar_reader.sessions

catalyst/data/dispatch_bar_reader.py

@@ -0,0 +1,149 @@
+#
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from abc import ABCMeta, abstractmethod
+
+from numpy import (
+    full,
+    nan,
+    int64,
+    zeros
+)
+from six import iteritems, with_metaclass
+
+from catalyst.utils.memoize import lazyval
+
+
+class AssetDispatchBarReader(with_metaclass(ABCMeta)):
+    """
+
+    Parameters
+    ----------
+    - trading_calendar : catalyst.utils.trading_calendar.TradingCalendar
+    - asset_finder : catalyst.assets.AssetFinder
+    - readers : dict
+        A dict mapping Asset type to the corresponding
+        [Minute|Session]BarReader
+    - last_available_dt : pd.Timestamp or None, optional
+        If not provided, infers it by using the min of the
+        last_available_dt values of the underlying readers.
+    """
+    def __init__(
+        self,
+        trading_calendar,
+        asset_finder,
+        readers,
+        last_available_dt=None,
+    ):
+        self._trading_calendar = trading_calendar
+        self._asset_finder = asset_finder
+        self._readers = readers
+        self._last_available_dt = last_available_dt
+
+        for t, r in iteritems(self._readers):
+            assert trading_calendar == r.trading_calendar, \
+                "All readers must share target trading_calendar. " \
+                "Reader={0} for type={1} uses calendar={2} which does not " \
+                "match the desired shared calendar={3} ".format(
+                    r, t, r.trading_calendar, trading_calendar)
+
+    @abstractmethod
+    def _dt_window_size(self, start_dt, end_dt):
+        pass
+
+    @property
+    def _asset_types(self):
+        return self._readers.keys()
+
+    def _make_raw_array_shape(self, start_dt, end_dt, num_sids):
+        return self._dt_window_size(start_dt, end_dt), num_sids
+
+    def _make_raw_array_out(self, field, shape):
+        if field != 'volume' and field != 'sid':
+            out = full(shape, nan)
+        else:
+            out = zeros(shape, dtype=int64)
+        return out
+
+    @property
+    def trading_calendar(self):
+        return self._trading_calendar
+
+    @lazyval
+    def last_available_dt(self):
+        if self._last_available_dt is not None:
+            return self._last_available_dt
+        else:
+            return min(r.last_available_dt for r in self._readers.values())
+
+    @lazyval
+    def first_trading_day(self):
+        return max(r.first_trading_day for r in self._readers.values())
+
+    def get_value(self, sid, dt, field):
+        asset = self._asset_finder.retrieve_asset(sid)
+        r = self._readers[type(asset)]
+        return r.get_value(asset, dt, field)
+
+    def get_last_traded_dt(self, asset, dt):
+        r = self._readers[type(asset)]
+        return r.get_last_traded_dt(asset, dt)
+
+    def load_raw_arrays(self, fields, start_dt, end_dt, sids):
+        asset_types = self._asset_types
+        sid_groups = {t: [] for t in asset_types}
+        out_pos = {t: [] for t in asset_types}
+
+        assets = self._asset_finder.retrieve_all(sids)
+
+        for i, asset in enumerate(assets):
+            t = type(asset)
+            sid_groups[t].append(asset)
+            out_pos[t].append(i)
+
+        batched_arrays = {
+            t: self._readers[t].load_raw_arrays(fields,
+                                                start_dt,
+                                                end_dt,
+                                                sid_groups[t])
+            for t in asset_types if sid_groups[t]}
+
+        results = []
+        shape = self._make_raw_array_shape(start_dt, end_dt, len(sids))
+
+        for i, field in enumerate(fields):
+            out = self._make_raw_array_out(field, shape)
+            for t, arrays in iteritems(batched_arrays):
+                out[:, out_pos[t]] = arrays[i]
+            results.append(out)
+
+        return results
+
+
+class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
+
+    def _dt_window_size(self, start_dt, end_dt):
+        return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
+
+
+class AssetDispatchSessionBarReader(AssetDispatchBarReader):
+
+    def _dt_window_size(self, start_dt, end_dt):
+        return len(self.trading_calendar.sessions_in_range(start_dt, end_dt))
+
+    @lazyval
+    def sessions(self):
+        return self.trading_calendar.sessions_in_range(
+            self.first_trading_day,
+            self.last_available_dt)

catalyst/data/history_loader.py

@@ -0,0 +1,597 @@
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import (
+    ABCMeta,
+    abstractmethod,
+    abstractproperty,
+)
+
+from numpy import concatenate
+from lru import LRU
+from pandas import isnull
+from pandas.tslib import normalize_date
+from toolz import sliding_window
+
+from six import with_metaclass
+
+from catalyst.assets import Equity, Future
+from catalyst.assets.continuous_futures import ContinuousFuture
+from catalyst.lib._int64window import AdjustedArrayWindow as Int64Window
+from catalyst.lib._float64window import AdjustedArrayWindow as Float64Window
+from catalyst.lib.adjustment import Float64Multiply, Float64Add
+from catalyst.utils.cache import ExpiringCache
+from catalyst.utils.math_utils import number_of_decimal_places
+from catalyst.utils.memoize import lazyval
+from catalyst.utils.numpy_utils import float64_dtype
+from catalyst.utils.pandas_utils import find_in_sorted_index
+
+# Default number of decimal places used for rounding asset prices.
+DEFAULT_ASSET_PRICE_DECIMALS = 3
+
+
+class HistoryCompatibleUSEquityAdjustmentReader(object):
+
+    def __init__(self, adjustment_reader):
+        self._adjustments_reader = adjustment_reader
+
+    def load_adjustments(self, columns, dts, assets):
+        """
+        Returns
+        -------
+        adjustments : list[dict[int -> Adjustment]]
+            A list, where each element corresponds to the `columns`, of
+            mappings from index to adjustment objects to apply at that index.
+        """
+        out = [None] * len(columns)
+        for i, column in enumerate(columns):
+            adjs = {}
+            for asset in assets:
+                adjs.update(self._get_adjustments_in_range(
+                    asset, dts, column))
+            out[i] = adjs
+        return out
+
+    def _get_adjustments_in_range(self, asset, dts, field):
+        """
+        Get the Float64Multiply objects to pass to an AdjustedArrayWindow.
+
+        For the use of AdjustedArrayWindow in the loader, which looks back
+        from current simulation time back to a window of data the dictionary is
+        structured with:
+        - the key into the dictionary for adjustments is the location of the
+        day from which the window is being viewed.
+        - the start of all multiply objects is always 0 (in each window all
+          adjustments are overlapping)
+        - the end of the multiply object is the location before the calendar
+          location of the adjustment action, making all days before the event
+          adjusted.
+
+        Parameters
+        ----------
+        asset : Asset
+            The assets for which to get adjustments.
+        dts : iterable of datetime64-like
+            The dts for which adjustment data is needed.
+        field : str
+            OHLCV field for which to get the adjustments.
+
+        Returns
+        -------
+        out : dict[loc -> Float64Multiply]
+            The adjustments as a dict of loc -> Float64Multiply
+        """
+        sid = int(asset)
+        start = normalize_date(dts[0])
+        end = normalize_date(dts[-1])
+        adjs = {}
+        if field != 'volume':
+            mergers = self._adjustments_reader.get_adjustments_for_sid(
+                'mergers', sid)
+            for m in mergers:
+                dt = m[0]
+                if start < dt <= end:
+                    end_loc = dts.searchsorted(dt)
+                    adj_loc = end_loc
+                    mult = Float64Multiply(0,
+                                           end_loc - 1,
+                                           0,
+                                           0,
+                                           m[1])
+                    try:
+                        adjs[adj_loc].append(mult)
+                    except KeyError:
+                        adjs[adj_loc] = [mult]
+            divs = self._adjustments_reader.get_adjustments_for_sid(
+                'dividends', sid)
+            for d in divs:
+                dt = d[0]
+                if start < dt <= end:
+                    end_loc = dts.searchsorted(dt)
+                    adj_loc = end_loc
+                    mult = Float64Multiply(0,
+                                           end_loc - 1,
+                                           0,
+                                           0,
+                                           d[1])
+                    try:
+                        adjs[adj_loc].append(mult)
+                    except KeyError:
+                        adjs[adj_loc] = [mult]
+        splits = self._adjustments_reader.get_adjustments_for_sid(
+            'splits', sid)
+        for s in splits:
+            dt = s[0]
+            if start < dt <= end:
+                if field == 'volume':
+                    ratio = 1.0 / s[1]
+                else:
+                    ratio = s[1]
+                end_loc = dts.searchsorted(dt)
+                adj_loc = end_loc
+                mult = Float64Multiply(0,
+                                       end_loc - 1,
+                                       0,
+                                       0,
+                                       ratio)
+                try:
+                    adjs[adj_loc].append(mult)
+                except KeyError:
+                    adjs[adj_loc] = [mult]
+        return adjs
+
+
+class ContinuousFutureAdjustmentReader(object):
+    """
+    Calculates adjustments for continuous futures, based on the
+    close and open of the contracts on the either side of each roll.
+    """
+
+    def __init__(self,
+                 trading_calendar,
+                 asset_finder,
+                 bar_reader,
+                 roll_finders,
+                 frequency):
+        self._trading_calendar = trading_calendar
+        self._asset_finder = asset_finder
+        self._bar_reader = bar_reader
+        self._roll_finders = roll_finders
+        self._frequency = frequency
+
+    def load_adjustments(self, columns, dts, assets):
+        """
+        Returns
+        -------
+        adjustments : list[dict[int -> Adjustment]]
+            A list, where each element corresponds to the `columns`, of
+            mappings from index to adjustment objects to apply at that index.
+        """
+        out = [None] * len(columns)
+        for i, column in enumerate(columns):
+            adjs = {}
+            for asset in assets:
+                adjs.update(self._get_adjustments_in_range(
+                    asset, dts, column))
+            out[i] = adjs
+        return out
+
+    def _make_adjustment(self,
+                         adjustment_type,
+                         front_close,
+                         back_close,
+                         end_loc):
+        adj_base = back_close - front_close
+        if adjustment_type == 'mul':
+            adj_value = 1.0 + adj_base / front_close
+            adj_class = Float64Multiply
+        elif adjustment_type == 'add':
+            adj_value = adj_base
+            adj_class = Float64Add
+        return adj_class(0,
+                         end_loc,
+                         0,
+                         0,
+                         adj_value)
+
+    def _get_adjustments_in_range(self, cf, dts, field):
+        if field == 'volume' or field == 'sid':
+            return {}
+        if cf.adjustment is None:
+            return {}
+        rf = self._roll_finders[cf.roll_style]
+        partitions = []
+
+        rolls = rf.get_rolls(cf.root_symbol, dts[0], dts[-1],
+                             cf.offset)
+
+        tc = self._trading_calendar
+
+        adjs = {}
+
+        for front, back in sliding_window(2, rolls):
+            front_sid, roll_dt = front
+            back_sid = back[0]
+            dt = tc.previous_session_label(roll_dt)
+            if self._frequency == 'minute':
+                dt = tc.open_and_close_for_session(dt)[1]
+                roll_dt = tc.open_and_close_for_session(roll_dt)[0]
+            partitions.append((front_sid,
+                               back_sid,
+                               dt,
+                               roll_dt))
+        for partition in partitions:
+            front_sid, back_sid, dt, roll_dt = partition
+            last_front_dt = self._bar_reader.get_last_traded_dt(
+                self._asset_finder.retrieve_asset(front_sid), dt)
+            last_back_dt = self._bar_reader.get_last_traded_dt(
+                self._asset_finder.retrieve_asset(back_sid), dt)
+            if isnull(last_front_dt) or isnull(last_back_dt):
+                continue
+            front_close = self._bar_reader.get_value(
+                front_sid, last_front_dt, 'close')
+            back_close = self._bar_reader.get_value(
+                back_sid, last_back_dt, 'close')
+            adj_loc = dts.searchsorted(roll_dt)
+            end_loc = adj_loc - 1
+            adj = self._make_adjustment(cf.adjustment,
+                                        front_close,
+                                        back_close,
+                                        end_loc)
+            try:
+                adjs[adj_loc].append(adj)
+            except KeyError:
+                adjs[adj_loc] = [adj]
+        return adjs
+
+
+class SlidingWindow(object):
+    """
+    Wrapper around an AdjustedArrayWindow which supports monotonically
+    increasing (by datetime) requests for a sized window of data.
+
+    Parameters
+    ----------
+    window : AdjustedArrayWindow
+       Window of pricing data with prefetched values beyond the current
+       simulation dt.
+    cal_start : int
+       Index in the overall calendar at which the window starts.
+    """
+
+    def __init__(self, window, size, cal_start, offset):
+        self.window = window
+        self.cal_start = cal_start
+        self.current = next(window)
+        self.offset = offset
+        self.most_recent_ix = self.cal_start + size
+
+    def get(self, end_ix):
+        """
+        Returns
+        -------
+        out : A np.ndarray of the equity pricing up to end_ix after adjustments
+              and rounding have been applied.
+        """
+        if self.most_recent_ix == end_ix:
+            return self.current
+
+        target = end_ix - self.cal_start - self.offset + 1
+        self.current = self.window.seek(target)
+
+        self.most_recent_ix = end_ix
+        return self.current
+
+
+class HistoryLoader(with_metaclass(ABCMeta)):
+    """
+    Loader for sliding history windows, with support for adjustments.
+
+    Parameters
+    ----------
+    trading_calendar: TradingCalendar
+        Contains the grouping logic needed to assign minutes to periods.
+    reader : DailyBarReader, MinuteBarReader
+        Reader for pricing bars.
+    adjustment_reader : SQLiteAdjustmentReader
+        Reader for adjustment data.
+    """
+    FIELDS = ('open', 'high', 'low', 'close', 'volume', 'sid')
+
+    def __init__(self, trading_calendar, reader, equity_adjustment_reader,
+                 asset_finder,
+                 roll_finders=None,
+                 sid_cache_size=1000,
+                 prefetch_length=0):
+        self.trading_calendar = trading_calendar
+        self._asset_finder = asset_finder
+        self._reader = reader
+        self._adjustment_readers = {}
+        if equity_adjustment_reader is not None:
+            self._adjustment_readers[Equity] = \
+                HistoryCompatibleUSEquityAdjustmentReader(
+                    equity_adjustment_reader)
+        if roll_finders:
+            self._adjustment_readers[ContinuousFuture] =\
+                ContinuousFutureAdjustmentReader(trading_calendar,
+                                                 asset_finder,
+                                                 reader,
+                                                 roll_finders,
+                                                 self._frequency)
+        self._window_blocks = {
+            field: ExpiringCache(LRU(sid_cache_size))
+            for field in self.FIELDS
+        }
+        self._prefetch_length = prefetch_length
+
+    @abstractproperty
+    def _frequency(self):
+        pass
+
+    @abstractproperty
+    def _calendar(self):
+        pass
+
+    @abstractmethod
+    def _array(self, start, end, assets, field):
+        pass
+
+    def _decimal_places_for_asset(self, asset, reference_date):
+        if isinstance(asset, Future) and asset.tick_size:
+            return number_of_decimal_places(asset.tick_size)
+        elif isinstance(asset, ContinuousFuture):
+            # Tick size should be the same for all contracts of a continuous
+            # future, so arbitrarily get the contract with next upcoming auto
+            # close date.
+            oc = self._asset_finder.get_ordered_contracts(asset.root_symbol)
+            contract_sid = oc.contract_before_auto_close(reference_date.value)
+            if contract_sid is not None:
+                contract = self._asset_finder.retrieve_asset(contract_sid)
+                if contract.tick_size:
+                    return number_of_decimal_places(contract.tick_size)
+        return DEFAULT_ASSET_PRICE_DECIMALS
+
+    def _ensure_sliding_windows(self, assets, dts, field,
+                                is_perspective_after):
+        """
+        Ensure that there is a Float64Multiply window for each asset that can
+        provide data for the given parameters.
+        If the corresponding window for the (assets, len(dts), field) does not
+        exist, then create a new one.
+        If a corresponding window does exist for (assets, len(dts), field), but
+        can not provide data for the current dts range, then create a new
+        one and replace the expired window.
+
+        Parameters
+        ----------
+        assets : iterable of Assets
+            The assets in the window
+        dts : iterable of datetime64-like
+            The datetimes for which to fetch data.
+            Makes an assumption that all dts are present and contiguous,
+            in the calendar.
+        field : str
+            The OHLCV field for which to retrieve data.
+        is_perspective_after : bool
+            see: `PricingHistoryLoader.history`
+
+        Returns
+        -------
+        out : list of Float64Window with sufficient data so that each asset's
+        window can provide `get` for the index corresponding with the last
+        value in `dts`
+        """
+        end = dts[-1]
+        size = len(dts)
+        asset_windows = {}
+        needed_assets = []
+        cal = self._calendar
+
+        assets = self._asset_finder.retrieve_all(assets)
+        end_ix = find_in_sorted_index(cal, end)
+
+        for asset in assets:
+            try:
+                window = self._window_blocks[field].get(
+                    (asset, size, is_perspective_after), end)
+            except KeyError:
+                needed_assets.append(asset)
+            else:
+                if end_ix < window.most_recent_ix:
+                    # Window needs reset. Requested end index occurs before the
+                    # end index from the previous history call for this window.
+                    # Grab new window instead of rewinding adjustments.
+                    needed_assets.append(asset)
+                else:
+                    asset_windows[asset] = window
+
+        if needed_assets:
+            offset = 0
+            start_ix = find_in_sorted_index(cal, dts[0])
+
+            prefetch_end_ix = min(end_ix + self._prefetch_length, len(cal) - 1)
+            prefetch_end = cal[prefetch_end_ix]
+            prefetch_dts = cal[start_ix:prefetch_end_ix + 1]
+            if is_perspective_after:
+                adj_end_ix = min(prefetch_end_ix + 1, len(cal) - 1)
+                adj_dts = cal[start_ix:adj_end_ix + 1]
+            else:
+                adj_dts = prefetch_dts
+            prefetch_len = len(prefetch_dts)
+            array = self._array(prefetch_dts, needed_assets, field)
+
+            if field == 'sid':
+                window_type = Int64Window
+            else:
+                window_type = Float64Window
+
+            view_kwargs = {}
+            if field == 'volume':
+                array = array.astype(float64_dtype)
+
+            for i, asset in enumerate(needed_assets):
+                adj_reader = None
+                try:
+                    adj_reader = self._adjustment_readers[type(asset)]
+                except KeyError:
+                    adj_reader = None
+                if adj_reader is not None:
+                    adjs = adj_reader.load_adjustments(
+                        [field], adj_dts, [asset])[0]
+                else:
+                    adjs = {}
+                window = window_type(
+                    array[:, i].reshape(prefetch_len, 1),
+                    view_kwargs,
+                    adjs,
+                    offset,
+                    size,
+                    int(is_perspective_after),
+                    self._decimal_places_for_asset(asset, dts[-1]),
+                )
+                sliding_window = SlidingWindow(window, size, start_ix, offset)
+                asset_windows[asset] = sliding_window
+                self._window_blocks[field].set(
+                    (asset, size, is_perspective_after),
+                    sliding_window,
+                    prefetch_end)
+
+        return [asset_windows[asset] for asset in assets]
+
+    def history(self, assets, dts, field, is_perspective_after):
+        """
+        A window of pricing data with adjustments applied assuming that the
+        end of the window is the day before the current simulation time.
+
+        Parameters
+        ----------
+        assets : iterable of Assets
+            The assets in the window.
+        dts : iterable of datetime64-like
+            The datetimes for which to fetch data.
+            Makes an assumption that all dts are present and contiguous,
+            in the calendar.
+        field : str
+            The OHLCV field for which to retrieve data.
+        is_perspective_after : bool
+            True, if the window is being viewed immediately after the last dt
+            in the sliding window.
+            False, if the window is viewed on the last dt.
+
+            This flag is used for handling the case where the last dt in the
+            requested window immediately precedes a corporate action, e.g.:
+
+            - is_perspective_after is True
+
+            When the viewpoint is after the last dt in the window, as when a
+            daily history window is accessed from a simulation that uses a
+            minute data frequency, the history call to this loader will not
+            include the current simulation dt. At that point in time, the raw
+            data for the last day in the window will require adjustment, so the
+            most recent adjustment with respect to the simulation time is
+            applied to the last dt in the requested window.
+
+            An example equity which has a 0.5 split ratio dated for 05-27,
+            with the dts for a history call of 5 bars with a '1d' frequency at
+            05-27 9:31. Simulation frequency is 'minute'.
+
+            (In this case this function is called with 4 daily dts, and the
+             calling function is responsible for stitching back on the
+             'current' dt)
+
+            |       |       |       |       | last dt | <-- viewer is here |
+            |       | 05-23 | 05-24 | 05-25 | 05-26   | 05-27 9:31         |
+            | raw   | 10.10 | 10.20 | 10.30 | 10.40   |                    |
+            | adj   |  5.05 |  5.10 |  5.15 |  5.25   |                    |
+
+            The adjustment is applied to the last dt, 05-26, and all previous
+            dts.
+
+            - is_perspective_after is False, daily
+
+            When the viewpoint is the same point in time as the last dt in the
+            window, as when a daily history window is accessed from a
+            simulation that uses a daily data frequency, the history call will
+            include the current dt. At that point in time, the raw data for the
+            last day in the window will be post-adjustment, so no adjustment
+            is applied to the last dt.
+
+            An example equity which has a 0.5 split ratio dated for 05-27,
+            with the dts for a history call of 5 bars with a '1d' frequency at
+            05-27 0:00. Simulation frequency is 'daily'.
+
+            |       |       |       |       |       | <-- viewer is here |
+            |       |       |       |       |       | last dt            |
+            |       | 05-23 | 05-24 | 05-25 | 05-26 | 05-27              |
+            | raw   | 10.10 | 10.20 | 10.30 | 10.40 | 5.25               |
+            | adj   |  5.05 |  5.10 |  5.15 |  5.20 | 5.25               |
+
+            Adjustments are applied 05-23 through 05-26 but not to the last dt,
+            05-27
+
+        Returns
+        -------
+        out : np.ndarray with shape(len(days between start, end), len(assets))
+        """
+        block = self._ensure_sliding_windows(assets,
+                                             dts,
+                                             field,
+                                             is_perspective_after)
+        end_ix = self._calendar.searchsorted(dts[-1])
+
+        return concatenate(
+            [window.get(end_ix) for window in block],
+            axis=1,
+        )
+
+
+class DailyHistoryLoader(HistoryLoader):
+
+    @property
+    def _frequency(self):
+        return 'daily'
+
+    @property
+    def _calendar(self):
+        return self._reader.sessions
+
+    def _array(self, dts, assets, field):
+        return self._reader.load_raw_arrays(
+            [field],
+            dts[0],
+            dts[-1],
+            assets,
+        )[0]
+
+
+class MinuteHistoryLoader(HistoryLoader):
+
+    @property
+    def _frequency(self):
+        return 'minute'
+
+    @lazyval
+    def _calendar(self):
+        mm = self.trading_calendar.all_minutes
+        start = mm.searchsorted(self._reader.first_trading_day)
+        end = mm.searchsorted(self._reader.last_available_dt, side='right')
+        return mm[start:end]
+
+    def _array(self, dts, assets, field):
+        return self._reader.load_raw_arrays(
+            [field],
+            dts[0],
+            dts[-1],
+            assets,
+        )[0]

catalyst/data/loader.py

@@ -0,0 +1,500 @@
+#
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from collections import OrderedDict
+
+import logbook
+import pandas as pd
+from pandas_datareader.data import DataReader
+import pytz
+from six import iteritems
+from six.moves.urllib_error import HTTPError
+
+from .benchmarks import get_benchmark_returns
+from . import treasuries, treasuries_can
+from ..utils.paths import (
+    cache_root,
+    data_root,
+)
+from ..utils.deprecate import deprecated
+from catalyst.utils.calendars import get_calendar
+
+
+logger = logbook.Logger('Loader')
+
+# Mapping from index symbol to appropriate bond data
+INDEX_MAPPING = {
+    'SPY':
+    (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
+    '^GSPTSE':
+    (treasuries_can, 'treasury_curves_can.csv', 'bankofcanada.ca'),
+    '^FTSE':  # use US treasuries until UK bonds implemented
+    (treasuries, 'treasury_curves.csv', 'www.federalreserve.gov'),
+}
+
+ONE_HOUR = pd.Timedelta(hours=1)
+
+
+def last_modified_time(path):
+    """
+    Get the last modified time of path as a Timestamp.
+    """
+    return pd.Timestamp(os.path.getmtime(path), unit='s', tz='UTC')
+
+
+def get_data_filepath(name, environ=None):
+    """
+    Returns a handle to data file.
+
+    Creates containing directory, if needed.
+    """
+    dr = data_root(environ)
+
+    if not os.path.exists(dr):
+        os.makedirs(dr)
+
+    return os.path.join(dr, name)
+
+
+def get_cache_filepath(name):
+    cr = cache_root()
+    if not os.path.exists(cr):
+        os.makedirs(cr)
+
+    return os.path.join(cr, name)
+
+
+def get_benchmark_filename(symbol):
+    return "%s_benchmark.csv" % symbol
+
+
+def has_data_for_dates(series_or_df, first_date, last_date):
+    """
+    Does `series_or_df` have data on or before first_date and on or after
+    last_date?
+    """
+    dts = series_or_df.index
+    if not isinstance(dts, pd.DatetimeIndex):
+        raise TypeError("Expected a DatetimeIndex, but got %s." % type(dts))
+    first, last = dts[[0, -1]]
+    return (first <= first_date) and (last >= last_date)
+
+
+def load_market_data(trading_day=None, trading_days=None, bm_symbol='SPY',
+                     environ=None):
+    """
+    Load benchmark returns and treasury yield curves for the given calendar and
+    benchmark symbol.
+
+    Benchmarks are downloaded as a Series from Google Finance.  Treasury curves
+    are US Treasury Bond rates and are downloaded from 'www.federalreserve.gov'
+    by default.  For Canadian exchanges, a loader for Canadian bonds from the
+    Bank of Canada is also available.
+
+    Results downloaded from the internet are cached in
+    ~/.catalyst/data. Subsequent loads will attempt to read from the cached
+    files before falling back to redownload.
+
+    Parameters
+    ----------
+    trading_day : pandas.CustomBusinessDay, optional
+        A trading_day used to determine the latest day for which we
+        expect to have data.  Defaults to an NYSE trading day.
+    trading_days : pd.DatetimeIndex, optional
+        A calendar of trading days.  Also used for determining what cached
+        dates we should expect to have cached. Defaults to the NYSE calendar.
+    bm_symbol : str, optional
+        Symbol for the benchmark index to load.  Defaults to 'SPY', the Google
+        ticker for the S&P 500.
+
+    Returns
+    -------
+    (benchmark_returns, treasury_curves) : (pd.Series, pd.DataFrame)
+
+    Notes
+    -----
+
+    Both return values are DatetimeIndexed with values dated to midnight in UTC
+    of each stored date.  The columns of `treasury_curves` are:
+
+    '1month', '3month', '6month',
+    '1year','2year','3year','5year','7year','10year','20year','30year'
+    """
+    if trading_day is None:
+        trading_day = get_calendar('NYSE').trading_day
+    if trading_days is None:
+        trading_days = get_calendar('NYSE').all_sessions
+
+    first_date = trading_days[0]
+    now = pd.Timestamp.utcnow()
+
+    # We expect to have benchmark and treasury data that's current up until
+    # **two** full trading days prior to the most recently completed trading
+    # day.
+    # Example:
+    # On Thu Oct 22 2015, the previous completed trading day is Wed Oct 21.
+    # However, data for Oct 21 doesn't become available until the early morning
+    # hours of Oct 22.  This means that there are times on the 22nd at which we
+    # cannot reasonably expect to have data for the 21st available.  To be
+    # conservative, we instead expect that at any time on the 22nd, we can
+    # download data for Tuesday the 20th, which is two full trading days prior
+    # to the date on which we're running a test.
+
+    # We'll attempt to download new data if the latest entry in our cache is
+    # before this date.
+    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
+
+    br = ensure_benchmark_data(
+        bm_symbol,
+        first_date,
+        last_date,
+        now,
+        # We need the trading_day to figure out the close prior to the first
+        # date so that we can compute returns for the first date.
+        trading_day,
+        environ,
+    )
+    tc = ensure_treasury_data(
+        bm_symbol,
+        first_date,
+        last_date,
+        now,
+        environ,
+    )
+    benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
+    treasury_curves = tc[tc.index.slice_indexer(first_date, last_date)]
+    return benchmark_returns, treasury_curves
+
+
+def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
+                          environ=None):
+    """
+    Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
+
+    Parameters
+    ----------
+    symbol : str
+        The symbol for the benchmark to load.
+    first_date : pd.Timestamp
+        First required date for the cache.
+    last_date : pd.Timestamp
+        Last required date for the cache.
+    now : pd.Timestamp
+        The current time.  This is used to prevent repeated attempts to
+        re-download data that isn't available due to scheduling quirks or other
+        failures.
+    trading_day : pd.CustomBusinessDay
+        A trading day delta.  Used to find the day before first_date so we can
+        get the close of the day prior to first_date.
+
+    We attempt to download data unless we already have data stored at the data
+    cache for `symbol` whose first entry is before or on `first_date` and whose
+    last entry is on or after `last_date`.
+
+    If we perform a download and the cache criteria are not satisfied, we wait
+    at least one hour before attempting a redownload.  This is determined by
+    comparing the current time to the result of os.path.getmtime on the cache
+    path.
+    """
+    filename = get_benchmark_filename(symbol)
+    data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
+                             environ)
+    if data is not None:
+        return data
+
+    # If no cached data was found or it was missing any dates then download the
+    # necessary data.
+    logger.info(
+        ('Downloading benchmark data for {symbol!r} '
+            'from {first_date} to {last_date}'),
+        symbol=symbol,
+        first_date=first_date - trading_day,
+        last_date=last_date
+    )
+
+    try:
+        data = get_benchmark_returns(
+            symbol,
+            first_date - trading_day,
+            last_date,
+        )
+        data.to_csv(get_data_filepath(filename, environ))
+    except (OSError, IOError, HTTPError):
+        logger.exception('Failed to cache the new benchmark returns')
+        raise
+    if not has_data_for_dates(data, first_date, last_date):
+        logger.warn("Still don't have expected data after redownload!")
+    return data
+
+
+def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
+    """
+    Ensure we have treasury data from treasury module associated with
+    `symbol`.
+
+    Parameters
+    ----------
+    symbol : str
+        Benchmark symbol for which we're loading associated treasury curves.
+    first_date : pd.Timestamp
+        First date required to be in the cache.
+    last_date : pd.Timestamp
+        Last date required to be in the cache.
+    now : pd.Timestamp
+        The current time.  This is used to prevent repeated attempts to
+        re-download data that isn't available due to scheduling quirks or other
+        failures.
+
+    We attempt to download data unless we already have data stored in the cache
+    for `module_name` whose first entry is before or on `first_date` and whose
+    last entry is on or after `last_date`.
+
+    If we perform a download and the cache criteria are not satisfied, we wait
+    at least one hour before attempting a redownload.  This is determined by
+    comparing the current time to the result of os.path.getmtime on the cache
+    path.
+    """
+    loader_module, filename, source = INDEX_MAPPING.get(
+        symbol, INDEX_MAPPING['SPY'],
+    )
+    first_date = max(first_date, loader_module.earliest_possible_date())
+
+    data = _load_cached_data(filename, first_date, last_date, now, 'treasury',
+                             environ)
+    if data is not None:
+        return data
+
+    # If no cached data was found or it was missing any dates then download the
+    # necessary data.
+    logger.info('Downloading treasury data for {symbol!r}.', symbol=symbol)
+
+    try:
+        data = loader_module.get_treasury_data(first_date, last_date)
+        data.to_csv(get_data_filepath(filename, environ))
+    except (OSError, IOError, HTTPError):
+        logger.exception('failed to cache treasury data')
+    if not has_data_for_dates(data, first_date, last_date):
+        logger.warn("Still don't have expected data after redownload!")
+    return data
+
+
+def _load_cached_data(filename, first_date, last_date, now, resource_name,
+                      environ=None):
+    if resource_name == 'benchmark':
+        from_csv = pd.Series.from_csv
+    else:
+        from_csv = pd.DataFrame.from_csv
+
+    # Path for the cache.
+    path = get_data_filepath(filename, environ)
+
+    # If the path does not exist, it means the first download has not happened
+    # yet, so don't try to read from 'path'.
+    if os.path.exists(path):
+        try:
+            data = from_csv(path)
+            data.index = data.index.to_datetime().tz_localize('UTC')
+            if has_data_for_dates(data, first_date, last_date):
+                return data
+
+            # Don't re-download if we've successfully downloaded and written a
+            # file in the last hour.
+            last_download_time = last_modified_time(path)
+            if (now - last_download_time) <= ONE_HOUR:
+                logger.warn(
+                    "Refusing to download new {resource} data because a "
+                    "download succeeded at {time}.",
+                    resource=resource_name,
+                    time=last_download_time,
+                )
+                return data
+
+        except (OSError, IOError, ValueError) as e:
+            # These can all be raised by various versions of pandas on various
+            # classes of malformed input.  Treat them all as cache misses.
+            logger.info(
+                "Loading data for {path} failed with error [{error}].",
+                path=path,
+                error=e,
+            )
+
+    logger.info(
+        "Cache at {path} does not have data from {start} to {end}.\n",
+        start=first_date,
+        end=last_date,
+        path=path,
+    )
+    return None
+
+
+def _load_raw_yahoo_data(indexes=None, stocks=None, start=None, end=None):
+    """Load closing prices from yahoo finance.
+
+    :Optional:
+        indexes : dict (Default: {'SPX': '^SPY'})
+            Financial indexes to load.
+        stocks : list (Default: ['AAPL', 'GE', 'IBM', 'MSFT',
+                                 'XOM', 'AA', 'JNJ', 'PEP', 'KO'])
+            Stock closing prices to load.
+        start : datetime (Default: datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc))
+            Retrieve prices from start date on.
+        end : datetime (Default: datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc))
+            Retrieve prices until end date.
+
+    :Note:
+        This is based on code presented in a talk by Wes McKinney:
+        http://wesmckinney.com/files/20111017/notebook_output.pdf
+    """
+    assert indexes is not None or stocks is not None, """
+must specify stocks or indexes"""
+
+    if start is None:
+        start = pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc)
+
+    if start is not None and end is not None:
+        assert start < end, "start date is later than end date."
+
+    data = OrderedDict()
+
+    if stocks is not None:
+        for stock in stocks:
+            logger.info('Loading stock: {}'.format(stock))
+            stock_pathsafe = stock.replace(os.path.sep, '--')
+            cache_filename = "{stock}-{start}-{end}.csv".format(
+                stock=stock_pathsafe,
+                start=start,
+                end=end).replace(':', '-')
+            cache_filepath = get_cache_filepath(cache_filename)
+            if os.path.exists(cache_filepath):
+                stkd = pd.DataFrame.from_csv(cache_filepath)
+            else:
+                stkd = DataReader(stock, 'yahoo', start, end).sort_index()
+                stkd.to_csv(cache_filepath)
+            data[stock] = stkd
+
+    if indexes is not None:
+        for name, ticker in iteritems(indexes):
+            logger.info('Loading index: {} ({})'.format(name, ticker))
+            stkd = DataReader(ticker, 'yahoo', start, end).sort_index()
+            data[name] = stkd
+
+    return data
+
+
+def load_from_yahoo(indexes=None,
+                    stocks=None,
+                    start=None,
+                    end=None,
+                    adjusted=True):
+    """
+    Loads price data from Yahoo into a dataframe for each of the indicated
+    assets.  By default, 'price' is taken from Yahoo's 'Adjusted Close',
+    which removes the impact of splits and dividends. If the argument
+    'adjusted' is False, then the non-adjusted 'close' field is used instead.
+
+    :param indexes: Financial indexes to load.
+    :type indexes: dict
+    :param stocks: Stock closing prices to load.
+    :type stocks: list
+    :param start: Retrieve prices from start date on.
+    :type start: datetime
+    :param end: Retrieve prices until end date.
+    :type end: datetime
+    :param adjusted: Adjust the price for splits and dividends.
+    :type adjusted: bool
+
+    """
+    data = _load_raw_yahoo_data(indexes, stocks, start, end)
+    if adjusted:
+        close_key = 'Adj Close'
+    else:
+        close_key = 'Close'
+    df = pd.DataFrame({key: d[close_key] for key, d in iteritems(data)})
+    df.index = df.index.tz_localize(pytz.utc)
+    return df
+
+
+@deprecated(
+    'load_bars_from_yahoo is deprecated, please register a'
+    ' yahoo_equities data bundle instead',
+)
+def load_bars_from_yahoo(indexes=None,
+                         stocks=None,
+                         start=None,
+                         end=None,
+                         adjusted=True):
+    """
+    Loads data from Yahoo into a panel with the following
+    column names for each indicated security:
+
+        - open
+        - high
+        - low
+        - close
+        - volume
+        - price
+
+    Note that 'price' is Yahoo's 'Adjusted Close', which removes the
+    impact of splits and dividends. If the argument 'adjusted' is True, then
+    the open, high, low, and close values are adjusted as well.
+
+    :param indexes: Financial indexes to load.
+    :type indexes: dict
+    :param stocks: Stock closing prices to load.
+    :type stocks: list
+    :param start: Retrieve prices from start date on.
+    :type start: datetime
+    :param end: Retrieve prices until end date.
+    :type end: datetime
+    :param adjusted: Adjust open/high/low/close for splits and dividends.
+        The 'price' field is always adjusted.
+    :type adjusted: bool
+
+    """
+    data = _load_raw_yahoo_data(indexes, stocks, start, end)
+    panel = pd.Panel(data)
+    # Rename columns
+    panel.minor_axis = ['open', 'high', 'low', 'close', 'volume', 'price']
+    panel.major_axis = panel.major_axis.tz_localize(pytz.utc)
+    # Adjust data
+    if adjusted:
+        adj_cols = ['open', 'high', 'low', 'close']
+        for ticker in panel.items:
+            ratio = (panel[ticker]['price'] / panel[ticker]['close'])
+            ratio_filtered = ratio.fillna(0).values
+            for col in adj_cols:
+                panel[ticker][col] *= ratio_filtered
+    return panel
+
+
+def load_prices_from_csv(filepath, identifier_col, tz='UTC'):
+    data = pd.read_csv(filepath, index_col=identifier_col)
+    data.index = pd.DatetimeIndex(data.index, tz=tz)
+    data.sort_index(inplace=True)
+    return data
+
+
+def load_prices_from_csv_folder(folderpath, identifier_col, tz='UTC'):
+    data = None
+    for file in os.listdir(folderpath):
+        if '.csv' not in file:
+            continue
+        raw = load_prices_from_csv(os.path.join(folderpath, file),
+                                   identifier_col, tz)
+        if data is None:
+            data = raw
+        else:
+            data = pd.concat([data, raw], axis=1)
+    return data

catalyst/data/resample.py

@@ -0,0 +1,732 @@
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections import OrderedDict
+from abc import ABCMeta, abstractmethod
+
+import numpy as np
+import pandas as pd
+from six import with_metaclass
+
+from catalyst.data._resample import (
+    _minute_to_session_open,
+    _minute_to_session_high,
+    _minute_to_session_low,
+    _minute_to_session_close,
+    _minute_to_session_volume,
+)
+from catalyst.data.bar_reader import NoDataOnDate
+from catalyst.data.minute_bars import MinuteBarReader
+from catalyst.data.session_bars import SessionBarReader
+from catalyst.utils.memoize import lazyval
+
+_MINUTE_TO_SESSION_OHCLV_HOW = OrderedDict((
+    ('open', 'first'),
+    ('high', 'max'),
+    ('low', 'min'),
+    ('close', 'last'),
+    ('volume', 'sum'),
+))
+
+
+def minute_frame_to_session_frame(minute_frame, calendar):
+
+    """
+    Resample a DataFrame with minute data into the frame expected by a
+    BcolzDailyBarWriter.
+
+    Parameters
+    ----------
+    minute_frame : pd.DataFrame
+        A DataFrame with the columns `open`, `high`, `low`, `close`, `volume`,
+        and `dt` (minute dts)
+    calendar : catalyst.utils.calendars.trading_calendar.TradingCalendar
+        A TradingCalendar on which session labels to resample from minute
+        to session.
+
+    Return
+    ------
+    session_frame : pd.DataFrame
+        A DataFrame with the columns `open`, `high`, `low`, `close`, `volume`,
+        and `day` (datetime-like).
+    """
+    how = OrderedDict((c, _MINUTE_TO_SESSION_OHCLV_HOW[c])
+                      for c in minute_frame.columns)
+    return minute_frame.groupby(calendar.minute_to_session_label).agg(how)
+
+
+def minute_to_session(column, close_locs, data, out):
+    """
+    Resample an array with minute data into an array with session data.
+
+    This function assumes that the minute data is the exact length of all
+    minutes in the sessions in the output.
+
+    Parameters
+    ----------
+    column : str
+        The `open`, `high`, `low`, `close`, or `volume` column.
+    close_locs : array[intp]
+        The locations in `data` which are the market close minutes.
+    data : array[float64|uint32]
+        The minute data to be sampled into session data.
+        The first value should align with the market open of the first session,
+        containing values for all minutes for all sessions. With the last value
+        being the market close of the last session.
+    out : array[float64|uint32]
+        The output array into which to write the sampled sessions.
+    """
+    if column == 'open':
+        _minute_to_session_open(close_locs, data, out)
+    elif column == 'high':
+        _minute_to_session_high(close_locs, data, out)
+    elif column == 'low':
+        _minute_to_session_low(close_locs, data, out)
+    elif column == 'close':
+        _minute_to_session_close(close_locs, data, out)
+    elif column == 'volume':
+        _minute_to_session_volume(close_locs, data, out)
+    return out
+
+
+class DailyHistoryAggregator(object):
+    """
+    Converts minute pricing data into a daily summary, to be used for the
+    last slot in a call to history with a frequency of `1d`.
+
+    This summary is the same as a daily bar rollup of minute data, with the
+    distinction that the summary is truncated to the `dt` requested.
+    i.e. the aggregation slides forward during a the course of simulation day.
+
+    Provides aggregation for `open`, `high`, `low`, `close`, and `volume`.
+    The aggregation rules for each price type is documented in their respective
+
+    """
+
+    def __init__(self, market_opens, minute_reader, trading_calendar):
+        self._market_opens = market_opens
+        self._minute_reader = minute_reader
+        self._trading_calendar = trading_calendar
+
+        # The caches are structured as (date, market_open, entries), where
+        # entries is a dict of asset -> (last_visited_dt, value)
+        #
+        # Whenever an aggregation method determines the current value,
+        # the entry for the respective asset should be overwritten with a new
+        # entry for the current dt.value (int) and aggregation value.
+        #
+        # When the requested dt's date is different from date the cache is
+        # flushed, so that the cache entries do not grow unbounded.
+        #
+        # Example cache:
+        # cache = (date(2016, 3, 17),
+        #          pd.Timestamp('2016-03-17 13:31', tz='UTC'),
+        #          {
+        #              1: (1458221460000000000, np.nan),
+        #              2: (1458221460000000000, 42.0),
+        #         })
+        self._caches = {
+            'open': None,
+            'high': None,
+            'low': None,
+            'close': None,
+            'volume': None
+        }
+
+        # The int value is used for deltas to avoid extra computation from
+        # creating new Timestamps.
+        self._one_min = pd.Timedelta('1 min').value
+
+    def _prelude(self, dt, field):
+        session = self._trading_calendar.minute_to_session_label(dt)
+        dt_value = dt.value
+        cache = self._caches[field]
+        if cache is None or cache[0] != session:
+            market_open = self._market_opens.loc[session]
+            cache = self._caches[field] = (session, market_open, {})
+
+        _, market_open, entries = cache
+        market_open = market_open.tz_localize('UTC')
+        if dt != market_open:
+            prev_dt = dt_value - self._one_min
+        else:
+            prev_dt = None
+        return market_open, prev_dt, dt_value, entries
+
+    def opens(self, assets, dt):
+        """
+        The open field's aggregation returns the first value that occurs
+        for the day, if there has been no data on or before the `dt` the open
+        is `nan`.
+
+        Once the first non-nan open is seen, that value remains constant per
+        asset for the remainder of the day.
+
+        Returns
+        -------
+        np.array with dtype=float64, in order of assets parameter.
+        """
+        market_open, prev_dt, dt_value, entries = self._prelude(dt, 'open')
+
+        opens = []
+        session_label = self._trading_calendar.minute_to_session_label(dt)
+
+        for asset in assets:
+            if not asset.is_alive_for_session(session_label):
+                opens.append(np.NaN)
+                continue
+
+            if prev_dt is None:
+                val = self._minute_reader.get_value(asset, dt, 'open')
+                entries[asset] = (dt_value, val)
+                opens.append(val)
+                continue
+            else:
+                try:
+                    last_visited_dt, first_open = entries[asset]
+                    if last_visited_dt == dt_value:
+                        opens.append(first_open)
+                        continue
+                    elif not pd.isnull(first_open):
+                        opens.append(first_open)
+                        entries[asset] = (dt_value, first_open)
+                        continue
+                    else:
+                        after_last = pd.Timestamp(
+                            last_visited_dt + self._one_min, tz='UTC')
+                        window = self._minute_reader.load_raw_arrays(
+                            ['open'],
+                            after_last,
+                            dt,
+                            [asset],
+                        )[0]
+                        nonnan = window[~pd.isnull(window)]
+                        if len(nonnan):
+                            val = nonnan[0]
+                        else:
+                            val = np.nan
+                        entries[asset] = (dt_value, val)
+                        opens.append(val)
+                        continue
+                except KeyError:
+                    window = self._minute_reader.load_raw_arrays(
+                        ['open'],
+                        market_open,
+                        dt,
+                        [asset],
+                    )[0]
+                    nonnan = window[~pd.isnull(window)]
+                    if len(nonnan):
+                        val = nonnan[0]
+                    else:
+                        val = np.nan
+                    entries[asset] = (dt_value, val)
+                    opens.append(val)
+                    continue
+        return np.array(opens)
+
+    def highs(self, assets, dt):
+        """
+        The high field's aggregation returns the largest high seen between
+        the market open and the current dt.
+        If there has been no data on or before the `dt` the high is `nan`.
+
+        Returns
+        -------
+        np.array with dtype=float64, in order of assets parameter.
+        """
+        market_open, prev_dt, dt_value, entries = self._prelude(dt, 'high')
+
+        highs = []
+        session_label = self._trading_calendar.minute_to_session_label(dt)
+
+        for asset in assets:
+            if not asset.is_alive_for_session(session_label):
+                highs.append(np.NaN)
+                continue
+
+            if prev_dt is None:
+                val = self._minute_reader.get_value(asset, dt, 'high')
+                entries[asset] = (dt_value, val)
+                highs.append(val)
+                continue
+            else:
+                try:
+                    last_visited_dt, last_max = entries[asset]
+                    if last_visited_dt == dt_value:
+                        highs.append(last_max)
+                        continue
+                    elif last_visited_dt == prev_dt:
+                        curr_val = self._minute_reader.get_value(
+                            asset, dt, 'high')
+                        if pd.isnull(curr_val):
+                            val = last_max
+                        elif pd.isnull(last_max):
+                            val = curr_val
+                        else:
+                            val = max(last_max, curr_val)
+                        entries[asset] = (dt_value, val)
+                        highs.append(val)
+                        continue
+                    else:
+                        after_last = pd.Timestamp(
+                            last_visited_dt + self._one_min, tz='UTC')
+                        window = self._minute_reader.load_raw_arrays(
+                            ['high'],
+                            after_last,
+                            dt,
+                            [asset],
+                        )[0].T
+                        val = np.nanmax(np.append(window, last_max))
+                        entries[asset] = (dt_value, val)
+                        highs.append(val)
+                        continue
+                except KeyError:
+                    window = self._minute_reader.load_raw_arrays(
+                        ['high'],
+                        market_open,
+                        dt,
+                        [asset],
+                    )[0].T
+                    val = np.nanmax(window)
+                    entries[asset] = (dt_value, val)
+                    highs.append(val)
+                    continue
+        return np.array(highs)
+
+    def lows(self, assets, dt):
+        """
+        The low field's aggregation returns the smallest low seen between
+        the market open and the current dt.
+        If there has been no data on or before the `dt` the low is `nan`.
+
+        Returns
+        -------
+        np.array with dtype=float64, in order of assets parameter.
+        """
+        market_open, prev_dt, dt_value, entries = self._prelude(dt, 'low')
+
+        lows = []
+        session_label = self._trading_calendar.minute_to_session_label(dt)
+
+        for asset in assets:
+            if not asset.is_alive_for_session(session_label):
+                lows.append(np.NaN)
+                continue
+
+            if prev_dt is None:
+                val = self._minute_reader.get_value(asset, dt, 'low')
+                entries[asset] = (dt_value, val)
+                lows.append(val)
+                continue
+            else:
+                try:
+                    last_visited_dt, last_min = entries[asset]
+                    if last_visited_dt == dt_value:
+                        lows.append(last_min)
+                        continue
+                    elif last_visited_dt == prev_dt:
+                        curr_val = self._minute_reader.get_value(
+                            asset, dt, 'low')
+                        val = np.nanmin([last_min, curr_val])
+                        entries[asset] = (dt_value, val)
+                        lows.append(val)
+                        continue
+                    else:
+                        after_last = pd.Timestamp(
+                            last_visited_dt + self._one_min, tz='UTC')
+                        window = self._minute_reader.load_raw_arrays(
+                            ['low'],
+                            after_last,
+                            dt,
+                            [asset],
+                        )[0].T
+                        val = np.nanmin(np.append(window, last_min))
+                        entries[asset] = (dt_value, val)
+                        lows.append(val)
+                        continue
+                except KeyError:
+                    window = self._minute_reader.load_raw_arrays(
+                        ['low'],
+                        market_open,
+                        dt,
+                        [asset],
+                    )[0].T
+                    val = np.nanmin(window)
+                    entries[asset] = (dt_value, val)
+                    lows.append(val)
+                    continue
+        return np.array(lows)
+
+    def closes(self, assets, dt):
+        """
+        The close field's aggregation returns the latest close at the given
+        dt.
+        If the close for the given dt is `nan`, the most recent non-nan
+        `close` is used.
+        If there has been no data on or before the `dt` the close is `nan`.
+
+        Returns
+        -------
+        np.array with dtype=float64, in order of assets parameter.
+        """
+        market_open, prev_dt, dt_value, entries = self._prelude(dt, 'close')
+
+        closes = []
+        session_label = self._trading_calendar.minute_to_session_label(dt)
+
+        def _get_filled_close(asset):
+            """
+            Returns the most recent non-nan close for the asset in this
+            session. If there has been no data in this session on or before the
+            `dt`, returns `nan`
+            """
+            window = self._minute_reader.load_raw_arrays(
+                ['close'],
+                market_open,
+                dt,
+                [asset],
+            )[0]
+            try:
+                return window[~np.isnan(window)][-1]
+            except IndexError:
+                return np.NaN
+
+        for asset in assets:
+            if not asset.is_alive_for_session(session_label):
+                closes.append(np.NaN)
+                continue
+
+            if prev_dt is None:
+                val = self._minute_reader.get_value(asset, dt, 'close')
+                entries[asset] = (dt_value, val)
+                closes.append(val)
+                continue
+            else:
+                try:
+                    last_visited_dt, last_close = entries[asset]
+                    if last_visited_dt == dt_value:
+                        closes.append(last_close)
+                        continue
+                    elif last_visited_dt == prev_dt:
+                        val = self._minute_reader.get_value(
+                            asset, dt, 'close')
+                        if pd.isnull(val):
+                            val = last_close
+                        entries[asset] = (dt_value, val)
+                        closes.append(val)
+                        continue
+                    else:
+                        val = self._minute_reader.get_value(
+                            asset, dt, 'close')
+                        if pd.isnull(val):
+                            val = _get_filled_close(asset)
+                        entries[asset] = (dt_value, val)
+                        closes.append(val)
+                        continue
+                except KeyError:
+                    val = self._minute_reader.get_value(
+                        asset, dt, 'close')
+                    if pd.isnull(val):
+                        val = _get_filled_close(asset)
+                    entries[asset] = (dt_value, val)
+                    closes.append(val)
+                    continue
+        return np.array(closes)
+
+    def volumes(self, assets, dt):
+        """
+        The volume field's aggregation returns the sum of all volumes
+        between the market open and the `dt`
+        If there has been no data on or before the `dt` the volume is 0.
+
+        Returns
+        -------
+        np.array with dtype=int64, in order of assets parameter.
+        """
+        market_open, prev_dt, dt_value, entries = self._prelude(dt, 'volume')
+
+        volumes = []
+        session_label = self._trading_calendar.minute_to_session_label(dt)
+
+        for asset in assets:
+            if not asset.is_alive_for_session(session_label):
+                volumes.append(0)
+                continue
+
+            if prev_dt is None:
+                val = self._minute_reader.get_value(asset, dt, 'volume')
+                entries[asset] = (dt_value, val)
+                volumes.append(val)
+                continue
+            else:
+                try:
+                    last_visited_dt, last_total = entries[asset]
+                    if last_visited_dt == dt_value:
+                        volumes.append(last_total)
+                        continue
+                    elif last_visited_dt == prev_dt:
+                        val = self._minute_reader.get_value(
+                            asset, dt, 'volume')
+                        val += last_total
+                        entries[asset] = (dt_value, val)
+                        volumes.append(val)
+                        continue
+                    else:
+                        after_last = pd.Timestamp(
+                            last_visited_dt + self._one_min, tz='UTC')
+                        window = self._minute_reader.load_raw_arrays(
+                            ['volume'],
+                            after_last,
+                            dt,
+                            [asset],
+                        )[0]
+                        val = np.nansum(window) + last_total
+                        entries[asset] = (dt_value, val)
+                        volumes.append(val)
+                        continue
+                except KeyError:
+                    window = self._minute_reader.load_raw_arrays(
+                        ['volume'],
+                        market_open,
+                        dt,
+                        [asset],
+                    )[0]
+                    val = np.nansum(window)
+                    entries[asset] = (dt_value, val)
+                    volumes.append(val)
+                    continue
+        return np.array(volumes)
+
+
+class MinuteResampleSessionBarReader(SessionBarReader):
+
+    def __init__(self, calendar, minute_bar_reader):
+        self._calendar = calendar
+        self._minute_bar_reader = minute_bar_reader
+
+    def _get_resampled(self, columns, start_session, end_session, assets):
+        range_open = self._calendar.session_open(start_session)
+        range_close = self._calendar.session_close(end_session)
+
+        minute_data = self._minute_bar_reader.load_raw_arrays(
+            columns,
+            range_open,
+            range_close,
+            assets,
+        )
+
+        # Get the index of the close minute for each session in the range.
+        # If the range contains only one session, the only close in the range
+        # is the last minute in the data. Otherwise, we need to get all the
+        # session closes and find their indices in the range of minutes.
+        if start_session == end_session:
+            close_ilocs = np.array([len(minute_data[0]) - 1], dtype=np.int64)
+        else:
+            minutes = self._calendar.minutes_in_range(
+                range_open,
+                range_close,
+            )
+            session_closes = self._calendar.session_closes_in_range(
+                start_session,
+                end_session,
+            )
+            close_ilocs = minutes.searchsorted(session_closes.values)
+
+        results = []
+        shape = (len(close_ilocs), len(assets))
+
+        for col in columns:
+            if col != 'volume':
+                out = np.full(shape, np.nan)
+            else:
+                out = np.zeros(shape, dtype=np.uint32)
+            results.append(out)
+
+        for i in range(len(assets)):
+            for j, column in enumerate(columns):
+                data = minute_data[j][:, i]
+                minute_to_session(column, close_ilocs, data, results[j][:, i])
+
+        return results
+
+    @property
+    def trading_calendar(self):
+        return self._calendar
+
+    def load_raw_arrays(self, columns, start_dt, end_dt, sids):
+        return self._get_resampled(columns, start_dt, end_dt, sids)
+
+    def get_value(self, sid, session, colname):
+        # WARNING: This will need caching or other optimization if used in a
+        # tight loop.
+        # This was developed to complete interface, but has not been tuned
+        # for real world use.
+        return self._get_resampled([colname], session, session, [sid])[0][0][0]
+
+    @lazyval
+    def sessions(self):
+        cal = self._calendar
+        first = self._minute_bar_reader.first_trading_day
+        last = cal.minute_to_session_label(
+            self._minute_bar_reader.last_available_dt)
+        return cal.sessions_in_range(first, last)
+
+    @lazyval
+    def last_available_dt(self):
+        return self.trading_calendar.minute_to_session_label(
+            self._minute_bar_reader.last_available_dt
+        )
+
+    @property
+    def first_trading_day(self):
+        return self._minute_bar_reader.first_trading_day
+
+    def get_last_traded_dt(self, asset, dt):
+        return self.trading_calendar.minute_to_session_label(
+            self._minute_bar_reader.get_last_traded_dt(asset, dt))
+
+
+class ReindexBarReader(with_metaclass(ABCMeta)):
+    """
+    A base class for readers which reindexes results, filling in the additional
+    indices with empty data.
+
+    Used to align the reading assets which trade on different calendars.
+
+    Currently only supports a ``trading_calendar`` which is a superset of the
+    ``reader``'s calendar.
+
+    Parameters
+    ----------
+
+    - trading_calendar : catalyst.utils.trading_calendar.TradingCalendar
+       The calendar to use when indexing results from the reader.
+    - reader : MinuteBarReader|SessionBarReader
+       The reader which has a calendar that is a subset of the desired
+       ``trading_calendar``.
+    - first_trading_session : pd.Timestamp
+       The first trading session the reader should provide. Must be specified,
+       since the ``reader``'s first session may not exactly align with the
+       desired calendar. Specifically, in the case where the first session
+       on the target calendar is a holiday on the ``reader``'s calendar.
+    - last_trading_session : pd.Timestamp
+       The last trading session the reader should provide. Must be specified,
+       since the ``reader``'s last session may not exactly align with the
+       desired calendar. Specifically, in the case where the last session
+       on the target calendar is a holiday on the ``reader``'s calendar.
+    """
+
+    def __init__(self,
+                 trading_calendar,
+                 reader,
+                 first_trading_session,
+                 last_trading_session):
+        self._trading_calendar = trading_calendar
+        self._reader = reader
+        self._first_trading_session = first_trading_session
+        self._last_trading_session = last_trading_session
+
+    @property
+    def last_available_dt(self):
+        return self._reader.last_available_dt
+
+    def get_last_traded_dt(self, sid, dt):
+        return self._reader.get_last_traded_dt(sid, dt)
+
+    @property
+    def first_trading_day(self):
+        return self._reader.first_trading_day
+
+    def get_value(self, sid, dt, field):
+        # Give an empty result if no data is present.
+        try:
+            return self._reader.get_value(sid, dt, field)
+        except NoDataOnDate:
+            if field == 'volume':
+                return 0
+            else:
+                return np.nan
+
+    @abstractmethod
+    def _outer_dts(self, start_dt, end_dt):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _inner_dts(self, start_dt, end_dt):
+        raise NotImplementedError
+
+    @property
+    def trading_calendar(self):
+        return self._trading_calendar
+
+    @lazyval
+    def sessions(self):
+        return self.trading_calendar.sessions_in_range(
+            self._first_trading_session,
+            self._last_trading_session
+        )
+
+    def load_raw_arrays(self, fields, start_dt, end_dt, sids):
+        outer_dts = self._outer_dts(start_dt, end_dt)
+        inner_dts = self._inner_dts(start_dt, end_dt)
+
+        indices = outer_dts.searchsorted(inner_dts)
+
+        shape = len(outer_dts), len(sids)
+
+        outer_results = []
+
+        if len(inner_dts) > 0:
+            inner_results = self._reader.load_raw_arrays(
+                fields, inner_dts[0], inner_dts[-1], sids)
+        else:
+            inner_results = None
+
+        for i, field in enumerate(fields):
+            if field != 'volume':
+                out = np.full(shape, np.nan)
+            else:
+                out = np.zeros(shape, dtype=np.uint32)
+
+            if inner_results is not None:
+                out[indices] = inner_results[i]
+
+            outer_results.append(out)
+
+        return outer_results
+
+
+class ReindexMinuteBarReader(ReindexBarReader, MinuteBarReader):
+    """
+    See: ``ReindexBarReader``
+    """
+
+    def _outer_dts(self, start_dt, end_dt):
+        return self._trading_calendar.minutes_in_range(start_dt, end_dt)
+
+    def _inner_dts(self, start_dt, end_dt):
+        return self._reader.calendar.minutes_in_range(start_dt, end_dt)
+
+
+class ReindexSessionBarReader(ReindexBarReader, SessionBarReader):
+    """
+    See: ``ReindexBarReader``
+    """
+
+    def _outer_dts(self, start_dt, end_dt):
+        return self.trading_calendar.sessions_in_range(start_dt, end_dt)
+
+    def _inner_dts(self, start_dt, end_dt):
+        return self._reader.trading_calendar.sessions_in_range(
+            start_dt, end_dt)

catalyst/data/session_bars.py

@@ -0,0 +1,36 @@
+# Copyright 2016 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from abc import abstractproperty
+
+from catalyst.data.bar_reader import BarReader
+
+
+class SessionBarReader(BarReader):
+    """
+    Reader for OHCLV pricing data at a session frequency.
+    """
+    @property
+    def data_frequency(self):
+        return 'session'
+
+    @abstractproperty
+    def sessions(self):
+        """
+        Returns
+        -------
+        sessions : DatetimeIndex
+           All session labels (unionining the range for all assets) which the
+           reader can provide.
+        """
+        pass

catalyst/data/treasuries.py

@@ -0,0 +1,101 @@
+#
+# Copyright 2013 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from operator import itemgetter
+import re
+
+import numpy as np
+import pandas as pd
+
+
+get_unit_and_periods = itemgetter('unit', 'periods')
+
+
+def parse_treasury_csv_column(column):
+    """
+    Parse a treasury CSV column into a more human-readable format.
+
+    Columns start with 'RIFLGFC', followed by Y or M (year or month), followed
+    by a two-digit number signifying number of years/months, followed by _N.B.
+    We only care about the middle two entries, which we turn into a string like
+    3month or 30year.
+    """
+    column_re = re.compile(
+        r"^(?P<prefix>RIFLGFC)"
+        "(?P<unit>[YM])"
+        "(?P<periods>[0-9]{2})"
+        "(?P<suffix>_N.B)$"
+    )
+
+    match = column_re.match(column)
+    if match is None:
+        raise ValueError("Couldn't parse CSV column %r." % column)
+    unit, periods = get_unit_and_periods(match.groupdict())
+
+    # Roundtrip through int to coerce '06' into '6'.
+    return str(int(periods)) + ('year' if unit == 'Y' else 'month')
+
+
+def earliest_possible_date():
+    """
+    The earliest date for which we can load data from this module.
+    """
+    # The US Treasury actually has data going back further than this, but it's
+    # pretty rare to find pricing data going back that far, and there's no
+    # reason to make people download benchmarks back to 1950 that they'll never
+    # be able to use.
+    return pd.Timestamp('1980', tz='UTC')
+
+
+def get_treasury_data(start_date, end_date):
+    return pd.read_csv(
+        "https://www.federalreserve.gov/datadownload/Output.aspx"
+        "?rel=H15"
+        "&series=bf17364827e38702b42a58cf8eaa3f78"
+        "&lastObs="
+        "&from="  # An unbounded query is ~2x faster than specifying dates.
+        "&to="
+        "&filetype=csv"
+        "&label=include"
+        "&layout=seriescolumn"
+        "&type=package",
+        skiprows=5,  # First 5 rows are useless headers.
+        parse_dates=['Time Period'],
+        na_values=['ND'],  # Presumably this stands for "No Data".
+        index_col=0,
+    ).loc[
+        start_date:end_date
+    ].dropna(
+        how='all'
+    ).rename(
+        columns=parse_treasury_csv_column
+    ).tz_localize('UTC') * 0.01  # Convert from 2.57% to 0.0257.
+
+
+def dataconverter(s):
+    try:
+        return float(s) / 100
+    except:
+        return np.nan
+
+
+def get_daily_10yr_treasury_data():
+    """Download daily 10 year treasury rates from the Federal Reserve and
+    return a pandas.Series."""
+    url = "https://www.federalreserve.gov/datadownload/Output.aspx?rel=H15" \
+          "&series=bcb44e57fb57efbe90002369321bfb3f&lastObs=&from=&to=" \
+          "&filetype=csv&label=include&layout=seriescolumn"
+    return pd.read_csv(url, header=5, index_col=0, names=['DATE', 'BC_10YEAR'],
+                       parse_dates=True, converters={1: dataconverter},
+                       squeeze=True)

catalyst/data/treasuries_can.py

@@ -0,0 +1,150 @@
+#
+# Copyright 2013 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+import six
+from toolz import curry
+from toolz.curried.operator import add as prepend
+
+COLUMN_NAMES = {
+    "V39063": '1month',
+    "V39065": '3month',
+    "V39066": '6month',
+    "V39067": '1year',
+    "V39051": '2year',
+    "V39052": '3year',
+    "V39053": '5year',
+    "V39054": '7year',
+    "V39055": '10year',
+    # Bank of Canada refers to this as 'Long' Rate, approximately 30 years.
+    "V39056": '30year',
+}
+BILL_IDS = ['V39063', 'V39065', 'V39066', 'V39067']
+BOND_IDS = ['V39051', 'V39052', 'V39053', 'V39054', 'V39055', 'V39056']
+
+
+@curry
+def _format_url(instrument_type,
+                instrument_ids,
+                start_date,
+                end_date,
+                earliest_allowed_date):
+    """
+    Format a URL for loading data from Bank of Canada.
+    """
+    return (
+        "http://www.bankofcanada.ca/stats/results/csv"
+        "?lP=lookup_{instrument_type}_yields.php"
+        "&sR={restrict}"
+        "&se={instrument_ids}"
+        "&dF={start}"
+        "&dT={end}".format(
+            instrument_type=instrument_type,
+            instrument_ids='-'.join(map(prepend("L_"), instrument_ids)),
+            restrict=earliest_allowed_date.strftime("%Y-%m-%d"),
+            start=start_date.strftime("%Y-%m-%d"),
+            end=end_date.strftime("%Y-%m-%d"),
+        )
+    )
+
+
+format_bill_url = _format_url('tbill', BILL_IDS)
+format_bond_url = _format_url('bond', BOND_IDS)
+
+
+def load_frame(url, skiprows):
+    """
+    Load a DataFrame of data from a Bank of Canada site.
+    """
+    return pd.read_csv(
+        url,
+        skiprows=skiprows,
+        skipinitialspace=True,
+        na_values=["Bank holiday", "Not available"],
+        parse_dates=["Date"],
+        index_col="Date",
+    ).dropna(how='all') \
+     .tz_localize('UTC') \
+     .rename(columns=COLUMN_NAMES)
+
+
+def check_known_inconsistencies(bill_data, bond_data):
+    """
+    There are a couple quirks in the data provided by Bank of Canada.
+    Check that no new quirks have been introduced in the latest download.
+    """
+    inconsistent_dates = bill_data.index.sym_diff(bond_data.index)
+    known_inconsistencies = [
+        # bill_data has an entry for 2010-02-15, which bond_data doesn't.
+        # bond_data has an entry for 2006-09-04, which bill_data doesn't.
+        # Both of these dates are bank holidays (Flag Day and Labor Day,
+        # respectively).
+        pd.Timestamp('2006-09-04', tz='UTC'),
+        pd.Timestamp('2010-02-15', tz='UTC'),
+        # 2013-07-25 comes back as "Not available" from the bills endpoint.
+        # This date doesn't seem to be a bank holiday, but the previous
+        # calendar implementation dropped this entry, so we drop it as well.
+        # If someone cares deeply about the integrity of the Canadian trading
+        # calendar, they may want to consider forward-filling here rather than
+        # dropping the row.
+        pd.Timestamp('2013-07-25', tz='UTC'),
+    ]
+    unexpected_inconsistences = inconsistent_dates.drop(known_inconsistencies)
+    if len(unexpected_inconsistences):
+        in_bills = bill_data.index.difference(bond_data.index).difference(
+            known_inconsistencies
+        )
+        in_bonds = bond_data.index.difference(bill_data.index).difference(
+            known_inconsistencies
+        )
+        raise ValueError(
+            "Inconsistent dates for Canadian treasury bills vs bonds. \n"
+            "Dates with bills but not bonds: {in_bills}.\n"
+            "Dates with bonds but not bills: {in_bonds}.".format(
+                in_bills=in_bills,
+                in_bonds=in_bonds,
+            )
+        )
+
+
+def earliest_possible_date():
+    """
+    The earliest date for which we can load data from this module.
+    """
+    today = pd.Timestamp('now', tz='UTC').normalize()
+    # Bank of Canada only has the last 10 years of data at any given time.
+    return today.replace(year=today.year - 10)
+
+
+def get_treasury_data(start_date, end_date):
+    bill_data = load_frame(
+        format_bill_url(start_date, end_date, start_date),
+        # We skip fewer rows here because we query for fewer bill fields,
+        # which makes the header smaller.
+        skiprows=18,
+    )
+    bond_data = load_frame(
+        format_bond_url(start_date, end_date, start_date),
+        skiprows=22,
+    )
+    check_known_inconsistencies(bill_data, bond_data)
+
+    # dropna('any') removes the rows for which we only had data for one of
+    # bills/bonds.
+    out = pd.concat([bond_data, bill_data], axis=1).dropna(how='any')
+    assert set(out.columns) == set(six.itervalues(COLUMN_NAMES))
+
+    # Multiply by 0.01 to convert from percentages to expected output format.
+    return out * 0.01