diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst index 96696bac..b5792fd5 100644 --- a/docs/source/appendix.rst +++ b/docs/source/appendix.rst @@ -15,6 +15,12 @@ The following methods are available for use in the ``initialize``, In all listed functions, the ``self`` argument is implicitly the currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance. +Data Object +``````````` + +.. autoclass:: zipline.protocol.BarData + :members: + Scheduling Functions ```````````````````` @@ -271,6 +277,9 @@ Readers .. autoclass:: zipline.assets.AssetFinderCachedEquities :members: +.. autoclass:: zipline.data.data_portal.DataPortal + :members: + Bundles ``````` .. autofunction:: zipline.data.bundles.register diff --git a/docs/source/whatsnew/1.0.0.txt b/docs/source/whatsnew/1.0.0.txt index 2aa6de05..4dc4cea3 100644 --- a/docs/source/whatsnew/1.0.0.txt +++ b/docs/source/whatsnew/1.0.0.txt @@ -12,6 +12,45 @@ Development Highlights ~~~~~~~~~~ +Zipline 1.0 Rewrite (:issue:`1105`) +``````````````````````````````````` + +We have rewritten a lot of Zipline and its basic concepts in order to improve +runtime performance. At the same time, we've introduced several new APIs. + +At a high level, earlier versions of Zipline simulations pulled from a +multiplexed stream of data sources, which were merged via heapq. This stream was +fed to the main simulation loop, driving the clock forward. This strong +dependency on reading all the data made it difficult to optimize simulation +performance because there was no connection between the amount of data we +fetched and the amount of data actually used by the algorithm. + +Now, we only fetch data when the algorithm needs it. A new class, +:class:`~zipline.data.data_portal.DataPortal`, dispatches data requests to +various data sources and returns the requested values. This makes the runtime of +a simulation scale much more closely with the complexity of the algorithm, +rather than with the number of assets provided by the data sources. + +Instead of the data stream driving the clock, now simulations iterate through a +pre-calculated set of day or minute timestamps. The timestamps are emitted by +:class:`~zipline.gens.sim_engine.MinuteSimulationClock` and +:class:`~zipline.gens.sim_engine.DailySimulationClock`, and consumed by the main +loop in :meth:`~zipline.gens.tradesimulation.AlgorithmSimulator.transform`. + +We've retired the ``data[sid(N)]`` and ``history`` APIs, replacing them with +several methods on the :class:`~zipline.protocol.BarData` object: +:meth:`~zipline.protocol.BarData.current`, +:meth:`~zipline.protocol.BarData.history`, +:meth:`~zipline.protocol.BarData.can_trade`, and +:meth:`~zipline.protocol.BarData.is_stale`. Old APIs will continue to work for +now, but will issue deprecation warnings. + +You can now pass in an adjustments source to the +:class:`~zipline.data.data_portal.DataPortal`, and we will apply adjustments to +the pricing data when looking backwards at data. Prices and volumes for +execution and presented to the algorithm in data.current are the as-traded value +of the asset. + New Entry Points (:issue:`1173` and :issue:`1178`) `````````````````````````````````````````````````` @@ -131,6 +170,9 @@ Enhancements implements the Bollinger Bands technical indicator: https://en.wikipedia.org/wiki/Bollinger_Bands (:issue:`1199`). +* Fetcher has been moved from Quantopian internal code into Zipline + (:issue:`1105`). + Experimental Features ~~~~~~~~~~~~~~~~~~~~~ diff --git a/zipline/_protocol.pyx b/zipline/_protocol.pyx index 64446caf..d8e2c86d 100644 --- a/zipline/_protocol.pyx +++ b/zipline/_protocol.pyx @@ -14,12 +14,13 @@ # limitations under the License. import warnings from contextlib import contextmanager +from functools import wraps from pandas.tslib import normalize_date import pandas as pd import numpy as np -from six import iteritems +from six import iteritems, PY2 from cpython cimport bool from collections import Iterable @@ -31,6 +32,17 @@ cdef bool _is_iterable(obj): return isinstance(obj, Iterable) and not isinstance(obj, str) +# Wraps doesn't work for method objects in python2. Docs should be generated +# with python3 so it is not a big deal. +if PY2: + def no_wraps_py2(f): + def dec(g): + return g + return dec +else: + no_wraps_py2 = wraps + + cdef class check_parameters(object): """ Asserts that the keywords passed into the wrapped function are included @@ -52,6 +64,7 @@ cdef class check_parameters(object): self.keys_to_types = dict(zip(keyword_names, types)) def __call__(self, func): + @no_wraps_py2(func) def assert_keywords_and_call(*args, **kwargs): cdef short i @@ -123,6 +136,27 @@ def handle_non_market_minutes(bar_data): cdef class BarData: + """ + Provides methods to access spot value or history windows of price data. + Also provides some utility methods to determine if an asset is alive, + has recent trade data, etc. + + This is what is passed as ``data`` to the ``handle_data`` function. + + Parameters + ---------- + data_portal : DataPortal + Provider for bar pricing data. + simulation_dt_func : callable + Function which returns the current simulation time. + This is usually bound to a method of TradingSimulation. + data_frequency : {'minute', 'daily'} + The frequency of the bar data; i.e. whether the data is + daily or minute bars + universe_func : callable, optional + Function which returns the current 'universe'. This is for + backwards compatibility with older API concepts. + """ cdef object data_portal cdef object simulation_dt_func cdef object data_frequency @@ -133,32 +167,10 @@ cdef class BarData: cdef bool _adjust_minutes - """ - Provides methods to access spot value or history windows of price data. - Also provides some utility methods to determine if an asset is alive, - has recent trade data, etc. - - This is what is passed as `data` to the `handle_data` function. - """ def __init__(self, data_portal, simulation_dt_func, data_frequency, universe_func=None): """ - Parameters - --------- - data_portal : DataPortal - Provider for bar pricing data. - simulation_dt_func: function - Function which returns the current simulation time. - This is usually bound to a method of TradingSimulation. - - data_frequency: string - The frequency of the bar data; i.e. whether the data is - 'daily' or 'minute' bars - - universe_func: function - Function which returns the current 'universe'. This is for - backwards compatibility with older API concepts. """ self.data_portal = data_portal self.simulation_dt_func = simulation_dt_func @@ -183,7 +195,7 @@ cdef class BarData: Returns ------- - SidView: Accessor into the given asset's data. + SidView : Accessor into the given asset's data. """ try: self._warn_deprecated("`data[sid(N)]` is deprecated. Use " @@ -226,14 +238,15 @@ cdef class BarData: Parameters ---------- assets : Asset or iterable of Assets - - fields : string or iterable of strings. Valid values are: "price", + fields : str or iterable[str]. + Valid values are: "price", "last_traded", "open", "high", "low", "close", "volume", or column - names in files read by fetch_csv. + names in files read by ``fetch_csv``. Returns ------- - Scalar, pandas Series, or pandas DataFrame. See notes below. + current_value : Scalar, pandas Series, or pandas DataFrame. + See notes below. Notes ----- @@ -403,7 +416,7 @@ cdef class BarData: Returns ------- - boolean or Series of booleans, indexed by asset. + can_trade : bool or pd.Series[bool] indexed by asset. """ dt = self.simulation_dt_func() @@ -522,8 +535,9 @@ cdef class BarData: Returns ------- - Series or DataFrame or Panel, depending on the dimensionality of - the 'assets' and 'fields' parameters. + history : Series or DataFrame or Panel + Return type depends on the dimensionality of the 'assets' and + 'fields' parameters. If single asset and field are passed in, the returned Series is indexed by dt. @@ -720,7 +734,7 @@ cdef class SidView: cdef object data_portal cdef object simulation_dt_func cdef object data_frequency - + """ This class exists to temporarily support the deprecated data[sid(N)] API. """ diff --git a/zipline/data/data_portal.py b/zipline/data/data_portal.py index 09e8994b..70db83ec 100644 --- a/zipline/data/data_portal.py +++ b/zipline/data/data_portal.py @@ -459,6 +459,39 @@ class DailyHistoryAggregator(object): class DataPortal(object): + """Interface to all of the data that a zipline simulation needs. + + This is used by the simulation runner to answer questions about the data, + like getting the prices of assets on a given day or to service history + calls. + + Parameters + ---------- + env : TradingEnvironment + The trading environment for the simulation. This includes the trading + calendar and benchmark data. + equity_daily_reader : BcolzDailyBarReader, optional + The daily bar ready for equities. This will be used to service + daily data backtests or daily history calls in a minute backetest. + If a daily bar reader is not provided but a minute bar reader is, + the minutes will be rolled up to serve the daily requests. + equity_minute_reader : BcolzMinuteBarReader, optional + The minute bar reader for equities. This will be used to service + minute data backtests or minute history calls. This can be used + to serve daily calls if no daily bar reader is provided. + future_daily_reader : BcolzDailyBarReader, optional + The daily bar ready for futures. This will be used to service + daily data backtests or daily history calls in a minute backetest. + If a daily bar reader is not provided but a minute bar reader is, + the minutes will be rolled up to serve the daily requests. + future_minute_reader : BcolzMinuteBarReader, optional + The minute bar reader for futures. This will be used to service + minute data backtests or minute history calls. This can be used + to serve daily calls if no daily bar reader is provided. + adjustment_reader : SQLiteAdjustmentWriter, optional + The adjustment reader. This is used to apply splits, dividends, and + other adjustment data to the raw data from the readers. + """ def __init__(self, env, equity_daily_reader=None, @@ -696,24 +729,26 @@ class DataPortal(object): of the desired asset's field at either the given dt. Parameters - --------- + ---------- asset : Asset The asset whose data is desired. - - field: string - The desired field of the asset. Valid values are "open", "high", - "low", "close", "volume", "price", and "last_traded". - - dt: pd.Timestamp + field : {'open', 'high', 'low', 'close', 'volume', + 'price', 'last_traded'} + The desired field of the asset. + dt : pd.Timestamp The timestamp for the desired value. - - data_frequency: string + data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- - The value of the desired field at the desired time. + value : float, int, or pd.Timestamp + The spot value of ``field`` for ``asset`` The return type is based + on the ``field`` requested. If the field is one of 'open', 'high', + 'low', 'close', or 'price', the value will be a float. If the + ``field`` is 'volume' the value will be a int. If the ``field`` is + 'last_traded' the value will be a Timestamp. """ if self._is_extra_source(asset, field, self._augmented_sources_map): return self._get_fetcher_value(asset, field, dt) @@ -755,28 +790,24 @@ class DataPortal(object): given field and list of assets Parameters - --------- + ---------- assets : list of type Asset, or Asset The asset, or assets whose adjustments are desired. - - field: string - The desired field of the asset. Valid values are "open", - "open_price", "high", "low", "close", "close_price", "volume", and - "price". - - dt: pd.Timestamp + field : {'open', 'high', 'low', 'close', 'volume', \ + 'price', 'last_traded'} + The desired field of the asset. + dt : pd.Timestamp The timestamp for the desired value. - perspective_dt : pd.Timestamp The timestamp from which the data is being viewed back from. - - data_frequency: string + data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- - The list of adjustments for the asset(s) + adjustments : list[Adjustment] + The adjustments to that field. """ if isinstance(assets, Asset): assets = [assets] @@ -828,28 +859,29 @@ class DataPortal(object): of the desired asset's field at the given dt with adjustments applied. Parameters - --------- + ---------- asset : Asset The asset whose data is desired. - - field: string - The desired field of the asset. Valid values are "open", - "open_price", "high", "low", "close", "close_price", "volume", and - "price". - - dt: pd.Timestamp + field : {'open', 'high', 'low', 'close', 'volume', \ + 'price', 'last_traded'} + The desired field of the asset. + dt : pd.Timestamp The timestamp for the desired value. - perspective_dt : pd.Timestamp The timestamp from which the data is being viewed back from. - - data_frequency: string + data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- - The value of the desired field at the desired time. + value : float, int, or pd.Timestamp + The value of the given ``field`` for ``asset`` at ``dt`` with any + adjustments known by ``perspective_dt`` applied. The return type is + based on the ``field`` requested. If the field is one of 'open', + 'high', 'low', 'close', or 'price', the value will be a float. If + the ``field`` is 'volume' the value will be a int. If the ``field`` + is 'last_traded' the value will be a Timestamp. """ if spot_value is None: # if this a fetcher field, we want to use perspective_dt (not dt) @@ -1172,7 +1204,7 @@ class DataPortal(object): history window. Data is fully adjusted. Parameters - --------- + ---------- assets : list of zipline.data.Asset objects The assets whose data is desired. @@ -1549,14 +1581,14 @@ class DataPortal(object): ---------- sids : container Sids for which we want splits. - - dt: pd.Timestamp - The date for which we are checking for splits. Note: this is + dt : pd.Timestamp + The date for which we are checking for splits. Note: this is expected to be midnight UTC. Returns ------- - list: List of splits, where each split is a (sid, ratio) tuple. + splits : list[(int, float)] + List of splits, where each split is a (sid, ratio) tuple. """ if self._adjustment_reader is None or not sids: return {} diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py index 5ba4c140..99682235 100644 --- a/zipline/data/minute_bars.py +++ b/zipline/data/minute_bars.py @@ -528,11 +528,11 @@ class BcolzMinuteBarWriter(object): cols : dict of str -> np.array dict of market data with the following characteristics. keys are ('open', 'high', 'low', 'close', 'volume') - open : float64 - high : float64 - low : float64 - close : float64 - volume : float64|int64 + open : float64 + high : float64 + low : float64 + close : float64 + volume : float64|int64 """ if not all(len(dts) == len(cols[name]) for name in self.COL_NAMES): raise BcolzMinuteWriterColumnMismatch( @@ -555,11 +555,11 @@ class BcolzMinuteBarWriter(object): cols : dict of str -> np.array dict of market data with the following characteristics. keys are ('open', 'high', 'low', 'close', 'volume') - open : float64 - high : float64 - low : float64 - close : float64 - volume : float64|int64 + open : float64 + high : float64 + low : float64 + close : float64 + volume : float64|int64 """ table = self._ensure_ctable(sid) diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py index 088f9dce..37256a3d 100644 --- a/zipline/data/us_equity_pricing.py +++ b/zipline/data/us_equity_pricing.py @@ -391,10 +391,41 @@ class BcolzDailyBarReader(DailyBarReader): """ Reader for raw pricing data written by BcolzDailyOHLCVWriter. - A Bcolz CTable is comprised of Columns and Attributes. + Parameters + ---------- + table : bcolz.ctable + The ctable contaning the pricing data, with attrs corresponding to the + Attributes list below. + read_all_threshold : int + The number of equities at which; below, the data is read by reading a + slice from the carray per asset. above, the data is read by pulling + all of the data for all assets into memory and then indexing into that + array for each day and asset pair. Used to tune performance of reads + when using a small or large number of equities. - Columns - ------- + Attributes + ---------- + The table with which this loader interacts contains the following + attributes: + + first_row : dict + Map from asset_id -> index of first row in the dataset with that id. + last_row : dict + Map from asset_id -> index of last row in the dataset with that id. + calendar_offset : dict + Map from asset_id -> calendar index of first row. + calendar : list[int64] + Calendar used to compute offsets, in asi8 format (ns since EPOCH). + + We use first_row and last_row together to quickly find ranges of rows to + load when reading an asset's data into memory. + + We use calendar_offset and calendar to orient loaded blocks within a + range of queried dates. + + Notes + ------ + A Bcolz CTable is comprised of Columns and Attributes. The table with which this loader interacts contains the following columns: ['open', 'high', 'low', 'close', 'volume', 'day', 'id']. @@ -419,41 +450,6 @@ class BcolzDailyBarReader(DailyBarReader): When read across the open, high, low, close, and volume with the same index should represent the same asset and day. - Parameters - ---------- - table : bcolz.ctable - The ctable contaning the pricing data, with attrs corresponding to the - Attributes list below. - read_all_threshold : int - The number of equities at which; - below, the data is read by reading a slice from the carray - per asset. - above, the data is read by pulling all of the data for all assets - into memory and then indexing into that array for each day and - asset pair. - Used to tune performance of reads when using a small or large number - of equities. - - Attributes - ---------- - The table with which this loader interacts contains the following - attributes: - - first_row : dict - Map from asset_id -> index of first row in the dataset with that id. - last_row : dict - Map from asset_id -> index of last row in the dataset with that id. - calendar_offset : dict - Map from asset_id -> calendar index of first row. - calendar : list[int64] - Calendar used to compute offsets, in asi8 format (ns since EPOCH). - - We use first_row and last_row together to quickly find ranges of rows to - load when reading an asset's data into memory. - - We use calendar_offset and calendar to orient loaded blocks within a - range of queried dates. - See Also -------- zipline.data.us_equity_pricing.BcolzDailyBarWriter @@ -1182,7 +1178,7 @@ class SQLiteAdjustmentReader(object): See Also -------- - zipline.data.us_equity_pricing.SQLiteAdjustmentWriter + :class:`zipline.data.us_equity_pricing.SQLiteAdjustmentWriter` """ @preprocess(conn=coerce_string(sqlite3.connect))