mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-09 17:12:54 +08:00
Merge pull request #1170 from quantopian/whats-new-1.0
Updated whatsnew with Q2 information.
This commit is contained in:
Joe Jevnik
@@ -15,6 +15,12 @@ The following methods are available for use in the ``initialize``,
|
||||
In all listed functions, the ``self`` argument is implicitly the
|
||||
currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance.
|
||||
|
||||
Data Object
|
||||
```````````
|
||||
|
||||
.. autoclass:: zipline.protocol.BarData
|
||||
:members:
|
||||
|
||||
Scheduling Functions
|
||||
````````````````````
|
||||
|
||||
@@ -271,6 +277,9 @@ Readers
|
||||
.. autoclass:: zipline.assets.AssetFinderCachedEquities
|
||||
:members:
|
||||
|
||||
.. autoclass:: zipline.data.data_portal.DataPortal
|
||||
:members:
|
||||
|
||||
Bundles
|
||||
```````
|
||||
.. autofunction:: zipline.data.bundles.register
|
||||
|
||||
@@ -12,6 +12,45 @@ Development
|
||||
Highlights
|
||||
~~~~~~~~~~
|
||||
|
||||
Zipline 1.0 Rewrite (:issue:`1105`)
|
||||
```````````````````````````````````
|
||||
|
||||
We have rewritten a lot of Zipline and its basic concepts in order to improve
|
||||
runtime performance. At the same time, we've introduced several new APIs.
|
||||
|
||||
At a high level, earlier versions of Zipline simulations pulled from a
|
||||
multiplexed stream of data sources, which were merged via heapq. This stream was
|
||||
fed to the main simulation loop, driving the clock forward. This strong
|
||||
dependency on reading all the data made it difficult to optimize simulation
|
||||
performance because there was no connection between the amount of data we
|
||||
fetched and the amount of data actually used by the algorithm.
|
||||
|
||||
Now, we only fetch data when the algorithm needs it. A new class,
|
||||
:class:`~zipline.data.data_portal.DataPortal`, dispatches data requests to
|
||||
various data sources and returns the requested values. This makes the runtime of
|
||||
a simulation scale much more closely with the complexity of the algorithm,
|
||||
rather than with the number of assets provided by the data sources.
|
||||
|
||||
Instead of the data stream driving the clock, now simulations iterate through a
|
||||
pre-calculated set of day or minute timestamps. The timestamps are emitted by
|
||||
:class:`~zipline.gens.sim_engine.MinuteSimulationClock` and
|
||||
:class:`~zipline.gens.sim_engine.DailySimulationClock`, and consumed by the main
|
||||
loop in :meth:`~zipline.gens.tradesimulation.AlgorithmSimulator.transform`.
|
||||
|
||||
We've retired the ``data[sid(N)]`` and ``history`` APIs, replacing them with
|
||||
several methods on the :class:`~zipline.protocol.BarData` object:
|
||||
:meth:`~zipline.protocol.BarData.current`,
|
||||
:meth:`~zipline.protocol.BarData.history`,
|
||||
:meth:`~zipline.protocol.BarData.can_trade`, and
|
||||
:meth:`~zipline.protocol.BarData.is_stale`. Old APIs will continue to work for
|
||||
now, but will issue deprecation warnings.
|
||||
|
||||
You can now pass in an adjustments source to the
|
||||
:class:`~zipline.data.data_portal.DataPortal`, and we will apply adjustments to
|
||||
the pricing data when looking backwards at data. Prices and volumes for
|
||||
execution and presented to the algorithm in data.current are the as-traded value
|
||||
of the asset.
|
||||
|
||||
New Entry Points (:issue:`1173` and :issue:`1178`)
|
||||
``````````````````````````````````````````````````
|
||||
|
||||
@@ -131,6 +170,9 @@ Enhancements
|
||||
implements the Bollinger Bands technical indicator:
|
||||
https://en.wikipedia.org/wiki/Bollinger_Bands (:issue:`1199`).
|
||||
|
||||
* Fetcher has been moved from Quantopian internal code into Zipline
|
||||
(:issue:`1105`).
|
||||
|
||||
|
||||
Experimental Features
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
+46
-32
@@ -14,12 +14,13 @@
|
||||
# limitations under the License.
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
|
||||
from pandas.tslib import normalize_date
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
from six import iteritems
|
||||
from six import iteritems, PY2
|
||||
from cpython cimport bool
|
||||
from collections import Iterable
|
||||
|
||||
@@ -31,6 +32,17 @@ cdef bool _is_iterable(obj):
|
||||
return isinstance(obj, Iterable) and not isinstance(obj, str)
|
||||
|
||||
|
||||
# Wraps doesn't work for method objects in python2. Docs should be generated
|
||||
# with python3 so it is not a big deal.
|
||||
if PY2:
|
||||
def no_wraps_py2(f):
|
||||
def dec(g):
|
||||
return g
|
||||
return dec
|
||||
else:
|
||||
no_wraps_py2 = wraps
|
||||
|
||||
|
||||
cdef class check_parameters(object):
|
||||
"""
|
||||
Asserts that the keywords passed into the wrapped function are included
|
||||
@@ -52,6 +64,7 @@ cdef class check_parameters(object):
|
||||
self.keys_to_types = dict(zip(keyword_names, types))
|
||||
|
||||
def __call__(self, func):
|
||||
@no_wraps_py2(func)
|
||||
def assert_keywords_and_call(*args, **kwargs):
|
||||
cdef short i
|
||||
|
||||
@@ -123,6 +136,27 @@ def handle_non_market_minutes(bar_data):
|
||||
|
||||
|
||||
cdef class BarData:
|
||||
"""
|
||||
Provides methods to access spot value or history windows of price data.
|
||||
Also provides some utility methods to determine if an asset is alive,
|
||||
has recent trade data, etc.
|
||||
|
||||
This is what is passed as ``data`` to the ``handle_data`` function.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_portal : DataPortal
|
||||
Provider for bar pricing data.
|
||||
simulation_dt_func : callable
|
||||
Function which returns the current simulation time.
|
||||
This is usually bound to a method of TradingSimulation.
|
||||
data_frequency : {'minute', 'daily'}
|
||||
The frequency of the bar data; i.e. whether the data is
|
||||
daily or minute bars
|
||||
universe_func : callable, optional
|
||||
Function which returns the current 'universe'. This is for
|
||||
backwards compatibility with older API concepts.
|
||||
"""
|
||||
cdef object data_portal
|
||||
cdef object simulation_dt_func
|
||||
cdef object data_frequency
|
||||
@@ -133,32 +167,10 @@ cdef class BarData:
|
||||
|
||||
cdef bool _adjust_minutes
|
||||
|
||||
"""
|
||||
Provides methods to access spot value or history windows of price data.
|
||||
Also provides some utility methods to determine if an asset is alive,
|
||||
has recent trade data, etc.
|
||||
|
||||
This is what is passed as `data` to the `handle_data` function.
|
||||
"""
|
||||
def __init__(self, data_portal, simulation_dt_func, data_frequency,
|
||||
universe_func=None):
|
||||
"""
|
||||
Parameters
|
||||
---------
|
||||
data_portal : DataPortal
|
||||
Provider for bar pricing data.
|
||||
|
||||
simulation_dt_func: function
|
||||
Function which returns the current simulation time.
|
||||
This is usually bound to a method of TradingSimulation.
|
||||
|
||||
data_frequency: string
|
||||
The frequency of the bar data; i.e. whether the data is
|
||||
'daily' or 'minute' bars
|
||||
|
||||
universe_func: function
|
||||
Function which returns the current 'universe'. This is for
|
||||
backwards compatibility with older API concepts.
|
||||
"""
|
||||
self.data_portal = data_portal
|
||||
self.simulation_dt_func = simulation_dt_func
|
||||
@@ -183,7 +195,7 @@ cdef class BarData:
|
||||
|
||||
Returns
|
||||
-------
|
||||
SidView: Accessor into the given asset's data.
|
||||
SidView : Accessor into the given asset's data.
|
||||
"""
|
||||
try:
|
||||
self._warn_deprecated("`data[sid(N)]` is deprecated. Use "
|
||||
@@ -226,14 +238,15 @@ cdef class BarData:
|
||||
Parameters
|
||||
----------
|
||||
assets : Asset or iterable of Assets
|
||||
|
||||
fields : string or iterable of strings. Valid values are: "price",
|
||||
fields : str or iterable[str].
|
||||
Valid values are: "price",
|
||||
"last_traded", "open", "high", "low", "close", "volume", or column
|
||||
names in files read by fetch_csv.
|
||||
names in files read by ``fetch_csv``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Scalar, pandas Series, or pandas DataFrame. See notes below.
|
||||
current_value : Scalar, pandas Series, or pandas DataFrame.
|
||||
See notes below.
|
||||
|
||||
Notes
|
||||
-----
|
||||
@@ -403,7 +416,7 @@ cdef class BarData:
|
||||
|
||||
Returns
|
||||
-------
|
||||
boolean or Series of booleans, indexed by asset.
|
||||
can_trade : bool or pd.Series[bool] indexed by asset.
|
||||
"""
|
||||
dt = self.simulation_dt_func()
|
||||
|
||||
@@ -522,8 +535,9 @@ cdef class BarData:
|
||||
|
||||
Returns
|
||||
-------
|
||||
Series or DataFrame or Panel, depending on the dimensionality of
|
||||
the 'assets' and 'fields' parameters.
|
||||
history : Series or DataFrame or Panel
|
||||
Return type depends on the dimensionality of the 'assets' and
|
||||
'fields' parameters.
|
||||
|
||||
If single asset and field are passed in, the returned Series is
|
||||
indexed by dt.
|
||||
@@ -720,7 +734,7 @@ cdef class SidView:
|
||||
cdef object data_portal
|
||||
cdef object simulation_dt_func
|
||||
cdef object data_frequency
|
||||
|
||||
|
||||
"""
|
||||
This class exists to temporarily support the deprecated data[sid(N)] API.
|
||||
"""
|
||||
|
||||
+71
-39
@@ -459,6 +459,39 @@ class DailyHistoryAggregator(object):
|
||||
|
||||
|
||||
class DataPortal(object):
|
||||
"""Interface to all of the data that a zipline simulation needs.
|
||||
|
||||
This is used by the simulation runner to answer questions about the data,
|
||||
like getting the prices of assets on a given day or to service history
|
||||
calls.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
env : TradingEnvironment
|
||||
The trading environment for the simulation. This includes the trading
|
||||
calendar and benchmark data.
|
||||
equity_daily_reader : BcolzDailyBarReader, optional
|
||||
The daily bar ready for equities. This will be used to service
|
||||
daily data backtests or daily history calls in a minute backetest.
|
||||
If a daily bar reader is not provided but a minute bar reader is,
|
||||
the minutes will be rolled up to serve the daily requests.
|
||||
equity_minute_reader : BcolzMinuteBarReader, optional
|
||||
The minute bar reader for equities. This will be used to service
|
||||
minute data backtests or minute history calls. This can be used
|
||||
to serve daily calls if no daily bar reader is provided.
|
||||
future_daily_reader : BcolzDailyBarReader, optional
|
||||
The daily bar ready for futures. This will be used to service
|
||||
daily data backtests or daily history calls in a minute backetest.
|
||||
If a daily bar reader is not provided but a minute bar reader is,
|
||||
the minutes will be rolled up to serve the daily requests.
|
||||
future_minute_reader : BcolzMinuteBarReader, optional
|
||||
The minute bar reader for futures. This will be used to service
|
||||
minute data backtests or minute history calls. This can be used
|
||||
to serve daily calls if no daily bar reader is provided.
|
||||
adjustment_reader : SQLiteAdjustmentWriter, optional
|
||||
The adjustment reader. This is used to apply splits, dividends, and
|
||||
other adjustment data to the raw data from the readers.
|
||||
"""
|
||||
def __init__(self,
|
||||
env,
|
||||
equity_daily_reader=None,
|
||||
@@ -696,24 +729,26 @@ class DataPortal(object):
|
||||
of the desired asset's field at either the given dt.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
----------
|
||||
asset : Asset
|
||||
The asset whose data is desired.
|
||||
|
||||
field: string
|
||||
The desired field of the asset. Valid values are "open", "high",
|
||||
"low", "close", "volume", "price", and "last_traded".
|
||||
|
||||
dt: pd.Timestamp
|
||||
field : {'open', 'high', 'low', 'close', 'volume',
|
||||
'price', 'last_traded'}
|
||||
The desired field of the asset.
|
||||
dt : pd.Timestamp
|
||||
The timestamp for the desired value.
|
||||
|
||||
data_frequency: string
|
||||
data_frequency : str
|
||||
The frequency of the data to query; i.e. whether the data is
|
||||
'daily' or 'minute' bars
|
||||
|
||||
Returns
|
||||
-------
|
||||
The value of the desired field at the desired time.
|
||||
value : float, int, or pd.Timestamp
|
||||
The spot value of ``field`` for ``asset`` The return type is based
|
||||
on the ``field`` requested. If the field is one of 'open', 'high',
|
||||
'low', 'close', or 'price', the value will be a float. If the
|
||||
``field`` is 'volume' the value will be a int. If the ``field`` is
|
||||
'last_traded' the value will be a Timestamp.
|
||||
"""
|
||||
if self._is_extra_source(asset, field, self._augmented_sources_map):
|
||||
return self._get_fetcher_value(asset, field, dt)
|
||||
@@ -755,28 +790,24 @@ class DataPortal(object):
|
||||
given field and list of assets
|
||||
|
||||
Parameters
|
||||
---------
|
||||
----------
|
||||
assets : list of type Asset, or Asset
|
||||
The asset, or assets whose adjustments are desired.
|
||||
|
||||
field: string
|
||||
The desired field of the asset. Valid values are "open",
|
||||
"open_price", "high", "low", "close", "close_price", "volume", and
|
||||
"price".
|
||||
|
||||
dt: pd.Timestamp
|
||||
field : {'open', 'high', 'low', 'close', 'volume', \
|
||||
'price', 'last_traded'}
|
||||
The desired field of the asset.
|
||||
dt : pd.Timestamp
|
||||
The timestamp for the desired value.
|
||||
|
||||
perspective_dt : pd.Timestamp
|
||||
The timestamp from which the data is being viewed back from.
|
||||
|
||||
data_frequency: string
|
||||
data_frequency : str
|
||||
The frequency of the data to query; i.e. whether the data is
|
||||
'daily' or 'minute' bars
|
||||
|
||||
Returns
|
||||
-------
|
||||
The list of adjustments for the asset(s)
|
||||
adjustments : list[Adjustment]
|
||||
The adjustments to that field.
|
||||
"""
|
||||
if isinstance(assets, Asset):
|
||||
assets = [assets]
|
||||
@@ -828,28 +859,29 @@ class DataPortal(object):
|
||||
of the desired asset's field at the given dt with adjustments applied.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
----------
|
||||
asset : Asset
|
||||
The asset whose data is desired.
|
||||
|
||||
field: string
|
||||
The desired field of the asset. Valid values are "open",
|
||||
"open_price", "high", "low", "close", "close_price", "volume", and
|
||||
"price".
|
||||
|
||||
dt: pd.Timestamp
|
||||
field : {'open', 'high', 'low', 'close', 'volume', \
|
||||
'price', 'last_traded'}
|
||||
The desired field of the asset.
|
||||
dt : pd.Timestamp
|
||||
The timestamp for the desired value.
|
||||
|
||||
perspective_dt : pd.Timestamp
|
||||
The timestamp from which the data is being viewed back from.
|
||||
|
||||
data_frequency: string
|
||||
data_frequency : str
|
||||
The frequency of the data to query; i.e. whether the data is
|
||||
'daily' or 'minute' bars
|
||||
|
||||
Returns
|
||||
-------
|
||||
The value of the desired field at the desired time.
|
||||
value : float, int, or pd.Timestamp
|
||||
The value of the given ``field`` for ``asset`` at ``dt`` with any
|
||||
adjustments known by ``perspective_dt`` applied. The return type is
|
||||
based on the ``field`` requested. If the field is one of 'open',
|
||||
'high', 'low', 'close', or 'price', the value will be a float. If
|
||||
the ``field`` is 'volume' the value will be a int. If the ``field``
|
||||
is 'last_traded' the value will be a Timestamp.
|
||||
"""
|
||||
if spot_value is None:
|
||||
# if this a fetcher field, we want to use perspective_dt (not dt)
|
||||
@@ -1172,7 +1204,7 @@ class DataPortal(object):
|
||||
history window. Data is fully adjusted.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
----------
|
||||
assets : list of zipline.data.Asset objects
|
||||
The assets whose data is desired.
|
||||
|
||||
@@ -1549,14 +1581,14 @@ class DataPortal(object):
|
||||
----------
|
||||
sids : container
|
||||
Sids for which we want splits.
|
||||
|
||||
dt: pd.Timestamp
|
||||
The date for which we are checking for splits. Note: this is
|
||||
dt : pd.Timestamp
|
||||
The date for which we are checking for splits. Note: this is
|
||||
expected to be midnight UTC.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list: List of splits, where each split is a (sid, ratio) tuple.
|
||||
splits : list[(int, float)]
|
||||
List of splits, where each split is a (sid, ratio) tuple.
|
||||
"""
|
||||
if self._adjustment_reader is None or not sids:
|
||||
return {}
|
||||
|
||||
+10
-10
@@ -528,11 +528,11 @@ class BcolzMinuteBarWriter(object):
|
||||
cols : dict of str -> np.array
|
||||
dict of market data with the following characteristics.
|
||||
keys are ('open', 'high', 'low', 'close', 'volume')
|
||||
open : float64
|
||||
high : float64
|
||||
low : float64
|
||||
close : float64
|
||||
volume : float64|int64
|
||||
open : float64
|
||||
high : float64
|
||||
low : float64
|
||||
close : float64
|
||||
volume : float64|int64
|
||||
"""
|
||||
if not all(len(dts) == len(cols[name]) for name in self.COL_NAMES):
|
||||
raise BcolzMinuteWriterColumnMismatch(
|
||||
@@ -555,11 +555,11 @@ class BcolzMinuteBarWriter(object):
|
||||
cols : dict of str -> np.array
|
||||
dict of market data with the following characteristics.
|
||||
keys are ('open', 'high', 'low', 'close', 'volume')
|
||||
open : float64
|
||||
high : float64
|
||||
low : float64
|
||||
close : float64
|
||||
volume : float64|int64
|
||||
open : float64
|
||||
high : float64
|
||||
low : float64
|
||||
close : float64
|
||||
volume : float64|int64
|
||||
"""
|
||||
table = self._ensure_ctable(sid)
|
||||
|
||||
|
||||
@@ -391,10 +391,41 @@ class BcolzDailyBarReader(DailyBarReader):
|
||||
"""
|
||||
Reader for raw pricing data written by BcolzDailyOHLCVWriter.
|
||||
|
||||
A Bcolz CTable is comprised of Columns and Attributes.
|
||||
Parameters
|
||||
----------
|
||||
table : bcolz.ctable
|
||||
The ctable contaning the pricing data, with attrs corresponding to the
|
||||
Attributes list below.
|
||||
read_all_threshold : int
|
||||
The number of equities at which; below, the data is read by reading a
|
||||
slice from the carray per asset. above, the data is read by pulling
|
||||
all of the data for all assets into memory and then indexing into that
|
||||
array for each day and asset pair. Used to tune performance of reads
|
||||
when using a small or large number of equities.
|
||||
|
||||
Columns
|
||||
-------
|
||||
Attributes
|
||||
----------
|
||||
The table with which this loader interacts contains the following
|
||||
attributes:
|
||||
|
||||
first_row : dict
|
||||
Map from asset_id -> index of first row in the dataset with that id.
|
||||
last_row : dict
|
||||
Map from asset_id -> index of last row in the dataset with that id.
|
||||
calendar_offset : dict
|
||||
Map from asset_id -> calendar index of first row.
|
||||
calendar : list[int64]
|
||||
Calendar used to compute offsets, in asi8 format (ns since EPOCH).
|
||||
|
||||
We use first_row and last_row together to quickly find ranges of rows to
|
||||
load when reading an asset's data into memory.
|
||||
|
||||
We use calendar_offset and calendar to orient loaded blocks within a
|
||||
range of queried dates.
|
||||
|
||||
Notes
|
||||
------
|
||||
A Bcolz CTable is comprised of Columns and Attributes.
|
||||
The table with which this loader interacts contains the following columns:
|
||||
|
||||
['open', 'high', 'low', 'close', 'volume', 'day', 'id'].
|
||||
@@ -419,41 +450,6 @@ class BcolzDailyBarReader(DailyBarReader):
|
||||
When read across the open, high, low, close, and volume with the same
|
||||
index should represent the same asset and day.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table : bcolz.ctable
|
||||
The ctable contaning the pricing data, with attrs corresponding to the
|
||||
Attributes list below.
|
||||
read_all_threshold : int
|
||||
The number of equities at which;
|
||||
below, the data is read by reading a slice from the carray
|
||||
per asset.
|
||||
above, the data is read by pulling all of the data for all assets
|
||||
into memory and then indexing into that array for each day and
|
||||
asset pair.
|
||||
Used to tune performance of reads when using a small or large number
|
||||
of equities.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
The table with which this loader interacts contains the following
|
||||
attributes:
|
||||
|
||||
first_row : dict
|
||||
Map from asset_id -> index of first row in the dataset with that id.
|
||||
last_row : dict
|
||||
Map from asset_id -> index of last row in the dataset with that id.
|
||||
calendar_offset : dict
|
||||
Map from asset_id -> calendar index of first row.
|
||||
calendar : list[int64]
|
||||
Calendar used to compute offsets, in asi8 format (ns since EPOCH).
|
||||
|
||||
We use first_row and last_row together to quickly find ranges of rows to
|
||||
load when reading an asset's data into memory.
|
||||
|
||||
We use calendar_offset and calendar to orient loaded blocks within a
|
||||
range of queried dates.
|
||||
|
||||
See Also
|
||||
--------
|
||||
zipline.data.us_equity_pricing.BcolzDailyBarWriter
|
||||
@@ -1182,7 +1178,7 @@ class SQLiteAdjustmentReader(object):
|
||||
|
||||
See Also
|
||||
--------
|
||||
zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
|
||||
:class:`zipline.data.us_equity_pricing.SQLiteAdjustmentWriter`
|
||||
"""
|
||||
|
||||
@preprocess(conn=coerce_string(sqlite3.connect))
|
||||
|
||||
Reference in New Issue
Block a user