wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-04 05:50:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge pull request #1490 from quantopian/use-load-adjustments-for-history"

Browse Source 
This reverts commit 86c7635b45, reversing
changes made to c77f2b92df.

Some real world cases hit errors with this change, due to the new offset
logic attempting to create Adjustments with invalid parameters.

Will identify exact conditions that cause this error and add as a test
case before remerging.
This commit is contained in:
Eddie Hebert
2016-09-19 08:53:31 -04:00
parent e21fa37e22
commit 6486e64530
3 changed files with 90 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zipline/data/_adjustments.pyx
+14 -33
View File
@@ -147,9 +147,7 @@ cdef _adjustments(object adjustments_db,
cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
list columns,
DatetimeIndex_t dates,
Int64Index_t assets,
int end_offset,
int date_offset):
Int64Index_t assets):
"""
Load a dictionary of Adjustment objects from adjustments_db
@@ -164,11 +162,6 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
Dates for which adjustments are needed
assets : pd.Int64Index
Assets for which adjustments are needed.
end_offset : int
Offset to apply when calculating the last row of the adjustment.
date_offset : int
Offset to apply when calculating the date on which to first apply the
adjustment.
Returns
-------
@@ -217,7 +210,7 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
int sid
double ratio
int eff_date
int date_loc, end_loc, out_loc
int date_loc
Py_ssize_t asset_ix
dict col_adjustments
@@ -235,30 +228,26 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
end_loc = date_loc + end_offset
out_loc = date_loc + date_offset
if not PyDict_Contains(asset_ixs, sid):
asset_ixs[sid] = assets.get_loc(sid)
asset_ix = asset_ixs[sid]
price_adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio)
price_adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
for i, column in enumerate(columns):
col_adjustments = results[i]
if column != 'volume':
try:
col_adjustments[out_loc].append(price_adj)
col_adjustments[date_loc].append(price_adj)
except KeyError:
col_adjustments[out_loc] = [price_adj]
col_adjustments[date_loc] = [price_adj]
else:
volume_adj = Float64Multiply(
0, end_loc, asset_ix, asset_ix, 1.0 / ratio
0, date_loc, asset_ix, asset_ix, 1.0 / ratio
)
try:
col_adjustments[out_loc].append(volume_adj)
col_adjustments[date_loc].append(volume_adj)
except KeyError:
col_adjustments[out_loc] = [volume_adj]
col_adjustments[date_loc] = [volume_adj]
# mergers affect prices only
for sid, ratio, eff_date in mergers:
@@ -267,22 +256,18 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
end_loc = date_loc + end_offset
out_loc = date_loc + date_offset
if not PyDict_Contains(asset_ixs, sid):
asset_ixs[sid] = assets.get_loc(sid)
asset_ix = asset_ixs[sid]
adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio)
adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
for i, column in enumerate(columns):
col_adjustments = results[i]
if column != 'volume':
try:
col_adjustments[out_loc].append(adj)
col_adjustments[date_loc].append(adj)
except KeyError:
col_adjustments[out_loc] = [adj]
col_adjustments[date_loc] = [adj]
# dividends affect prices only
for sid, ratio, eff_date in dividends:
@@ -291,22 +276,18 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection
date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds)
end_loc = date_loc + end_offset
out_loc = date_loc + date_offset
if not PyDict_Contains(asset_ixs, sid):
asset_ixs[sid] = assets.get_loc(sid)
asset_ix = asset_ixs[sid]
adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio)
adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio)
for i, column in enumerate(columns):
col_adjustments = results[i]
if column != 'volume':
try:
col_adjustments[out_loc].append(adj)
col_adjustments[date_loc].append(adj)
except KeyError:
col_adjustments[out_loc] = [adj]
col_adjustments[date_loc] = [adj]
return results
zipline/data/history_loader.py
+75 -5
View File
@@ -20,11 +20,12 @@ from abc import (
from lru import LRU
from numpy import around, hstack
from pandas import Int64Index
from pandas.tslib import normalize_date
from six import with_metaclass
from zipline.lib._float64window import AdjustedArrayWindow as Float64Window
from zipline.lib.adjustment import Float64Multiply
from zipline.utils.cache import ExpiringCache
from zipline.utils.memoize import lazyval
from zipline.utils.numpy_utils import float64_dtype
@@ -140,10 +141,79 @@ class HistoryLoader(with_metaclass(ABCMeta)):
-------
out : The adjustments as a dict of loc -> Float64Multiply
"""
end_offset = -1
date_offset = -1 if is_perspective_after else 0
return self._adjustments_reader.load_adjustments(
[field], dts, Int64Index([asset]), end_offset, date_offset)[0]
sid = int(asset)
start = normalize_date(dts[0])
end = normalize_date(dts[-1])
adjs = {}
if field != 'volume':
mergers = self._adjustments_reader.get_adjustments_for_sid(
'mergers', sid)
for m in mergers:
dt = m[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
if is_perspective_after:
# Set adjustment pop location so that it applies
# to last value if adjustment occurs immediately after
# the last slot.
adj_loc -= 1
mult = Float64Multiply(0,
end_loc - 1,
0,
0,
m[1])
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
divs = self._adjustments_reader.get_adjustments_for_sid(
'dividends', sid)
for d in divs:
dt = d[0]
if start < dt <= end:
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
if is_perspective_after:
# Set adjustment pop location so that it applies
# to last value if adjustment occurs immediately after
# the last slot.
adj_loc -= 1
mult = Float64Multiply(0,
end_loc - 1,
0,
0,
d[1])
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
splits = self._adjustments_reader.get_adjustments_for_sid(
'splits', sid)
for s in splits:
dt = s[0]
if start < dt <= end:
if field == 'volume':
ratio = 1.0 / s[1]
else:
ratio = s[1]
end_loc = dts.searchsorted(dt)
adj_loc = end_loc
if is_perspective_after:
# Set adjustment pop location so that it applies
# to last value if adjustment occurs immediately after
# the last slot.
adj_loc -= 1
mult = Float64Multiply(0,
end_loc - 1,
0,
0,
ratio)
try:
adjs[adj_loc].append(mult)
except KeyError:
adjs[adj_loc] = [mult]
return adjs
def _ensure_sliding_windows(self, assets, dts, field,
is_perspective_after):
zipline/data/us_equity_pricing.py
+1 -4
View File
@@ -1234,15 +1234,12 @@ class SQLiteAdjustmentReader(object):
def __init__(self, conn):
self.conn = conn
def load_adjustments(self, columns, dates, assets,
end_offset=0, date_offset=0):
def load_adjustments(self, columns, dates, assets):
return load_adjustments_from_sqlite(
self.conn,
list(columns),
dates,
assets,
end_offset,
date_offset
)
def get_adjustments_for_sid(self, table_name, sid):