From 6486e645305e172a4ab0644546cfbdb1cb9fdfa7 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Mon, 19 Sep 2016 08:53:31 -0400 Subject: [PATCH] Revert "Merge pull request #1490 from quantopian/use-load-adjustments-for-history" This reverts commit 86c7635b451a47e733a3d5c944da11b59d78ec41, reversing changes made to c77f2b92df218dc71311e0fbeff408675a065c84. Some real world cases hit errors with this change, due to the new offset logic attempting to create Adjustments with invalid parameters. Will identify exact conditions that cause this error and add as a test case before remerging. --- zipline/data/_adjustments.pyx | 47 ++++++------------ zipline/data/history_loader.py | 80 +++++++++++++++++++++++++++++-- zipline/data/us_equity_pricing.py | 5 +- 3 files changed, 90 insertions(+), 42 deletions(-) diff --git a/zipline/data/_adjustments.pyx b/zipline/data/_adjustments.pyx index 3ec36bc5..c7900385 100644 --- a/zipline/data/_adjustments.pyx +++ b/zipline/data/_adjustments.pyx @@ -147,9 +147,7 @@ cdef _adjustments(object adjustments_db, cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection list columns, DatetimeIndex_t dates, - Int64Index_t assets, - int end_offset, - int date_offset): + Int64Index_t assets): """ Load a dictionary of Adjustment objects from adjustments_db @@ -164,11 +162,6 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection Dates for which adjustments are needed assets : pd.Int64Index Assets for which adjustments are needed. - end_offset : int - Offset to apply when calculating the last row of the adjustment. - date_offset : int - Offset to apply when calculating the date on which to first apply the - adjustment. Returns ------- @@ -217,7 +210,7 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection int sid double ratio int eff_date - int date_loc, end_loc, out_loc + int date_loc Py_ssize_t asset_ix dict col_adjustments @@ -235,30 +228,26 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds) - end_loc = date_loc + end_offset - - out_loc = date_loc + date_offset - if not PyDict_Contains(asset_ixs, sid): asset_ixs[sid] = assets.get_loc(sid) asset_ix = asset_ixs[sid] - price_adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio) + price_adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio) for i, column in enumerate(columns): col_adjustments = results[i] if column != 'volume': try: - col_adjustments[out_loc].append(price_adj) + col_adjustments[date_loc].append(price_adj) except KeyError: - col_adjustments[out_loc] = [price_adj] + col_adjustments[date_loc] = [price_adj] else: volume_adj = Float64Multiply( - 0, end_loc, asset_ix, asset_ix, 1.0 / ratio + 0, date_loc, asset_ix, asset_ix, 1.0 / ratio ) try: - col_adjustments[out_loc].append(volume_adj) + col_adjustments[date_loc].append(volume_adj) except KeyError: - col_adjustments[out_loc] = [volume_adj] + col_adjustments[date_loc] = [volume_adj] # mergers affect prices only for sid, ratio, eff_date in mergers: @@ -267,22 +256,18 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds) - end_loc = date_loc + end_offset - - out_loc = date_loc + date_offset - if not PyDict_Contains(asset_ixs, sid): asset_ixs[sid] = assets.get_loc(sid) asset_ix = asset_ixs[sid] - adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio) + adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio) for i, column in enumerate(columns): col_adjustments = results[i] if column != 'volume': try: - col_adjustments[out_loc].append(adj) + col_adjustments[date_loc].append(adj) except KeyError: - col_adjustments[out_loc] = [adj] + col_adjustments[date_loc] = [adj] # dividends affect prices only for sid, ratio, eff_date in dividends: @@ -291,22 +276,18 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection date_loc = _lookup_dt(date_ixs, eff_date, _dates_seconds) - end_loc = date_loc + end_offset - - out_loc = date_loc + date_offset - if not PyDict_Contains(asset_ixs, sid): asset_ixs[sid] = assets.get_loc(sid) asset_ix = asset_ixs[sid] - adj = Float64Multiply(0, end_loc, asset_ix, asset_ix, ratio) + adj = Float64Multiply(0, date_loc, asset_ix, asset_ix, ratio) for i, column in enumerate(columns): col_adjustments = results[i] if column != 'volume': try: - col_adjustments[out_loc].append(adj) + col_adjustments[date_loc].append(adj) except KeyError: - col_adjustments[out_loc] = [adj] + col_adjustments[date_loc] = [adj] return results diff --git a/zipline/data/history_loader.py b/zipline/data/history_loader.py index 753b1a3f..ab31df80 100644 --- a/zipline/data/history_loader.py +++ b/zipline/data/history_loader.py @@ -20,11 +20,12 @@ from abc import ( from lru import LRU from numpy import around, hstack -from pandas import Int64Index +from pandas.tslib import normalize_date from six import with_metaclass from zipline.lib._float64window import AdjustedArrayWindow as Float64Window +from zipline.lib.adjustment import Float64Multiply from zipline.utils.cache import ExpiringCache from zipline.utils.memoize import lazyval from zipline.utils.numpy_utils import float64_dtype @@ -140,10 +141,79 @@ class HistoryLoader(with_metaclass(ABCMeta)): ------- out : The adjustments as a dict of loc -> Float64Multiply """ - end_offset = -1 - date_offset = -1 if is_perspective_after else 0 - return self._adjustments_reader.load_adjustments( - [field], dts, Int64Index([asset]), end_offset, date_offset)[0] + sid = int(asset) + start = normalize_date(dts[0]) + end = normalize_date(dts[-1]) + adjs = {} + if field != 'volume': + mergers = self._adjustments_reader.get_adjustments_for_sid( + 'mergers', sid) + for m in mergers: + dt = m[0] + if start < dt <= end: + end_loc = dts.searchsorted(dt) + adj_loc = end_loc + if is_perspective_after: + # Set adjustment pop location so that it applies + # to last value if adjustment occurs immediately after + # the last slot. + adj_loc -= 1 + mult = Float64Multiply(0, + end_loc - 1, + 0, + 0, + m[1]) + try: + adjs[adj_loc].append(mult) + except KeyError: + adjs[adj_loc] = [mult] + divs = self._adjustments_reader.get_adjustments_for_sid( + 'dividends', sid) + for d in divs: + dt = d[0] + if start < dt <= end: + end_loc = dts.searchsorted(dt) + adj_loc = end_loc + if is_perspective_after: + # Set adjustment pop location so that it applies + # to last value if adjustment occurs immediately after + # the last slot. + adj_loc -= 1 + mult = Float64Multiply(0, + end_loc - 1, + 0, + 0, + d[1]) + try: + adjs[adj_loc].append(mult) + except KeyError: + adjs[adj_loc] = [mult] + splits = self._adjustments_reader.get_adjustments_for_sid( + 'splits', sid) + for s in splits: + dt = s[0] + if start < dt <= end: + if field == 'volume': + ratio = 1.0 / s[1] + else: + ratio = s[1] + end_loc = dts.searchsorted(dt) + adj_loc = end_loc + if is_perspective_after: + # Set adjustment pop location so that it applies + # to last value if adjustment occurs immediately after + # the last slot. + adj_loc -= 1 + mult = Float64Multiply(0, + end_loc - 1, + 0, + 0, + ratio) + try: + adjs[adj_loc].append(mult) + except KeyError: + adjs[adj_loc] = [mult] + return adjs def _ensure_sliding_windows(self, assets, dts, field, is_perspective_after): diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py index 7c44607d..16771cc1 100644 --- a/zipline/data/us_equity_pricing.py +++ b/zipline/data/us_equity_pricing.py @@ -1234,15 +1234,12 @@ class SQLiteAdjustmentReader(object): def __init__(self, conn): self.conn = conn - def load_adjustments(self, columns, dates, assets, - end_offset=0, date_offset=0): + def load_adjustments(self, columns, dates, assets): return load_adjustments_from_sqlite( self.conn, list(columns), dates, assets, - end_offset, - date_offset ) def get_adjustments_for_sid(self, table_name, sid):