diff --git a/zipline/algorithm.py b/zipline/algorithm.py index b2172c00..c74256ab 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -110,6 +110,7 @@ from zipline.utils.input_validation import ( from zipline.utils.calendars.trading_calendar import days_at_time from zipline.utils.cache import CachedObject, Expired from zipline.utils.calendars import get_calendar +from zipline.utils.compat import exc_clear import zipline.utils.events from zipline.utils.events import ( @@ -125,6 +126,7 @@ from zipline.utils.math_utils import ( tolerant_equals, round_if_near_integer ) +from zipline.utils.pandas_utils import clear_dataframe_indexer_caches from zipline.utils.preprocess import preprocess from zipline.utils.security_list import SecurityList @@ -2341,9 +2343,40 @@ class TradingAlgorithm(object): Internal implementation of `pipeline_output`. """ today = normalize_date(self.get_datetime()) + data = NO_DATA = object() try: data = self._pipeline_cache.unwrap(today) except Expired: + # We can't handle the exception in this block because in Python 3 + # sys.exc_info isn't cleared until we leave the block. See note + # below for why we need to clear exc_info. + pass + + if data is NO_DATA: + # Try to deterministically garbage collect the previous result by + # removing any references to it. There are at least three sources + # of references: + + # 1. self._pipeline_cache holds a reference. + # 2. The dataframe itself holds a reference via cached .iloc/.loc + # accessors. + # 3. The traceback held in sys.exc_info includes stack frames in + # which self._pipeline_cache is a local variable. + + # We remove the above sources of references in reverse order: + + # 3. Clear the traceback. This is no-op in Python 3. + exc_clear() + + # 2. Clear the .loc/.iloc caches. + clear_dataframe_indexer_caches( + self._pipeline_cache._unsafe_get_value() + ) + + # 1. Clear the reference to self._pipeline_cache. + self._pipeline_cache = None + + # Calculate the next block. data, valid_until = self._run_pipeline( pipeline, today, next(chunks), ) diff --git a/zipline/data/history_loader.py b/zipline/data/history_loader.py index ccea65ec..df0b2789 100644 --- a/zipline/data/history_loader.py +++ b/zipline/data/history_loader.py @@ -380,7 +380,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): assets = self._asset_finder.retrieve_all(assets) try: - end_ix = self._calendar.get_loc(end) + end_ix = self._calendar.searchsorted(end) except KeyError: raise KeyError("{0} not in calendar [{1}...{2}]".format( end, self._calendar[0], self._calendar[-1])) @@ -405,7 +405,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): offset = 0 try: - start_ix = self._calendar.get_loc(start) + start_ix = self._calendar.searchsorted(start) except KeyError: raise KeyError("{0} not in calendar [{1}...{2}]".format( start, self._calendar[0], self._calendar[-1])) @@ -537,7 +537,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): dts, field, is_perspective_after) - end_ix = self._calendar.get_loc(dts[-1]) + end_ix = self._calendar.searchsorted(dts[-1]) return concatenate( [window.get(end_ix) for window in block], diff --git a/zipline/utils/cache.py b/zipline/utils/cache.py index 427a67d6..f86d4cef 100644 --- a/zipline/utils/cache.py +++ b/zipline/utils/cache.py @@ -1,7 +1,7 @@ """ Caching utilities for zipline """ -from collections import namedtuple, MutableMapping +from collections import MutableMapping import errno import os import pickle @@ -20,7 +20,7 @@ class Expired(Exception): """ -class CachedObject(namedtuple("_CachedObject", "value expires")): +class CachedObject(object): """ A simple struct for maintaining a cached object with an expiration date. @@ -47,6 +47,9 @@ class CachedObject(namedtuple("_CachedObject", "value expires")): ... Expired: 2014-01-01 00:00:00+00:00 """ + def __init__(self, value, expires): + self._value = value + self._expires = expires def unwrap(self, dt): """ @@ -62,9 +65,13 @@ class CachedObject(namedtuple("_CachedObject", "value expires")): Expired Raised when `dt` is greater than self.expires. """ - if dt > self.expires: - raise Expired(self.expires) - return self.value + if dt > self._expires: + raise Expired(self._expires) + return self._value + + def _unsafe_get_value(self): + """You almost certainly shouldn't use this.""" + return self._value class ExpiringCache(object): diff --git a/zipline/utils/calendars/trading_calendar.py b/zipline/utils/calendars/trading_calendar.py index f17ec923..c98419fc 100644 --- a/zipline/utils/calendars/trading_calendar.py +++ b/zipline/utils/calendars/trading_calendar.py @@ -38,7 +38,7 @@ from zipline.utils.input_validation import ( coerce, preprocess, ) -from zipline.utils.memoize import remember_last, lazyval +from zipline.utils.memoize import lazyval start_default = pd.Timestamp('1990-01-01', tz='UTC') end_base = pd.Timestamp('today', tz='UTC') @@ -655,8 +655,7 @@ class TradingCalendar(with_metaclass(ABCMeta)): def last_session(self): return self.all_sessions[-1] - @property - @remember_last + @lazyval def all_minutes(self): """ Returns a DatetimeIndex representing all the minutes in this calendar. diff --git a/zipline/utils/compat.py b/zipline/utils/compat.py index cf82ba62..08693a11 100644 --- a/zipline/utils/compat.py +++ b/zipline/utils/compat.py @@ -1,4 +1,5 @@ from six import PY2 +import sys if PY2: @@ -8,9 +9,17 @@ if PY2: mappingproxy.argtypes = [py_object] mappingproxy.restype = py_object + def exc_clear(): + sys.exc_clear() + else: from types import MappingProxyType as mappingproxy + def exc_clear(): + # exc_clear was removed in Python 3. The except statement automatically + # clears the exception. + pass + unicode = type(u'') diff --git a/zipline/utils/pandas_utils.py b/zipline/utils/pandas_utils.py index 84017a41..633960e1 100644 --- a/zipline/utils/pandas_utils.py +++ b/zipline/utils/pandas_utils.py @@ -166,3 +166,28 @@ def ignore_pandas_nan_categorical_warning(): category=FutureWarning, ) yield + + +_INDEXER_NAMES = [ + '_' + name for (name, _) in pd.core.indexing.get_indexers_list() +] + + +def clear_dataframe_indexer_caches(df): + """ + Clear cached attributes from a pandas DataFrame. + + By default pandas memoizes indexers (`iloc`, `loc`, `ix`, etc.) objects on + DataFrames, resulting in refcycles that can lead to unexpectedly long-lived + DataFrames. This function attempts to clear those cycles by deleting the + cached indexers from the frame. + + Parameters + ---------- + df : pd.DataFrame + """ + for attr in _INDEXER_NAMES: + try: + delattr(df, attr) + except AttributeError: + pass