From 52ed9093ebd5fa86ed406d87aafd68cf9826d5eb Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 15 Nov 2016 11:07:30 -0500 Subject: [PATCH 1/5] PERF: Deterministically GC pipeline results. Any DataFrame that's had `.loc` or `.iloc `called on it participates in a cycle, which means they're not immediately garbage collected when they go out of scope. This matters for pipeline results because they consume multiple megabytes per column, which means that a pipeline result with many columns can hold take up over 100MB. By manually breaking DataFrame cycles, we can ensure that we never hold multiple pipeline results in memory at once. --- zipline/algorithm.py | 26 ++++++++++++++++++++++++++ zipline/utils/cache.py | 17 ++++++++++++----- zipline/utils/pandas_utils.py | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index b2172c00..302366db 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -22,6 +22,7 @@ import pandas as pd from contextlib2 import ExitStack from pandas.tseries.tools import normalize_date import numpy as np +import sys from itertools import chain, repeat from numbers import Integral @@ -125,6 +126,7 @@ from zipline.utils.math_utils import ( tolerant_equals, round_if_near_integer ) +from zipline.utils.pandas_utils import clear_dataframe_indexer_caches from zipline.utils.preprocess import preprocess from zipline.utils.security_list import SecurityList @@ -2344,6 +2346,30 @@ class TradingAlgorithm(object): try: data = self._pipeline_cache.unwrap(today) except Expired: + # Try to deterministically garbage collect the previous result by + # removing any references to it. There are at least three sources + # of references: + + # 1. self._pipeline_cache holds a reference. + # 2. The dataframe itself holds a reference via cached .iloc/.loc + # accessors. + # 3. The traceback held in sys.exc_info includes stack frames in + # which self._pipeline_cache is a local variable. + + # We remove the above sources of references in reverse order: + + # 3. Clear the traceback. + sys.exc_clear() + + # 2. Clear the .loc/.iloc caches. + clear_dataframe_indexer_caches( + self._pipeline_cache._unsafe_get_value() + ) + + # 1. Clear the reference to self._pipeline_cache. + self._pipeline_cache = None + + # Calculate the next block. data, valid_until = self._run_pipeline( pipeline, today, next(chunks), ) diff --git a/zipline/utils/cache.py b/zipline/utils/cache.py index 427a67d6..f86d4cef 100644 --- a/zipline/utils/cache.py +++ b/zipline/utils/cache.py @@ -1,7 +1,7 @@ """ Caching utilities for zipline """ -from collections import namedtuple, MutableMapping +from collections import MutableMapping import errno import os import pickle @@ -20,7 +20,7 @@ class Expired(Exception): """ -class CachedObject(namedtuple("_CachedObject", "value expires")): +class CachedObject(object): """ A simple struct for maintaining a cached object with an expiration date. @@ -47,6 +47,9 @@ class CachedObject(namedtuple("_CachedObject", "value expires")): ... Expired: 2014-01-01 00:00:00+00:00 """ + def __init__(self, value, expires): + self._value = value + self._expires = expires def unwrap(self, dt): """ @@ -62,9 +65,13 @@ class CachedObject(namedtuple("_CachedObject", "value expires")): Expired Raised when `dt` is greater than self.expires. """ - if dt > self.expires: - raise Expired(self.expires) - return self.value + if dt > self._expires: + raise Expired(self._expires) + return self._value + + def _unsafe_get_value(self): + """You almost certainly shouldn't use this.""" + return self._value class ExpiringCache(object): diff --git a/zipline/utils/pandas_utils.py b/zipline/utils/pandas_utils.py index 84017a41..1b81897a 100644 --- a/zipline/utils/pandas_utils.py +++ b/zipline/utils/pandas_utils.py @@ -166,3 +166,22 @@ def ignore_pandas_nan_categorical_warning(): category=FutureWarning, ) yield + + +def clear_dataframe_indexer_caches(df): + """ + Clear cached attributes from a pandas DataFrame. + + By default pandas memoizes `iloc`, `loc` objects on DataFrames, resulting + in refcycles that can lead to unexpectedly long-lived DataFrames. This + function attempts to clear those cycles. + + Parameters + ---------- + df : pd.DataFrame + """ + for attr in ('_loc', '_iloc'): + try: + delattr(df, attr) + except AttributeError: + pass From 8aacac1cb662f5ca57b23aacc13387fa65c71e60 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 22 Nov 2016 12:33:58 -0500 Subject: [PATCH 2/5] PERF: Use searchsorted instead of get_loc. On pandas < 18, `get_loc` triggers allocation of a large hash table, so we don't want to call get_loc on minutely `DatetimeIndex`es. --- zipline/data/history_loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zipline/data/history_loader.py b/zipline/data/history_loader.py index ccea65ec..df0b2789 100644 --- a/zipline/data/history_loader.py +++ b/zipline/data/history_loader.py @@ -380,7 +380,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): assets = self._asset_finder.retrieve_all(assets) try: - end_ix = self._calendar.get_loc(end) + end_ix = self._calendar.searchsorted(end) except KeyError: raise KeyError("{0} not in calendar [{1}...{2}]".format( end, self._calendar[0], self._calendar[-1])) @@ -405,7 +405,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): offset = 0 try: - start_ix = self._calendar.get_loc(start) + start_ix = self._calendar.searchsorted(start) except KeyError: raise KeyError("{0} not in calendar [{1}...{2}]".format( start, self._calendar[0], self._calendar[-1])) @@ -537,7 +537,7 @@ class HistoryLoader(with_metaclass(ABCMeta)): dts, field, is_perspective_after) - end_ix = self._calendar.get_loc(dts[-1]) + end_ix = self._calendar.searchsorted(dts[-1]) return concatenate( [window.get(end_ix) for window in block], From 7a5f5da6b2995c194fca93367c5a700d044291eb Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 22 Nov 2016 12:35:10 -0500 Subject: [PATCH 3/5] MAINT: Use lazyval instead of two decorators. --- zipline/utils/calendars/trading_calendar.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zipline/utils/calendars/trading_calendar.py b/zipline/utils/calendars/trading_calendar.py index f17ec923..c98419fc 100644 --- a/zipline/utils/calendars/trading_calendar.py +++ b/zipline/utils/calendars/trading_calendar.py @@ -38,7 +38,7 @@ from zipline.utils.input_validation import ( coerce, preprocess, ) -from zipline.utils.memoize import remember_last, lazyval +from zipline.utils.memoize import lazyval start_default = pd.Timestamp('1990-01-01', tz='UTC') end_base = pd.Timestamp('today', tz='UTC') @@ -655,8 +655,7 @@ class TradingCalendar(with_metaclass(ABCMeta)): def last_session(self): return self.all_sessions[-1] - @property - @remember_last + @lazyval def all_minutes(self): """ Returns a DatetimeIndex representing all the minutes in this calendar. From 16e78bbccdaaec8434f5c0e2eaff39ea4bfacc39 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 22 Nov 2016 16:34:53 -0500 Subject: [PATCH 4/5] BUG: sys.exc_clear is py2 only. --- zipline/algorithm.py | 13 ++++++++++--- zipline/utils/compat.py | 9 +++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 302366db..c74256ab 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -22,7 +22,6 @@ import pandas as pd from contextlib2 import ExitStack from pandas.tseries.tools import normalize_date import numpy as np -import sys from itertools import chain, repeat from numbers import Integral @@ -111,6 +110,7 @@ from zipline.utils.input_validation import ( from zipline.utils.calendars.trading_calendar import days_at_time from zipline.utils.cache import CachedObject, Expired from zipline.utils.calendars import get_calendar +from zipline.utils.compat import exc_clear import zipline.utils.events from zipline.utils.events import ( @@ -2343,9 +2343,16 @@ class TradingAlgorithm(object): Internal implementation of `pipeline_output`. """ today = normalize_date(self.get_datetime()) + data = NO_DATA = object() try: data = self._pipeline_cache.unwrap(today) except Expired: + # We can't handle the exception in this block because in Python 3 + # sys.exc_info isn't cleared until we leave the block. See note + # below for why we need to clear exc_info. + pass + + if data is NO_DATA: # Try to deterministically garbage collect the previous result by # removing any references to it. There are at least three sources # of references: @@ -2358,8 +2365,8 @@ class TradingAlgorithm(object): # We remove the above sources of references in reverse order: - # 3. Clear the traceback. - sys.exc_clear() + # 3. Clear the traceback. This is no-op in Python 3. + exc_clear() # 2. Clear the .loc/.iloc caches. clear_dataframe_indexer_caches( diff --git a/zipline/utils/compat.py b/zipline/utils/compat.py index cf82ba62..08693a11 100644 --- a/zipline/utils/compat.py +++ b/zipline/utils/compat.py @@ -1,4 +1,5 @@ from six import PY2 +import sys if PY2: @@ -8,9 +9,17 @@ if PY2: mappingproxy.argtypes = [py_object] mappingproxy.restype = py_object + def exc_clear(): + sys.exc_clear() + else: from types import MappingProxyType as mappingproxy + def exc_clear(): + # exc_clear was removed in Python 3. The except statement automatically + # clears the exception. + pass + unicode = type(u'') From 9f28b7cede30f5f4f60e7329ac39f8191e1b6328 Mon Sep 17 00:00:00 2001 From: Scott Sanderson Date: Tue, 22 Nov 2016 17:11:04 -0500 Subject: [PATCH 5/5] MAINT: Hit more dataframe indexers. --- zipline/utils/pandas_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/zipline/utils/pandas_utils.py b/zipline/utils/pandas_utils.py index 1b81897a..633960e1 100644 --- a/zipline/utils/pandas_utils.py +++ b/zipline/utils/pandas_utils.py @@ -168,19 +168,25 @@ def ignore_pandas_nan_categorical_warning(): yield +_INDEXER_NAMES = [ + '_' + name for (name, _) in pd.core.indexing.get_indexers_list() +] + + def clear_dataframe_indexer_caches(df): """ Clear cached attributes from a pandas DataFrame. - By default pandas memoizes `iloc`, `loc` objects on DataFrames, resulting - in refcycles that can lead to unexpectedly long-lived DataFrames. This - function attempts to clear those cycles. + By default pandas memoizes indexers (`iloc`, `loc`, `ix`, etc.) objects on + DataFrames, resulting in refcycles that can lead to unexpectedly long-lived + DataFrames. This function attempts to clear those cycles by deleting the + cached indexers from the frame. Parameters ---------- df : pd.DataFrame """ - for attr in ('_loc', '_iloc'): + for attr in _INDEXER_NAMES: try: delattr(df, attr) except AttributeError: