Merge pull request #1599 from quantopian/memory-savings

Memory savings

zipline/algorithm.py

@@ -110,6 +110,7 @@ from zipline.utils.input_validation import (
 from zipline.utils.calendars.trading_calendar import days_at_time
 from zipline.utils.cache import CachedObject, Expired
 from zipline.utils.calendars import get_calendar
+from zipline.utils.compat import exc_clear
 
 import zipline.utils.events
 from zipline.utils.events import (
@@ -125,6 +126,7 @@ from zipline.utils.math_utils import (
     tolerant_equals,
     round_if_near_integer
 )
+from zipline.utils.pandas_utils import clear_dataframe_indexer_caches
 from zipline.utils.preprocess import preprocess
 from zipline.utils.security_list import SecurityList
 
@@ -2341,9 +2343,40 @@ class TradingAlgorithm(object):
         Internal implementation of `pipeline_output`.
         """
         today = normalize_date(self.get_datetime())
+        data = NO_DATA = object()
         try:
             data = self._pipeline_cache.unwrap(today)
         except Expired:
+            # We can't handle the exception in this block because in Python 3
+            # sys.exc_info isn't cleared until we leave the block.  See note
+            # below for why we need to clear exc_info.
+            pass
+
+        if data is NO_DATA:
+            # Try to deterministically garbage collect the previous result by
+            # removing any references to it. There are at least three sources
+            # of references:
+
+            # 1. self._pipeline_cache holds a reference.
+            # 2. The dataframe itself holds a reference via cached .iloc/.loc
+            #    accessors.
+            # 3. The traceback held in sys.exc_info includes stack frames in
+            #    which self._pipeline_cache is a local variable.
+
+            # We remove the above sources of references in reverse order:
+
+            # 3. Clear the traceback.  This is no-op in Python 3.
+            exc_clear()
+
+            # 2. Clear the .loc/.iloc caches.
+            clear_dataframe_indexer_caches(
+                self._pipeline_cache._unsafe_get_value()
+            )
+
+            # 1. Clear the reference to self._pipeline_cache.
+            self._pipeline_cache = None
+
+            # Calculate the next block.
             data, valid_until = self._run_pipeline(
                 pipeline, today, next(chunks),
             )

zipline/data/history_loader.py

@@ -380,7 +380,7 @@ class HistoryLoader(with_metaclass(ABCMeta)):
         assets = self._asset_finder.retrieve_all(assets)
 
         try:
-            end_ix = self._calendar.get_loc(end)
+            end_ix = self._calendar.searchsorted(end)
         except KeyError:
             raise KeyError("{0} not in calendar [{1}...{2}]".format(
                 end, self._calendar[0], self._calendar[-1]))
@@ -405,7 +405,7 @@ class HistoryLoader(with_metaclass(ABCMeta)):
 
             offset = 0
             try:
-                start_ix = self._calendar.get_loc(start)
+                start_ix = self._calendar.searchsorted(start)
             except KeyError:
                 raise KeyError("{0} not in calendar [{1}...{2}]".format(
                     start, self._calendar[0], self._calendar[-1]))
@@ -537,7 +537,7 @@ class HistoryLoader(with_metaclass(ABCMeta)):
                                              dts,
                                              field,
                                              is_perspective_after)
-        end_ix = self._calendar.get_loc(dts[-1])
+        end_ix = self._calendar.searchsorted(dts[-1])
 
         return concatenate(
             [window.get(end_ix) for window in block],

zipline/utils/cache.py

@@ -1,7 +1,7 @@
 """
 Caching utilities for zipline
 """
-from collections import namedtuple, MutableMapping
+from collections import MutableMapping
 import errno
 import os
 import pickle
@@ -20,7 +20,7 @@ class Expired(Exception):
     """
 
 
-class CachedObject(namedtuple("_CachedObject", "value expires")):
+class CachedObject(object):
     """
     A simple struct for maintaining a cached object with an expiration date.
 
@@ -47,6 +47,9 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
         ...
     Expired: 2014-01-01 00:00:00+00:00
     """
+    def __init__(self, value, expires):
+        self._value = value
+        self._expires = expires
 
     def unwrap(self, dt):
         """
@@ -62,9 +65,13 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
         Expired
             Raised when `dt` is greater than self.expires.
         """
-        if dt > self.expires:
-            raise Expired(self.expires)
-        return self.value
+        if dt > self._expires:
+            raise Expired(self._expires)
+        return self._value
+
+    def _unsafe_get_value(self):
+        """You almost certainly shouldn't use this."""
+        return self._value
 
 
 class ExpiringCache(object):

zipline/utils/calendars/trading_calendar.py

@@ -38,7 +38,7 @@ from zipline.utils.input_validation import (
     coerce,
     preprocess,
 )
-from zipline.utils.memoize import remember_last, lazyval
+from zipline.utils.memoize import lazyval
 
 start_default = pd.Timestamp('1990-01-01', tz='UTC')
 end_base = pd.Timestamp('today', tz='UTC')
@@ -655,8 +655,7 @@ class TradingCalendar(with_metaclass(ABCMeta)):
     def last_session(self):
         return self.all_sessions[-1]
 
-    @property
-    @remember_last
+    @lazyval
     def all_minutes(self):
         """
         Returns a DatetimeIndex representing all the minutes in this calendar.

zipline/utils/compat.py

@@ -1,4 +1,5 @@
 from six import PY2
+import sys
 
 
 if PY2:
@@ -8,9 +9,17 @@ if PY2:
     mappingproxy.argtypes = [py_object]
     mappingproxy.restype = py_object
 
+    def exc_clear():
+        sys.exc_clear()
+
 else:
     from types import MappingProxyType as mappingproxy
 
+    def exc_clear():
+        # exc_clear was removed in Python 3. The except statement automatically
+        # clears the exception.
+        pass
+
 
 unicode = type(u'')
 

zipline/utils/pandas_utils.py

@@ -166,3 +166,28 @@ def ignore_pandas_nan_categorical_warning():
             category=FutureWarning,
         )
         yield
+
+
+_INDEXER_NAMES = [
+    '_' + name for (name, _) in pd.core.indexing.get_indexers_list()
+]
+
+
+def clear_dataframe_indexer_caches(df):
+    """
+    Clear cached attributes from a pandas DataFrame.
+
+    By default pandas memoizes indexers (`iloc`, `loc`, `ix`, etc.) objects on
+    DataFrames, resulting in refcycles that can lead to unexpectedly long-lived
+    DataFrames. This function attempts to clear those cycles by deleting the
+    cached indexers from the frame.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+    """
+    for attr in _INDEXER_NAMES:
+        try:
+            delattr(df, attr)
+        except AttributeError:
+            pass