PERF: Deterministically GC pipeline results.

Any DataFrame that's had `.loc` or `.iloc `called on it participates in
a cycle, which means they're not immediately garbage collected when they
go out of scope.  This matters for pipeline results because they consume
multiple megabytes per column, which means that a pipeline result with
many columns can hold take up over 100MB.  By manually breaking
DataFrame cycles, we can ensure that we never hold multiple pipeline
results in memory at once.
This commit is contained in:
Scott Sanderson
2016-11-22 14:26:58 -05:00
parent f3a36fe97f
commit 52ed9093eb
3 changed files with 57 additions and 5 deletions
+12 -5
View File
@@ -1,7 +1,7 @@
"""
Caching utilities for zipline
"""
from collections import namedtuple, MutableMapping
from collections import MutableMapping
import errno
import os
import pickle
@@ -20,7 +20,7 @@ class Expired(Exception):
"""
class CachedObject(namedtuple("_CachedObject", "value expires")):
class CachedObject(object):
"""
A simple struct for maintaining a cached object with an expiration date.
@@ -47,6 +47,9 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
...
Expired: 2014-01-01 00:00:00+00:00
"""
def __init__(self, value, expires):
self._value = value
self._expires = expires
def unwrap(self, dt):
"""
@@ -62,9 +65,13 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
Expired
Raised when `dt` is greater than self.expires.
"""
if dt > self.expires:
raise Expired(self.expires)
return self.value
if dt > self._expires:
raise Expired(self._expires)
return self._value
def _unsafe_get_value(self):
"""You almost certainly shouldn't use this."""
return self._value
class ExpiringCache(object):