mirror of
https://github.com/wassname/catalyst.git
synced 2026-09-09 11:19:23 +08:00
PERF: Deterministically GC pipeline results.
Any DataFrame that's had `.loc` or `.iloc `called on it participates in a cycle, which means they're not immediately garbage collected when they go out of scope. This matters for pipeline results because they consume multiple megabytes per column, which means that a pipeline result with many columns can hold take up over 100MB. By manually breaking DataFrame cycles, we can ensure that we never hold multiple pipeline results in memory at once.
This commit is contained in:
+12
-5
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Caching utilities for zipline
|
||||
"""
|
||||
from collections import namedtuple, MutableMapping
|
||||
from collections import MutableMapping
|
||||
import errno
|
||||
import os
|
||||
import pickle
|
||||
@@ -20,7 +20,7 @@ class Expired(Exception):
|
||||
"""
|
||||
|
||||
|
||||
class CachedObject(namedtuple("_CachedObject", "value expires")):
|
||||
class CachedObject(object):
|
||||
"""
|
||||
A simple struct for maintaining a cached object with an expiration date.
|
||||
|
||||
@@ -47,6 +47,9 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
|
||||
...
|
||||
Expired: 2014-01-01 00:00:00+00:00
|
||||
"""
|
||||
def __init__(self, value, expires):
|
||||
self._value = value
|
||||
self._expires = expires
|
||||
|
||||
def unwrap(self, dt):
|
||||
"""
|
||||
@@ -62,9 +65,13 @@ class CachedObject(namedtuple("_CachedObject", "value expires")):
|
||||
Expired
|
||||
Raised when `dt` is greater than self.expires.
|
||||
"""
|
||||
if dt > self.expires:
|
||||
raise Expired(self.expires)
|
||||
return self.value
|
||||
if dt > self._expires:
|
||||
raise Expired(self._expires)
|
||||
return self._value
|
||||
|
||||
def _unsafe_get_value(self):
|
||||
"""You almost certainly shouldn't use this."""
|
||||
return self._value
|
||||
|
||||
|
||||
class ExpiringCache(object):
|
||||
|
||||
Reference in New Issue
Block a user