mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-13 17:42:42 +08:00
PERF: Cap memory usage by minute bar carrays.
Instead of letting the cache of carrays grow unbounded, use an LRUCache to cap the number of equities for any given column. Tested with the size 1000, on an algo that was using pipeline which was using over 3000, runtimes were similar, but the memory usage was successfully capped to around 1.2GB. Also, tested with an algorithm which bought and hold just one equity and no major slow down was seen when using the LRUCache vs. a dictionary. We may want to follow this up with an extension to `carray` which is not as memory hungry per column; e.g. by not loading repeated/similar metadata or releasing the last read chunk after a certain amount of time.
This commit is contained in:
@@ -16,6 +16,7 @@ import os
|
||||
from os.path import join
|
||||
from textwrap import dedent
|
||||
|
||||
from cachetools import LRUCache
|
||||
import bcolz
|
||||
from bcolz import ctable
|
||||
from intervaltree import IntervalTree
|
||||
@@ -628,8 +629,9 @@ class BcolzMinuteBarReader(object):
|
||||
--------
|
||||
zipline.data.minute_bars.BcolzMinuteBarWriter
|
||||
"""
|
||||
def __init__(self, rootdir):
|
||||
FIELDS = ('open', 'high', 'low', 'close', 'volume')
|
||||
|
||||
def __init__(self, rootdir, sid_cache_size=1000):
|
||||
self._rootdir = rootdir
|
||||
|
||||
metadata = self._get_metadata()
|
||||
@@ -646,11 +648,8 @@ class BcolzMinuteBarReader(object):
|
||||
self._ohlc_inverse = 1.0 / metadata.ohlc_ratio
|
||||
|
||||
self._carrays = {
|
||||
'open': {},
|
||||
'high': {},
|
||||
'low': {},
|
||||
'close': {},
|
||||
'volume': {},
|
||||
field: LRUCache(maxsize=sid_cache_size)
|
||||
for field in self.FIELDS
|
||||
}
|
||||
|
||||
self._last_get_value_dt_position = None
|
||||
|
||||
Reference in New Issue
Block a user