From 1248dcde368b9c0cacf990aa118c88b6892fd446 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Tue, 3 May 2016 15:08:03 -0400 Subject: [PATCH] PERF: Cap memory usage by minute bar carrays. Instead of letting the cache of carrays grow unbounded, use an LRUCache to cap the number of equities for any given column. Tested with the size 1000, on an algo that was using pipeline which was using over 3000, runtimes were similar, but the memory usage was successfully capped to around 1.2GB. Also, tested with an algorithm which bought and hold just one equity and no major slow down was seen when using the LRUCache vs. a dictionary. We may want to follow this up with an extension to `carray` which is not as memory hungry per column; e.g. by not loading repeated/similar metadata or releasing the last read chunk after a certain amount of time. --- zipline/data/minute_bars.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py index 83531c96..9574156a 100644 --- a/zipline/data/minute_bars.py +++ b/zipline/data/minute_bars.py @@ -16,6 +16,7 @@ import os from os.path import join from textwrap import dedent +from cachetools import LRUCache import bcolz from bcolz import ctable from intervaltree import IntervalTree @@ -628,8 +629,9 @@ class BcolzMinuteBarReader(object): -------- zipline.data.minute_bars.BcolzMinuteBarWriter """ - def __init__(self, rootdir): + FIELDS = ('open', 'high', 'low', 'close', 'volume') + def __init__(self, rootdir, sid_cache_size=1000): self._rootdir = rootdir metadata = self._get_metadata() @@ -646,11 +648,8 @@ class BcolzMinuteBarReader(object): self._ohlc_inverse = 1.0 / metadata.ohlc_ratio self._carrays = { - 'open': {}, - 'high': {}, - 'low': {}, - 'close': {}, - 'volume': {}, + field: LRUCache(maxsize=sid_cache_size) + for field in self.FIELDS } self._last_get_value_dt_position = None