From ddb3113d6b8d007f66e8e7ad9e4a7456ddfec518 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 14 Apr 2016 11:24:17 -0400 Subject: [PATCH 1/2] BLD: Added cachetools conda recipe --- conda/cachetools/bld.bat | 8 +++++ conda/cachetools/build.sh | 9 ++++++ conda/cachetools/meta.yaml | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 conda/cachetools/bld.bat create mode 100644 conda/cachetools/build.sh create mode 100644 conda/cachetools/meta.yaml diff --git a/conda/cachetools/bld.bat b/conda/cachetools/bld.bat new file mode 100644 index 00000000..87b1481d --- /dev/null +++ b/conda/cachetools/bld.bat @@ -0,0 +1,8 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 + +:: Add more build steps here, if they are necessary. + +:: See +:: http://docs.continuum.io/conda/build.html +:: for a list of environment variables that are set during the build process. diff --git a/conda/cachetools/build.sh b/conda/cachetools/build.sh new file mode 100644 index 00000000..4d7fc032 --- /dev/null +++ b/conda/cachetools/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +$PYTHON setup.py install + +# Add more build steps here, if they are necessary. + +# See +# http://docs.continuum.io/conda/build.html +# for a list of environment variables that are set during the build process. diff --git a/conda/cachetools/meta.yaml b/conda/cachetools/meta.yaml new file mode 100644 index 00000000..5d571dfb --- /dev/null +++ b/conda/cachetools/meta.yaml @@ -0,0 +1,61 @@ +package: + name: cachetools + version: "1.1.6" + +source: + fn: cachetools-1.1.6.tar.gz + url: https://pypi.python.org/packages/source/c/cachetools/cachetools-1.1.6.tar.gz + md5: 387d7f34effd9335ae55bd0762e77bfa +# patches: + # List any patch files here + # - fix.patch + +# build: + # noarch_python: True + # preserve_egg_dir: True + # entry_points: + # Put any entry points (scripts to be generated automatically) here. The + # syntax is module:function. For example + # + # - cachetools = cachetools:main + # + # Would create an entry point called cachetools that calls cachetools.main() + + + # If this is a new build for the same version, increment the build + # number. If you do not include this key, it defaults to 0. + # number: 1 + +requirements: + build: + - python + - setuptools + + run: + - python + +test: + # Python imports + imports: + - cachetools + + # commands: + # You can put test commands to be run here. Use this to test that the + # entry points work. + + + # You can also put a file called run_test.py in the recipe that will be run + # at test time. + + # requires: + # Put any additional test requirements here. For example + # - nose + +about: + home: https://github.com/tkem/cachetools + license: MIT License + summary: 'Extensible memoizing collections and decorators' + +# See +# http://docs.continuum.io/conda/build.html for +# more information about meta.yaml From e1b376a49b23fb468e5eaf5cda0d1463fbf8975b Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Thu, 14 Apr 2016 16:01:42 -0400 Subject: [PATCH 2/2] BUG: Add limit to memory growth on sliding windows Add a cap of 5 sliding windows (one per OHCLV column) to the history loader's cache of sliding windos. This prevents unbounded growth on algorithms that call history with a highly varied list of equities. To follow is splitting the cache up by column and by sid, so that the loader does not re-prefetch sids which have already been read with sufficient data; however this patch is enough to fix the issue where an algo with high rotation can add up a megabyte per day of memory on algorithms which rotate on a 5% dollar volume pipeline. With this cap those algorithms have more plateaus with regard to memory consumption. This patch requires new dependency of `cachetools` library. --- etc/requirements.txt | 2 ++ zipline/data/us_equity_loader.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/etc/requirements.txt b/etc/requirements.txt index 7ffa9527..6abd8169 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -58,3 +58,5 @@ sqlalchemy==1.0.8 # for intervaltree sortedcontainers==1.4.4 intervaltree==2.1.0 + +cachetools==1.1.5 diff --git a/zipline/data/us_equity_loader.py b/zipline/data/us_equity_loader.py index b1690960..cb9a9a14 100644 --- a/zipline/data/us_equity_loader.py +++ b/zipline/data/us_equity_loader.py @@ -17,6 +17,8 @@ from abc import ( abstractmethod, abstractproperty, ) + +from cachetools import LRUCache from numpy import dtype, around from pandas.tslib import normalize_date @@ -25,7 +27,7 @@ from six import iteritems, with_metaclass from zipline.pipeline.data.equity_pricing import USEquityPricing from zipline.lib._float64window import AdjustedArrayWindow as Float64Window from zipline.lib.adjustment import Float64Multiply -from zipline.utils.cache import CachedObject, Expired +from zipline.utils.cache import ExpiringCache from zipline.utils.memoize import lazyval @@ -82,7 +84,8 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)): self.env = env self._reader = reader self._adjustments_reader = adjustment_reader - self._window_blocks = {} + # TODO: Split cache into 'by column' and 'by sid'. + self._window_blocks = ExpiringCache(LRUCache(maxsize=5)) @abstractproperty def _prefetch_length(self): @@ -218,11 +221,7 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)): size = len(dts) assets_key = frozenset(assets) try: - block_cache = self._window_blocks[(assets_key, field, size)] - try: - return block_cache.unwrap(end) - except Expired: - pass + return self._window_blocks.get((assets_key, field, size), end) except KeyError: pass @@ -253,8 +252,9 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)): size ) block = SlidingWindow(window, size, start_ix, offset) - self._window_blocks[(assets_key, field, size)] = CachedObject( - block, prefetch_end) + self._window_blocks.set((assets_key, field, size), + block, + prefetch_end) return block def history(self, assets, dts, field):