Merge pull request #1133 from quantopian/fix-us-equity-loader

Limit leak in us equity loader on stock rotation

conda/cachetools/bld.bat

@@ -0,0 +1,8 @@
+"%PYTHON%" setup.py install
+if errorlevel 1 exit 1
+
+:: Add more build steps here, if they are necessary.
+
+:: See
+:: http://docs.continuum.io/conda/build.html
+:: for a list of environment variables that are set during the build process.

conda/cachetools/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+$PYTHON setup.py install
+
+# Add more build steps here, if they are necessary.
+
+# See
+# http://docs.continuum.io/conda/build.html
+# for a list of environment variables that are set during the build process.

conda/cachetools/meta.yaml

@@ -0,0 +1,61 @@
+package:
+  name: cachetools
+  version: "1.1.6"
+
+source:
+  fn: cachetools-1.1.6.tar.gz
+  url: https://pypi.python.org/packages/source/c/cachetools/cachetools-1.1.6.tar.gz
+  md5: 387d7f34effd9335ae55bd0762e77bfa
+#  patches:
+   # List any patch files here
+   # - fix.patch
+
+# build:
+  # noarch_python: True
+  # preserve_egg_dir: True
+  # entry_points:
+    # Put any entry points (scripts to be generated automatically) here. The
+    # syntax is module:function.  For example
+    #
+    # - cachetools = cachetools:main
+    #
+    # Would create an entry point called cachetools that calls cachetools.main()
+
+
+  # If this is a new build for the same version, increment the build
+  # number. If you do not include this key, it defaults to 0.
+  # number: 1
+
+requirements:
+  build:
+    - python
+    - setuptools
+
+  run:
+    - python
+
+test:
+  # Python imports
+  imports:
+    - cachetools
+
+  # commands:
+    # You can put test commands to be run here.  Use this to test that the
+    # entry points work.
+
+
+  # You can also put a file called run_test.py in the recipe that will be run
+  # at test time.
+
+  # requires:
+    # Put any additional test requirements here.  For example
+    # - nose
+
+about:
+  home: https://github.com/tkem/cachetools
+  license: MIT License
+  summary: 'Extensible memoizing collections and decorators'
+
+# See
+# http://docs.continuum.io/conda/build.html for
+# more information about meta.yaml

etc/requirements.txt

@@ -58,3 +58,5 @@ sqlalchemy==1.0.8
 # for intervaltree
 sortedcontainers==1.4.4
 intervaltree==2.1.0
+
+cachetools==1.1.5

zipline/data/us_equity_loader.py

@@ -17,6 +17,8 @@ from abc import (
     abstractmethod,
     abstractproperty,
 )
+
+from cachetools import LRUCache
 from numpy import dtype, around
 from pandas.tslib import normalize_date
 
@@ -25,7 +27,7 @@ from six import iteritems, with_metaclass
 from zipline.pipeline.data.equity_pricing import USEquityPricing
 from zipline.lib._float64window import AdjustedArrayWindow as Float64Window
 from zipline.lib.adjustment import Float64Multiply
-from zipline.utils.cache import CachedObject, Expired
+from zipline.utils.cache import ExpiringCache
 from zipline.utils.memoize import lazyval
 
 
@@ -82,7 +84,8 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)):
         self.env = env
         self._reader = reader
         self._adjustments_reader = adjustment_reader
-        self._window_blocks = {}
+        # TODO: Split cache into 'by column' and 'by sid'.
+        self._window_blocks = ExpiringCache(LRUCache(maxsize=5))
 
     @abstractproperty
     def _prefetch_length(self):
@@ -218,11 +221,7 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)):
         size = len(dts)
         assets_key = frozenset(assets)
         try:
-            block_cache = self._window_blocks[(assets_key, field, size)]
-            try:
-                return block_cache.unwrap(end)
-            except Expired:
-                pass
+            return self._window_blocks.get((assets_key, field, size), end)
         except KeyError:
             pass
 
@@ -253,8 +252,9 @@ class USEquityHistoryLoader(with_metaclass(ABCMeta)):
             size
         )
         block = SlidingWindow(window, size, start_ix, offset)
-        self._window_blocks[(assets_key, field, size)] = CachedObject(
-            block, prefetch_end)
+        self._window_blocks.set((assets_key, field, size),
+                                block,
+                                prefetch_end)
         return block
 
     def history(self, assets, dts, field):