Merge pull request #17 from fx-kirin/hash_every_args

Hash every args including pandas and numpy objects.

cachier/pickle_core.py

@@ -152,10 +152,32 @@ class _PickleCore(_BaseCore):
             return key, self._get_cache().get(key, None)
 
     def get_entry(self, args, kwds):
-        key = args + tuple(sorted(kwds.items()))
-        # print('key type={}, key={}'.format(type(key), key))
+        key = tuple(self._hash_args(key) for key in args + tuple(sorted(kwds.items())))
         return self.get_entry_by_key(key)
 
+    def _hash_args(self, value):
+        try:
+            import pandas
+            if isinstance(value, pandas.DataFrame):
+                return pandas.util.hash_pandas_object(value).sum()
+        except ImportError:
+            pass
+        if hasattr(value, "tobytes"):  # For numpy
+            return hash(value.tobytes())
+        if hasattr(value, "__iter__"):  # For iterators
+            if isinstance(value, (list, tuple)):
+                hash_array = []
+                for elem in value:
+                    hash_array.append(self._hash_args(elem))
+                return tuple(hash_array)
+        if hasattr(value, "items"):  # For dict
+            hash_array = []
+            for key, elem in value.items:
+                hash_array.append(key)
+                hash_array.append(self._hash_args(elem))
+            return tuple(hash_array)
+        return hash(value)
+
     def set_entry(self, key, func_res):
         with self.lock:
             cache = self._get_cache()

setup.py

@@ -15,7 +15,7 @@ except ImportError:
 import versioneer
 
 
-TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo']
+TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo', 'numpy', 'pandas']
 
 README_RST = ''
 with open('README.rst') as f:

tests/test_numpy_pandas.py

@@ -0,0 +1,73 @@
+"""Test for the Cachier python package."""
+
+# This file is part of Cachier.
+# https://github.com/shaypal5/cachier
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/MIT-license
+# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
+
+# from os.path import (
+#     realpath,
+#     dirname
+# )
+from time import sleep, time
+
+import numpy as np
+import pandas as pd
+
+from cachier import cachier
+
+# Numpy and pandas tests
+
+
+@cachier()
+def _numpy_sum_takes_2_seconds(a):
+    """ Numpy cache """
+    sleep(2)
+    return a.sum()
+
+
+@cachier()
+def _pandas_sum_takes_2_seconds(df):
+    """ Numpy cache """
+    sleep(2)
+    return df.sum()
+
+
+def test_numpy_narray():
+    """Basic numpy core functionality."""
+    a = np.zeros(1000)
+    _numpy_sum_takes_2_seconds.clear_cache()
+    _numpy_sum_takes_2_seconds(a)
+    start = time()
+    _numpy_sum_takes_2_seconds(a)
+    end = time()
+    assert end - start < 1
+
+    a[0] = 3
+    start = time()
+    _numpy_sum_takes_2_seconds(a)
+    end = time()
+    assert end - start > 2.0
+
+    _numpy_sum_takes_2_seconds.clear_cache()
+
+
+def test_pandas_dataframe():
+    """Basic Pickle core functionality."""
+    a = np.zeros(1000)
+    df = pd.DataFrame(a)
+    _numpy_sum_takes_2_seconds.clear_cache()
+    _numpy_sum_takes_2_seconds(df)
+    start = time()
+    _numpy_sum_takes_2_seconds(df)
+    end = time()
+    assert end - start < 1
+    _numpy_sum_takes_2_seconds.clear_cache()
+
+    df.iloc[0, 0] = 3
+    start = time()
+    _numpy_sum_takes_2_seconds(a)
+    end = time()
+    assert end - start > 2.0