diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index 062cb96..f7cac16 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -152,10 +152,32 @@ class _PickleCore(_BaseCore): return key, self._get_cache().get(key, None) def get_entry(self, args, kwds): - key = args + tuple(sorted(kwds.items())) - # print('key type={}, key={}'.format(type(key), key)) + key = tuple(self._hash_args(key) for key in args + tuple(sorted(kwds.items()))) return self.get_entry_by_key(key) + def _hash_args(self, value): + try: + import pandas + if isinstance(value, pandas.DataFrame): + return pandas.util.hash_pandas_object(value).sum() + except ImportError: + pass + if hasattr(value, "tobytes"): # For numpy + return hash(value.tobytes()) + if hasattr(value, "__iter__"): # For iterators + if isinstance(value, (list, tuple)): + hash_array = [] + for elem in value: + hash_array.append(self._hash_args(elem)) + return tuple(hash_array) + if hasattr(value, "items"): # For dict + hash_array = [] + for key, elem in value.items: + hash_array.append(key) + hash_array.append(self._hash_args(elem)) + return tuple(hash_array) + return hash(value) + def set_entry(self, key, func_res): with self.lock: cache = self._get_cache() diff --git a/setup.py b/setup.py index c438286..3ffbfb5 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ except ImportError: import versioneer -TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo'] +TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo', 'numpy', 'pandas'] README_RST = '' with open('README.rst') as f: diff --git a/tests/test_numpy_pandas.py b/tests/test_numpy_pandas.py new file mode 100644 index 0000000..5f3a6b0 --- /dev/null +++ b/tests/test_numpy_pandas.py @@ -0,0 +1,73 @@ +"""Test for the Cachier python package.""" + +# This file is part of Cachier. +# https://github.com/shaypal5/cachier + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/MIT-license +# Copyright (c) 2016, Shay Palachy + +# from os.path import ( +# realpath, +# dirname +# ) +from time import sleep, time + +import numpy as np +import pandas as pd + +from cachier import cachier + +# Numpy and pandas tests + + +@cachier() +def _numpy_sum_takes_2_seconds(a): + """ Numpy cache """ + sleep(2) + return a.sum() + + +@cachier() +def _pandas_sum_takes_2_seconds(df): + """ Numpy cache """ + sleep(2) + return df.sum() + + +def test_numpy_narray(): + """Basic numpy core functionality.""" + a = np.zeros(1000) + _numpy_sum_takes_2_seconds.clear_cache() + _numpy_sum_takes_2_seconds(a) + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start < 1 + + a[0] = 3 + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start > 2.0 + + _numpy_sum_takes_2_seconds.clear_cache() + + +def test_pandas_dataframe(): + """Basic Pickle core functionality.""" + a = np.zeros(1000) + df = pd.DataFrame(a) + _numpy_sum_takes_2_seconds.clear_cache() + _numpy_sum_takes_2_seconds(df) + start = time() + _numpy_sum_takes_2_seconds(df) + end = time() + assert end - start < 1 + _numpy_sum_takes_2_seconds.clear_cache() + + df.iloc[0, 0] = 3 + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start > 2.0