From 3f8750361a1159b9154be9161af26d23c524d3eb Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 12:45:30 +0900 Subject: [PATCH 1/6] make every argument hached --- cachier/pickle_core.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index 062cb96..10eac09 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -152,10 +152,34 @@ class _PickleCore(_BaseCore): return key, self._get_cache().get(key, None) def get_entry(self, args, kwds): - key = args + tuple(sorted(kwds.items())) - # print('key type={}, key={}'.format(type(key), key)) + key = tuple(self.hash_args(key) for key in args + tuple(sorted(kwds.items()))) return self.get_entry_by_key(key) + def hash_args(self, value): + try: + import pandas + if isinstance(value, pandas.DataFrame): + return(pandas.util.hash_pandas_object(value)) + except ImportError: + pass + if hasattr(value, "to_bytes"): # For numpy + try: + return hash(value.to_bytes()) + except TypeError: + pass + elif hasattr(value, "__iter__"): # For iterators + hash_array = [] + for elem in value: + hash_array.append(value) + return tuple(hash_array) + elif hasattr(value, "items"): # For dict + hash_array = [] + for key, elem in value.items: + hash_array.append(key) + hash_array.append(elem) + return tuple(hash_array) + return hash(value) + def set_entry(self, key, func_res): with self.lock: cache = self._get_cache() From 6ecf666b293db8452a5e16ef329502ad7e32222e Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 13:03:34 +0900 Subject: [PATCH 2/6] add test --- cachier/pickle_core.py | 9 ++--- tests/test_numpy_pandas.py | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 tests/test_numpy_pandas.py diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index 10eac09..046f810 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -159,14 +159,11 @@ class _PickleCore(_BaseCore): try: import pandas if isinstance(value, pandas.DataFrame): - return(pandas.util.hash_pandas_object(value)) + return(pandas.util.hash_pandas_object(value).sum()) except ImportError: pass - if hasattr(value, "to_bytes"): # For numpy - try: - return hash(value.to_bytes()) - except TypeError: - pass + if hasattr(value, "tobytes"): # For numpy + return hash(value.tobytes()) elif hasattr(value, "__iter__"): # For iterators hash_array = [] for elem in value: diff --git a/tests/test_numpy_pandas.py b/tests/test_numpy_pandas.py new file mode 100644 index 0000000..434821f --- /dev/null +++ b/tests/test_numpy_pandas.py @@ -0,0 +1,83 @@ +"""Test for the Cachier python package.""" + +# This file is part of Cachier. +# https://github.com/shaypal5/cachier + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/MIT-license +# Copyright (c) 2016, Shay Palachy + +# from os.path import ( +# realpath, +# dirname +# ) +import os +from time import time, sleep +from datetime import timedelta +from random import random +import threading + +try: + import queue +except ImportError: # python 2 + import Queue as queue + +from cachier import cachier +from cachier.pickle_core import DEF_CACHIER_DIR + +import numpy as np +import pandas as pd + +# Pickle core tests + + +@cachier() +def _numpy_sum_takes_2_seconds(a): + """ Numpy cache """ + sleep(2) + return a.sum() + + +@cachier() +def _pandas_sum_takes_2_seconds(df): + """ Numpy cache """ + sleep(2) + return df.sum() + + +def test_numpy_narray(): + """Basic numpy core functionality.""" + a = np.zeros(1000) + _numpy_sum_takes_2_seconds.clear_cache() + _numpy_sum_takes_2_seconds(a) + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start < 1 + + a[0] = 3 + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start > 2.0 + + _numpy_sum_takes_2_seconds.clear_cache() + + +def test_pandas_dataframe(): + """Basic Pickle core functionality.""" + a = np.zeros(1000) + df = pd.DataFrame(a) + _numpy_sum_takes_2_seconds.clear_cache() + _numpy_sum_takes_2_seconds(df) + start = time() + _numpy_sum_takes_2_seconds(df) + end = time() + assert end - start < 1 + _numpy_sum_takes_2_seconds.clear_cache() + + df.iloc[0, 0] = 3 + start = time() + _numpy_sum_takes_2_seconds(a) + end = time() + assert end - start > 2.0 From c8b81b9e9a37f51f38c60bd3c4458595de1aa90f Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 13:09:11 +0900 Subject: [PATCH 3/6] fix deepsource report --- cachier/pickle_core.py | 6 +++--- tests/test_numpy_pandas.py | 12 +----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index 046f810..f32390e 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -159,17 +159,17 @@ class _PickleCore(_BaseCore): try: import pandas if isinstance(value, pandas.DataFrame): - return(pandas.util.hash_pandas_object(value).sum()) + return pandas.util.hash_pandas_object(value).sum() except ImportError: pass if hasattr(value, "tobytes"): # For numpy return hash(value.tobytes()) - elif hasattr(value, "__iter__"): # For iterators + if hasattr(value, "__iter__"): # For iterators hash_array = [] for elem in value: hash_array.append(value) return tuple(hash_array) - elif hasattr(value, "items"): # For dict + if hasattr(value, "items"): # For dict hash_array = [] for key, elem in value.items: hash_array.append(key) diff --git a/tests/test_numpy_pandas.py b/tests/test_numpy_pandas.py index 434821f..a06954d 100644 --- a/tests/test_numpy_pandas.py +++ b/tests/test_numpy_pandas.py @@ -11,24 +11,14 @@ # realpath, # dirname # ) -import os from time import time, sleep -from datetime import timedelta -from random import random -import threading - -try: - import queue -except ImportError: # python 2 - import Queue as queue from cachier import cachier -from cachier.pickle_core import DEF_CACHIER_DIR import numpy as np import pandas as pd -# Pickle core tests +# Numpy and pandas tests @cachier() From c55ac3a07fe9ddb41e58085afb16896340f58d16 Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 13:20:55 +0900 Subject: [PATCH 4/6] fix list bug --- cachier/pickle_core.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index f32390e..f7cac16 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -152,10 +152,10 @@ class _PickleCore(_BaseCore): return key, self._get_cache().get(key, None) def get_entry(self, args, kwds): - key = tuple(self.hash_args(key) for key in args + tuple(sorted(kwds.items()))) + key = tuple(self._hash_args(key) for key in args + tuple(sorted(kwds.items()))) return self.get_entry_by_key(key) - def hash_args(self, value): + def _hash_args(self, value): try: import pandas if isinstance(value, pandas.DataFrame): @@ -165,15 +165,16 @@ class _PickleCore(_BaseCore): if hasattr(value, "tobytes"): # For numpy return hash(value.tobytes()) if hasattr(value, "__iter__"): # For iterators - hash_array = [] - for elem in value: - hash_array.append(value) - return tuple(hash_array) + if isinstance(value, (list, tuple)): + hash_array = [] + for elem in value: + hash_array.append(self._hash_args(elem)) + return tuple(hash_array) if hasattr(value, "items"): # For dict hash_array = [] for key, elem in value.items: hash_array.append(key) - hash_array.append(elem) + hash_array.append(self._hash_args(elem)) return tuple(hash_array) return hash(value) From ba7d78abd146a0ccc3a52165ee5a93f8378e85c5 Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 13:46:48 +0900 Subject: [PATCH 5/6] add test dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9d4364b..7a17545 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ except ImportError: import versioneer -TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo'] +TEST_REQUIRES = ['pytest', 'coverage', 'pytest-cov', 'pymongo', 'numpy', 'pandas'] README_RST = '' with open('README.rst') as f: From d5ee92df9ac95043c810502b2a4ee6f4e51eda7b Mon Sep 17 00:00:00 2001 From: fx_kirin Date: Wed, 8 Jan 2020 13:47:14 +0900 Subject: [PATCH 6/6] change import order --- tests/test_numpy_pandas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_numpy_pandas.py b/tests/test_numpy_pandas.py index a06954d..5f3a6b0 100644 --- a/tests/test_numpy_pandas.py +++ b/tests/test_numpy_pandas.py @@ -11,13 +11,13 @@ # realpath, # dirname # ) -from time import time, sleep - -from cachier import cachier +from time import sleep, time import numpy as np import pandas as pd +from cachier import cachier + # Numpy and pandas tests