diff --git a/README.rst b/README.rst index 95a9a88..b09c11e 100644 --- a/README.rst +++ b/README.rst @@ -77,6 +77,12 @@ You can add a default, pickle-based, persistent cache to your function - meaning """Your function now has a persistent cache mapped by argument values!""" return {'arg1': arg1, 'arg2': arg2} +You can get the fully qualified path to the directory of cache files used by ``cachier`` (``~/.cachier`` by default) by calling the ``cache_dpath()`` function: + +.. code-block:: python + + foo.clear_cache() + Resetting a Cache ----------------- @@ -156,6 +162,13 @@ Pickle Core The default core for Cachier is pickle based, meaning each function will store its cache is a separate pickle file in the ``~/.cachier`` directory. Naturally, this kind of cache is both machine-specific and user-specific. +You can configure ``cachier`` to use another directory by providing the ``cache_dir`` parameter with the path to that directory: + +.. code-block:: python + + @cachier(cache_dir='~/.temp/.cache') + + You can slightly optimise pickle-based caching if you know your code will only be used in a single thread environment by setting: .. code-block:: python diff --git a/cachier/core.py b/cachier/core.py index 20aa210..793fe38 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -22,7 +22,6 @@ from .pickle_core import _PickleCore from .mongo_core import _MongoCore, RecalculationNeeded - MAX_WORKERS_ENVAR_NAME = 'CACHIER_MAX_WORKERS' DEFAULT_MAX_WORKERS = 8 @@ -57,7 +56,9 @@ def _function_thread(core, key, func, args, kwds): except BaseException as exc: # pylint: disable=W0703 print( 'Function call failed with the following exception:\n{}'.format( - exc)) + exc + ) + ) def _calc_entry(core, key, func, args, kwds): @@ -71,8 +72,14 @@ def _calc_entry(core, key, func, args, kwds): finally: core.mark_entry_not_calculated(key) -def cachier(stale_after=None, next_time=False, pickle_reload=True, - mongetter=None): + +def cachier( + stale_after=None, + next_time=False, + pickle_reload=True, + mongetter=None, + cache_dir=None, +): """A persistent, stale-free memoization decorator. The positional and keyword arguments to the wrapped function must be @@ -84,23 +91,27 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, Arguments --------- - stale_after (optional) : datetime.timedelta + stale_after : datetime.timedelta, optional The time delta afterwhich a cached result is considered stale. Calls made after the result goes stale will trigger a recalculation of the result, but whether a stale or fresh result will be returned is determined by the optional next_time argument. - next_time (optional) : bool + next_time : bool, optional If set to True, a stale result will be returned when finding one, not waiting for the calculation of the fresh result to return. Defaults to False. - pickle_reload (optional) : bool + pickle_reload : bool, optional If set to True, in-memory cache will be reloaded on each cache read, enabling different threads to share cache. Should be set to False for faster reads in single-thread programs. Defaults to True. - mongetter (optional) : callable + mongetter : callable, optional A callable that takes no arguments and returns a pymongo.Collection object with writing permissions. If unset a local pickle cache is used instead. + cache_dir : str, optional + A fully qualified path to a file directory to be used for cache files. + The running process must have running permissions to this folder. If + not provided, a default directory at `~/.cachier/` is used. """ # print('Inside the wrapper maker') # print('mongetter={}'.format(mongetter)) @@ -111,7 +122,11 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, core = _MongoCore(mongetter, stale_after, next_time) else: core = _PickleCore( # pylint: disable=R0204 - stale_after, next_time, pickle_reload) + stale_after=stale_after, + next_time=next_time, + reload=pickle_reload, + cache_dir=cache_dir, + ) def _cachier_decorator(func): core.set_func(func) @@ -122,7 +137,7 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, ignore_cache = kwds.pop('ignore_cache', False) overwrite_cache = kwds.pop('overwrite_cache', False) verbose_cache = kwds.pop('verbose_cache', False) - _print = lambda x: None # skipcq: FLK-E731 + _print = lambda x: None # skipcq: FLK-E731 # noqa: E731 if verbose_cache: _print = print if ignore_cache: @@ -146,14 +161,21 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, try: return core.wait_on_entry_calc(key) except RecalculationNeeded: - return _calc_entry(core, key, func, args, kwds) + return _calc_entry( + core, key, func, args, kwds + ) if next_time: _print('Async calc and return stale') try: core.mark_entry_being_calculated(key) _get_executor().submit( - _function_thread, core, key, func, - args, kwds) + _function_thread, + core, + key, + func, + args, + kwds, + ) finally: core.mark_entry_not_calculated(key) return entry['value'] @@ -178,8 +200,16 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, """Marks all entries in this cache as not being calculated.""" core.clear_being_calculated() + def cache_dpath(): + """Returns the path to the cache dir, if exists; None if not.""" + try: + return core.expended_cache_dir + except AttributeError: + return None + func_wrapper.clear_cache = clear_cache func_wrapper.clear_being_calculated = clear_being_calculated + func_wrapper.cache_dpath = cache_dpath return func_wrapper return _cachier_decorator diff --git a/cachier/pickle_core.py b/cachier/pickle_core.py index a7f5c8c..86359e7 100644 --- a/cachier/pickle_core.py +++ b/cachier/pickle_core.py @@ -15,7 +15,8 @@ import threading import portalocker # to lock on pickle cache IO from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler -#Altenative: https://github.com/WoLpH/portalocker + +# Altenative: https://github.com/WoLpH/portalocker from .base_core import _BaseCore @@ -25,11 +26,23 @@ except NameError: # we're on python 2 FileNotFoundError = IOError -CACHIER_DIR = '~/.cachier/' -EXPANDED_CACHIER_DIR = os.path.expanduser(CACHIER_DIR) +DEF_CACHIER_DIR = '~/.cachier/' class _PickleCore(_BaseCore): + """The pickle core class for cachier. + + Parameters + ---------- + stale_after : datetime.timedelta, optional + See _BaseCore documentation. + next_time : bool, optional + See _BaseCore documentation. + pickle_reload : bool, optional + See core.cachier() documentation. + cache_dir : str, optional. + See core.cachier() documentation. + """ class CacheChangeHandler(PatternMatchingEventHandler): """Handles cache-file modification events.""" @@ -40,7 +53,7 @@ class _PickleCore(_BaseCore): patterns=["*" + filename], ignore_patterns=None, ignore_directories=True, - case_sensitive=False + case_sensitive=False, ) self.core = core self.key = key @@ -62,38 +75,45 @@ class _PickleCore(_BaseCore): self.value = entry['value'] self.observer.stop() # else: - # print('NOT stoping observer... :(') + # print('NOT stoping observer... :(') except TypeError: self.value = None self.observer.stop() def on_created(self, event): - self._check_calculation() # pragma: no cover + self._check_calculation() # pragma: no cover def on_modified(self, event): self._check_calculation() - def __init__(self, stale_after, next_time, reload): + def __init__(self, stale_after, next_time, reload, cache_dir): _BaseCore.__init__(self, stale_after, next_time) self.cache = None self.reload = reload + self.cache_dir = DEF_CACHIER_DIR + if cache_dir is not None: + self.cache_dir = cache_dir + self.expended_cache_dir = os.path.expanduser(self.cache_dir) self.lock = threading.RLock() def _cache_fname(self): if not hasattr(self, 'cache_fname'): self.cache_fname = '.{}.{}'.format( - self.func.__module__, self.func.__name__) + self.func.__module__, self.func.__name__ + ) return self.cache_fname def _cache_fpath(self): if not hasattr(self, 'cache_fpath'): # print(EXPANDED_CACHIER_DIR) - if not os.path.exists(EXPANDED_CACHIER_DIR): - os.makedirs(EXPANDED_CACHIER_DIR) - self.cache_fpath = os.path.abspath(os.path.join( - os.path.realpath(EXPANDED_CACHIER_DIR), - self._cache_fname() - )) + if not os.path.exists(self.expended_cache_dir): + os.makedirs(self.expended_cache_dir) + self.cache_fpath = os.path.abspath( + os.path.join( + os.path.realpath(self.expended_cache_dir), + self._cache_fname(), + ) + ) return self.cache_fpath def _reload_cache(self): @@ -141,7 +161,7 @@ class _PickleCore(_BaseCore): 'value': func_res, 'time': datetime.now(), 'stale': False, - 'being_calculated': False + 'being_calculated': False, } self._save_cache(cache) @@ -155,7 +175,7 @@ class _PickleCore(_BaseCore): 'value': None, 'time': datetime.now(), 'stale': False, - 'being_calculated': True + 'being_calculated': True, } self._save_cache(cache) @@ -175,16 +195,12 @@ class _PickleCore(_BaseCore): if not entry['being_calculated']: return entry['value'] event_handler = _PickleCore.CacheChangeHandler( - filename=self._cache_fname(), - core=self, - key=key + filename=self._cache_fname(), core=self, key=key ) observer = Observer() event_handler.inject_observer(observer) observer.schedule( - event_handler, - path=EXPANDED_CACHIER_DIR, - recursive=True + event_handler, path=self.expended_cache_dir, recursive=True ) observer.start() observer.join(timeout=1.0) diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py index 00a0b49..0fef615 100644 --- a/tests/test_pickle_core.py +++ b/tests/test_pickle_core.py @@ -25,7 +25,7 @@ except ImportError: # python 2 import Queue as queue from cachier import cachier -from cachier.pickle_core import EXPANDED_CACHIER_DIR +from cachier.pickle_core import DEF_CACHIER_DIR # Pickle core tests @@ -227,6 +227,7 @@ def _bad_cache(arg_1, arg_2): # _BAD_CACHE_FNAME = '.__main__._bad_cache' _BAD_CACHE_FNAME = '.tests.test_pickle_core._bad_cache' +EXPANDED_CACHIER_DIR = os.path.expanduser(DEF_CACHIER_DIR) _BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME) @@ -366,3 +367,28 @@ def test_error_throwing_func(): sleep(1.5) res2 = _error_throwing_func(4) assert res1 == res2 + + +# test custom cache dir for pickle core + +CUSTOM_DIR = '~/.exparrot' +EXPANDED_CUSTOM_DIR = os.path.expanduser(CUSTOM_DIR) + + +@cachier(next_time=False, cache_dir=CUSTOM_DIR) +def _takes_5_seconds_custom_dir(arg_1, arg_2): + """Some function.""" + sleep(5) + return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2) + + +def test_pickle_core_custom_cache_dir(): + """Basic Pickle core functionality.""" + _takes_5_seconds_custom_dir.clear_cache() + _takes_5_seconds_custom_dir('a', 'b') + start = time() + _takes_5_seconds_custom_dir('a', 'b', verbose_cache=True) + end = time() + assert end - start < 1 + _takes_5_seconds_custom_dir.clear_cache() + assert _takes_5_seconds_custom_dir.cache_dpath() == EXPANDED_CUSTOM_DIR