wassname/cachier
1
0
Fork 0
mirror of https://github.com/wassname/cachier.git synced 2026-09-24 13:20:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

added cache_dir argument for custom cach dir paths

Browse Source
This commit is contained in:
Shay Palachy
2019-12-03 14:39:54 +02:00
parent 30d6f1451b
commit 376aade8ae
4 changed files with 121 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.rst
+13
View File
@@ -77,6 +77,12 @@ You can add a default, pickle-based, persistent cache to your function - meaning
"""Your function now has a persistent cache mapped by argument values!"""
return {'arg1': arg1, 'arg2': arg2}
You can get the fully qualified path to the directory of cache files used by ``cachier`` (``~/.cachier`` by default) by calling the ``cache_dpath()`` function:
.. code-block:: python
foo.clear_cache()
Resetting a Cache
-----------------
@@ -156,6 +162,13 @@ Pickle Core
The default core for Cachier is pickle based, meaning each function will store its cache is a separate pickle file in the ``~/.cachier`` directory. Naturally, this kind of cache is both machine-specific and user-specific.
You can configure ``cachier`` to use another directory by providing the ``cache_dir`` parameter with the path to that directory:
.. code-block:: python
@cachier(cache_dir='~/.temp/.cache')
You can slightly optimise pickle-based caching if you know your code will only be used in a single thread environment by setting:
.. code-block:: python
cachier/core.py
+43 -13
View File
@@ -22,7 +22,6 @@ from .pickle_core import _PickleCore
from .mongo_core import _MongoCore, RecalculationNeeded
MAX_WORKERS_ENVAR_NAME = 'CACHIER_MAX_WORKERS'
DEFAULT_MAX_WORKERS = 8
@@ -57,7 +56,9 @@ def _function_thread(core, key, func, args, kwds):
except BaseException as exc: # pylint: disable=W0703
print(
'Function call failed with the following exception:\n{}'.format(
exc))
exc
)
)
def _calc_entry(core, key, func, args, kwds):
@@ -71,8 +72,14 @@ def _calc_entry(core, key, func, args, kwds):
finally:
core.mark_entry_not_calculated(key)
def cachier(stale_after=None, next_time=False, pickle_reload=True,
mongetter=None):
def cachier(
stale_after=None,
next_time=False,
pickle_reload=True,
mongetter=None,
cache_dir=None,
):
"""A persistent, stale-free memoization decorator.
The positional and keyword arguments to the wrapped function must be
@@ -84,23 +91,27 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True,
Arguments
---------
stale_after (optional) : datetime.timedelta
stale_after : datetime.timedelta, optional
The time delta afterwhich a cached result is considered stale. Calls
made after the result goes stale will trigger a recalculation of the
result, but whether a stale or fresh result will be returned is
determined by the optional next_time argument.
next_time (optional) : bool
next_time : bool, optional
If set to True, a stale result will be returned when finding one, not
waiting for the calculation of the fresh result to return. Defaults to
False.
pickle_reload (optional) : bool
pickle_reload : bool, optional
If set to True, in-memory cache will be reloaded on each cache read,
enabling different threads to share cache. Should be set to False for
faster reads in single-thread programs. Defaults to True.
mongetter (optional) : callable
mongetter : callable, optional
A callable that takes no arguments and returns a pymongo.Collection
object with writing permissions. If unset a local pickle cache is used
instead.
cache_dir : str, optional
A fully qualified path to a file directory to be used for cache files.
The running process must have running permissions to this folder. If
not provided, a default directory at `~/.cachier/` is used.
"""
# print('Inside the wrapper maker')
# print('mongetter={}'.format(mongetter))
@@ -111,7 +122,11 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True,
core = _MongoCore(mongetter, stale_after, next_time)
else:
core = _PickleCore( # pylint: disable=R0204
stale_after, next_time, pickle_reload)
stale_after=stale_after,
next_time=next_time,
reload=pickle_reload,
cache_dir=cache_dir,
)
def _cachier_decorator(func):
core.set_func(func)
@@ -122,7 +137,7 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True,
ignore_cache = kwds.pop('ignore_cache', False)
overwrite_cache = kwds.pop('overwrite_cache', False)
verbose_cache = kwds.pop('verbose_cache', False)
_print = lambda x: None # skipcq: FLK-E731
_print = lambda x: None # skipcq: FLK-E731 # noqa: E731
if verbose_cache:
_print = print
if ignore_cache:
@@ -146,14 +161,21 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True,
try:
return core.wait_on_entry_calc(key)
except RecalculationNeeded:
return _calc_entry(core, key, func, args, kwds)
return _calc_entry(
core, key, func, args, kwds
)
if next_time:
_print('Async calc and return stale')
try:
core.mark_entry_being_calculated(key)
_get_executor().submit(
_function_thread, core, key, func,
args, kwds)
_function_thread,
core,
key,
func,
args,
kwds,
)
finally:
core.mark_entry_not_calculated(key)
return entry['value']
@@ -178,8 +200,16 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True,
"""Marks all entries in this cache as not being calculated."""
core.clear_being_calculated()
def cache_dpath():
"""Returns the path to the cache dir, if exists; None if not."""
try:
return core.expended_cache_dir
except AttributeError:
return None
func_wrapper.clear_cache = clear_cache
func_wrapper.clear_being_calculated = clear_being_calculated
func_wrapper.cache_dpath = cache_dpath
return func_wrapper
return _cachier_decorator
cachier/pickle_core.py
+38 -22
View File
@@ -15,7 +15,8 @@ import threading
import portalocker # to lock on pickle cache IO
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
#Altenative: https://github.com/WoLpH/portalocker
# Altenative: https://github.com/WoLpH/portalocker
from .base_core import _BaseCore
@@ -25,11 +26,23 @@ except NameError: # we're on python 2
FileNotFoundError = IOError
CACHIER_DIR = '~/.cachier/'
EXPANDED_CACHIER_DIR = os.path.expanduser(CACHIER_DIR)
DEF_CACHIER_DIR = '~/.cachier/'
class _PickleCore(_BaseCore):
"""The pickle core class for cachier.
Parameters
----------
stale_after : datetime.timedelta, optional
See _BaseCore documentation.
next_time : bool, optional
See _BaseCore documentation.
pickle_reload : bool, optional
See core.cachier() documentation.
cache_dir : str, optional.
See core.cachier() documentation.
"""
class CacheChangeHandler(PatternMatchingEventHandler):
"""Handles cache-file modification events."""
@@ -40,7 +53,7 @@ class _PickleCore(_BaseCore):
patterns=["*" + filename],
ignore_patterns=None,
ignore_directories=True,
case_sensitive=False
case_sensitive=False,
)
self.core = core
self.key = key
@@ -62,38 +75,45 @@ class _PickleCore(_BaseCore):
self.value = entry['value']
self.observer.stop()
# else:
# print('NOT stoping observer... :(')
# print('NOT stoping observer... :(')
except TypeError:
self.value = None
self.observer.stop()
def on_created(self, event):
self._check_calculation() # pragma: no cover
self._check_calculation() # pragma: no cover
def on_modified(self, event):
self._check_calculation()
def __init__(self, stale_after, next_time, reload):
def __init__(self, stale_after, next_time, reload, cache_dir):
_BaseCore.__init__(self, stale_after, next_time)
self.cache = None
self.reload = reload
self.cache_dir = DEF_CACHIER_DIR
if cache_dir is not None:
self.cache_dir = cache_dir
self.expended_cache_dir = os.path.expanduser(self.cache_dir)
self.lock = threading.RLock()
def _cache_fname(self):
if not hasattr(self, 'cache_fname'):
self.cache_fname = '.{}.{}'.format(
self.func.__module__, self.func.__name__)
self.func.__module__, self.func.__name__
)
return self.cache_fname
def _cache_fpath(self):
if not hasattr(self, 'cache_fpath'):
# print(EXPANDED_CACHIER_DIR)
if not os.path.exists(EXPANDED_CACHIER_DIR):
os.makedirs(EXPANDED_CACHIER_DIR)
self.cache_fpath = os.path.abspath(os.path.join(
os.path.realpath(EXPANDED_CACHIER_DIR),
self._cache_fname()
))
if not os.path.exists(self.expended_cache_dir):
os.makedirs(self.expended_cache_dir)
self.cache_fpath = os.path.abspath(
os.path.join(
os.path.realpath(self.expended_cache_dir),
self._cache_fname(),
)
)
return self.cache_fpath
def _reload_cache(self):
@@ -141,7 +161,7 @@ class _PickleCore(_BaseCore):
'value': func_res,
'time': datetime.now(),
'stale': False,
'being_calculated': False
'being_calculated': False,
}
self._save_cache(cache)
@@ -155,7 +175,7 @@ class _PickleCore(_BaseCore):
'value': None,
'time': datetime.now(),
'stale': False,
'being_calculated': True
'being_calculated': True,
}
self._save_cache(cache)
@@ -175,16 +195,12 @@ class _PickleCore(_BaseCore):
if not entry['being_calculated']:
return entry['value']
event_handler = _PickleCore.CacheChangeHandler(
filename=self._cache_fname(),
core=self,
key=key
filename=self._cache_fname(), core=self, key=key
)
observer = Observer()
event_handler.inject_observer(observer)
observer.schedule(
event_handler,
path=EXPANDED_CACHIER_DIR,
recursive=True
event_handler, path=self.expended_cache_dir, recursive=True
)
observer.start()
observer.join(timeout=1.0)
tests/test_pickle_core.py
+27 -1
View File
@@ -25,7 +25,7 @@ except ImportError: # python 2
import Queue as queue
from cachier import cachier
from cachier.pickle_core import EXPANDED_CACHIER_DIR
from cachier.pickle_core import DEF_CACHIER_DIR
# Pickle core tests
@@ -227,6 +227,7 @@ def _bad_cache(arg_1, arg_2):
# _BAD_CACHE_FNAME = '.__main__._bad_cache'
_BAD_CACHE_FNAME = '.tests.test_pickle_core._bad_cache'
EXPANDED_CACHIER_DIR = os.path.expanduser(DEF_CACHIER_DIR)
_BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME)
@@ -366,3 +367,28 @@ def test_error_throwing_func():
sleep(1.5)
res2 = _error_throwing_func(4)
assert res1 == res2
# test custom cache dir for pickle core
CUSTOM_DIR = '~/.exparrot'
EXPANDED_CUSTOM_DIR = os.path.expanduser(CUSTOM_DIR)
@cachier(next_time=False, cache_dir=CUSTOM_DIR)
def _takes_5_seconds_custom_dir(arg_1, arg_2):
"""Some function."""
sleep(5)
return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
def test_pickle_core_custom_cache_dir():
"""Basic Pickle core functionality."""
_takes_5_seconds_custom_dir.clear_cache()
_takes_5_seconds_custom_dir('a', 'b')
start = time()
_takes_5_seconds_custom_dir('a', 'b', verbose_cache=True)
end = time()
assert end - start < 1
_takes_5_seconds_custom_dir.clear_cache()
assert _takes_5_seconds_custom_dir.cache_dpath() == EXPANDED_CUSTOM_DIR