mirror of
https://github.com/wassname/cachier.git
synced 2026-09-09 11:19:08 +08:00
coverage upped to ~93%
This commit is contained in:
+2
-1
@@ -25,4 +25,5 @@ ignore:
|
||||
- "versioneer.py"
|
||||
- "tests"
|
||||
- "**/_version.py"
|
||||
- "**/__init__.py"
|
||||
- "**/__init__.py"
|
||||
- "cachier/scripts"
|
||||
@@ -4,6 +4,7 @@ omit =
|
||||
tests/*
|
||||
cachier/_version.py
|
||||
cachier/__init__.py
|
||||
**/scripts/**
|
||||
[report]
|
||||
show_missing = True
|
||||
# Regexes for lines to exclude from consideration
|
||||
|
||||
Vendored
+1339
-108
File diff suppressed because it is too large
Load Diff
+6
-4
@@ -33,19 +33,21 @@ from .pickle_core import _PickleCore
|
||||
from .mongo_core import _MongoCore
|
||||
|
||||
|
||||
|
||||
MAX_WORKERS_ENVAR_NAME = 'CACHIER_MAX_WORKERS'
|
||||
DEFAULT_MAX_WORKERS = 8
|
||||
|
||||
|
||||
def _max_workers():
|
||||
try:
|
||||
return int(os.environ['CACHIER_MAX_WORKERS'])
|
||||
return int(os.environ[MAX_WORKERS_ENVAR_NAME])
|
||||
except KeyError:
|
||||
os.environ['CACHIER_MAX_WORKERS'] = str(DEFAULT_MAX_WORKERS)
|
||||
os.environ[MAX_WORKERS_ENVAR_NAME] = str(DEFAULT_MAX_WORKERS)
|
||||
return DEFAULT_MAX_WORKERS
|
||||
|
||||
|
||||
def _set_max_workets(max_workers):
|
||||
os.environ['CACHIER_MAX_WORKERS'] = str(max_workers)
|
||||
def _set_max_workers(max_workers):
|
||||
os.environ[MAX_WORKERS_ENVAR_NAME] = str(max_workers)
|
||||
_get_executor(True)
|
||||
|
||||
|
||||
|
||||
+75
-62
@@ -9,9 +9,10 @@
|
||||
|
||||
import os
|
||||
import pickle # for local caching
|
||||
import fcntl # to lock on pickle cache IO
|
||||
from datetime import datetime
|
||||
import threading
|
||||
|
||||
import portalocker # to lock on pickle cache IO
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import PatternMatchingEventHandler
|
||||
#Altenative: https://github.com/WoLpH/portalocker
|
||||
@@ -50,14 +51,19 @@ class _PickleCore(_BaseCore):
|
||||
entry = self.core.get_entry_by_key(self.key, True)[1]
|
||||
# print(self.key)
|
||||
# print(entry)
|
||||
if not entry['being_calculated']:
|
||||
# print('stoping observer!')
|
||||
self.value = entry['value']
|
||||
try:
|
||||
if not entry['being_calculated']:
|
||||
# print('stoping observer!')
|
||||
self.value = entry['value']
|
||||
self.observer.stop()
|
||||
# else:
|
||||
# print('NOT stoping observer... :(')
|
||||
except TypeError:
|
||||
self.value = None
|
||||
self.observer.stop()
|
||||
# print('NOT stoping observer... :(')
|
||||
|
||||
def on_created(self, event):
|
||||
self._check_calculation()
|
||||
self._check_calculation() # pragma: no cover
|
||||
|
||||
def on_modified(self, event):
|
||||
self._check_calculation()
|
||||
@@ -66,6 +72,7 @@ class _PickleCore(_BaseCore):
|
||||
_BaseCore.__init__(self, stale_after, next_time)
|
||||
self.cache = None
|
||||
self.reload = reload
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def _cache_fname(self):
|
||||
if not hasattr(self, 'cache_fname'):
|
||||
@@ -85,37 +92,37 @@ class _PickleCore(_BaseCore):
|
||||
return self.cache_fpath
|
||||
|
||||
def _reload_cache(self):
|
||||
fpath = self._cache_fpath()
|
||||
try:
|
||||
with open(fpath, 'rb') as cache_file:
|
||||
fcntl.flock(cache_file, fcntl.LOCK_SH)
|
||||
try:
|
||||
self.cache = pickle.load(cache_file)
|
||||
except EOFError:
|
||||
self.cache = {}
|
||||
fcntl.flock(cache_file, fcntl.LOCK_UN)
|
||||
except FileNotFoundError:
|
||||
self.cache = {}
|
||||
with self.lock:
|
||||
fpath = self._cache_fpath()
|
||||
try:
|
||||
with portalocker.Lock(fpath, mode='rb') as cache_file:
|
||||
try:
|
||||
self.cache = pickle.load(cache_file)
|
||||
except EOFError:
|
||||
self.cache = {}
|
||||
except FileNotFoundError:
|
||||
self.cache = {}
|
||||
|
||||
def _get_cache(self):
|
||||
if not self.cache:
|
||||
self._reload_cache()
|
||||
return self.cache
|
||||
with self.lock:
|
||||
if not self.cache:
|
||||
self._reload_cache()
|
||||
return self.cache
|
||||
|
||||
def _save_cache(self, cache):
|
||||
self.cache = cache
|
||||
fpath = self._cache_fpath()
|
||||
with open(fpath, 'wb') as cache_file:
|
||||
fcntl.flock(cache_file, fcntl.LOCK_EX)
|
||||
pickle.dump(cache, cache_file)
|
||||
fcntl.flock(cache_file, fcntl.LOCK_UN)
|
||||
self._reload_cache()
|
||||
with self.lock:
|
||||
self.cache = cache
|
||||
fpath = self._cache_fpath()
|
||||
with portalocker.Lock(fpath, mode='wb') as cache_file:
|
||||
pickle.dump(cache, cache_file)
|
||||
self._reload_cache()
|
||||
|
||||
def get_entry_by_key(self, key, reload=False): # pylint: disable=W0221
|
||||
# print('{}, {}'.format(self.reload, reload))
|
||||
if self.reload or reload:
|
||||
self._reload_cache()
|
||||
return key, self._get_cache().get(key, None)
|
||||
with self.lock:
|
||||
# print('{}, {}'.format(self.reload, reload))
|
||||
if self.reload or reload:
|
||||
self._reload_cache()
|
||||
return key, self._get_cache().get(key, None)
|
||||
|
||||
def get_entry(self, args, kwds):
|
||||
key = args + tuple(sorted(kwds.items()))
|
||||
@@ -123,40 +130,45 @@ class _PickleCore(_BaseCore):
|
||||
return self.get_entry_by_key(key)
|
||||
|
||||
def set_entry(self, key, func_res):
|
||||
cache = self._get_cache()
|
||||
cache[key] = {
|
||||
'value': func_res,
|
||||
'time': datetime.now(),
|
||||
'stale': False,
|
||||
'being_calculated': False
|
||||
}
|
||||
self._save_cache(cache)
|
||||
|
||||
def mark_entry_being_calculated(self, key):
|
||||
cache = self._get_cache()
|
||||
try:
|
||||
cache[key]['being_calculated'] = True
|
||||
except KeyError:
|
||||
with self.lock:
|
||||
cache = self._get_cache()
|
||||
cache[key] = {
|
||||
'value': None,
|
||||
'value': func_res,
|
||||
'time': datetime.now(),
|
||||
'stale': False,
|
||||
'being_calculated': True
|
||||
'being_calculated': False
|
||||
}
|
||||
self._save_cache(cache)
|
||||
self._save_cache(cache)
|
||||
|
||||
def mark_entry_being_calculated(self, key):
|
||||
with self.lock:
|
||||
cache = self._get_cache()
|
||||
try:
|
||||
cache[key]['being_calculated'] = True
|
||||
except KeyError:
|
||||
cache[key] = {
|
||||
'value': None,
|
||||
'time': datetime.now(),
|
||||
'stale': False,
|
||||
'being_calculated': True
|
||||
}
|
||||
self._save_cache(cache)
|
||||
|
||||
def mark_entry_not_calculated(self, key):
|
||||
cache = self._get_cache()
|
||||
try:
|
||||
cache[key]['being_calculated'] = False
|
||||
self._save_cache(cache)
|
||||
except KeyError:
|
||||
pass # that's ok, we don't need an entry in that case
|
||||
with self.lock:
|
||||
cache = self._get_cache()
|
||||
try:
|
||||
cache[key]['being_calculated'] = False
|
||||
self._save_cache(cache)
|
||||
except KeyError:
|
||||
pass # that's ok, we don't need an entry in that case
|
||||
|
||||
def wait_on_entry_calc(self, key):
|
||||
entry = self._get_cache()[key]
|
||||
if not entry['being_calculated']:
|
||||
return entry['value']
|
||||
with self.lock:
|
||||
self._reload_cache()
|
||||
entry = self._get_cache()[key]
|
||||
if not entry['being_calculated']:
|
||||
return entry['value']
|
||||
event_handler = _PickleCore.CacheChangeHandler(
|
||||
filename=self._cache_fname(),
|
||||
core=self,
|
||||
@@ -170,7 +182,7 @@ class _PickleCore(_BaseCore):
|
||||
recursive=True
|
||||
)
|
||||
observer.start()
|
||||
observer.join(timeout=2.0)
|
||||
observer.join(timeout=1.0)
|
||||
if observer.isAlive():
|
||||
# print('Timedout waiting. Starting again...')
|
||||
return self.wait_on_entry_calc(key)
|
||||
@@ -181,7 +193,8 @@ class _PickleCore(_BaseCore):
|
||||
self._save_cache({})
|
||||
|
||||
def clear_being_calculated(self):
|
||||
cache = self._get_cache()
|
||||
for key in cache:
|
||||
cache[key]['being_calculated'] = False
|
||||
self._save_cache(cache)
|
||||
with self.lock:
|
||||
cache = self._get_cache()
|
||||
for key in cache:
|
||||
cache[key]['being_calculated'] = False
|
||||
self._save_cache(cache)
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
"A command-line interface for cachier."
|
||||
|
||||
import click
|
||||
|
||||
from cachier.core import _set_max_workers
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""A command-line interface for cachier."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("Limits the number of worker threads used by cachier.")
|
||||
@click.argument('max_workers', nargs=1, type=int)
|
||||
def set_max_workers(max_workers):
|
||||
"""Limits the number of worker threads used by cachier."""
|
||||
_set_max_workers(max_workers)
|
||||
@@ -33,6 +33,10 @@ setup(
|
||||
author_email='shay.palachy@gmail.com',
|
||||
url='https://github.com/shaypal5/cachier',
|
||||
packages=['cachier'],
|
||||
entry_points='''
|
||||
[console_scripts]
|
||||
cachier=cachier.scripts.cli:cli
|
||||
''',
|
||||
install_requires=[
|
||||
'watchdog'
|
||||
],
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
from .test_cachier import *
|
||||
+5
-1
@@ -1,7 +1,10 @@
|
||||
"""Configuration file for pytest."""
|
||||
|
||||
import pytest
|
||||
import shutil
|
||||
|
||||
from .test_mongo_core import _test_mongetter
|
||||
from cachier.pickle_core import EXPANDED_CACHIER_DIR
|
||||
|
||||
|
||||
def mongo_finalizer():
|
||||
@@ -14,5 +17,6 @@ def mongo_finalizer():
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def do_something(request):
|
||||
"""Sessions-scopre pytest hook."""
|
||||
"""Session-scope pytest hook."""
|
||||
shutil.rmtree(EXPANDED_CACHIER_DIR)
|
||||
request.addfinalizer(mongo_finalizer)
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
"""Test for the Cachier python package."""
|
||||
|
||||
# This file is part of Cachier.
|
||||
# https://github.com/shaypal5/cachier
|
||||
|
||||
# Licensed under the MIT license:
|
||||
# http://www.opensource.org/licenses/MIT-license
|
||||
# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
|
||||
|
||||
# from os.path import (
|
||||
# realpath,
|
||||
# dirname
|
||||
# )
|
||||
from time import (
|
||||
time,
|
||||
sleep
|
||||
)
|
||||
from datetime import timedelta
|
||||
from random import random
|
||||
|
||||
from cachier import cachier
|
||||
|
||||
|
||||
# Pickle core tests
|
||||
|
||||
@cachier(next_time=False)
|
||||
def _takes_5_seconds(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(5)
|
||||
return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
|
||||
|
||||
|
||||
def test_pickle_core():
|
||||
"""Basic Pickle core functionality."""
|
||||
_takes_5_seconds.clear_cache()
|
||||
stringi = _takes_5_seconds('a', 'b')
|
||||
start = time()
|
||||
stringi = _takes_5_seconds('a', 'b')
|
||||
end = time()
|
||||
assert end - start < 1
|
||||
_takes_5_seconds.clear_cache()
|
||||
|
||||
|
||||
DELTA = timedelta(seconds=3)
|
||||
|
||||
@cachier(stale_after=DELTA, next_time=False)
|
||||
def _stale_after_seconds(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
return random()
|
||||
|
||||
|
||||
def test_stale_after():
|
||||
"""Testing the stale_after functionality."""
|
||||
_stale_after_seconds.clear_cache()
|
||||
val1 = _stale_after_seconds(1, 2)
|
||||
val2 = _stale_after_seconds(1, 2)
|
||||
val3 = _stale_after_seconds(1, 3)
|
||||
assert val1 == val2
|
||||
assert val1 != val3
|
||||
sleep(3)
|
||||
val4 = _stale_after_seconds(1, 2)
|
||||
assert val4 != val1
|
||||
_stale_after_seconds.clear_cache()
|
||||
|
||||
|
||||
@cachier(stale_after=DELTA, next_time=True)
|
||||
def _stale_after_next_time(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
return random()
|
||||
|
||||
|
||||
def test_stale_after_next_time():
|
||||
"""Testing the stale_after with next_time functionality."""
|
||||
_stale_after_next_time.clear_cache()
|
||||
val1 = _stale_after_next_time(1, 2)
|
||||
val2 = _stale_after_next_time(1, 2)
|
||||
val3 = _stale_after_next_time(1, 3)
|
||||
assert val1 == val2
|
||||
assert val1 != val3
|
||||
sleep(3)
|
||||
val4 = _stale_after_next_time(1, 2)
|
||||
assert val4 == val1
|
||||
val5 = _stale_after_next_time(1, 2)
|
||||
assert val5 != val1
|
||||
_stale_after_next_time.clear_cache()
|
||||
|
||||
|
||||
|
||||
@cachier()
|
||||
def _random_num():
|
||||
return random()
|
||||
|
||||
|
||||
@cachier()
|
||||
def _random_num_with_arg(a):
|
||||
# print(a)
|
||||
return random()
|
||||
|
||||
|
||||
def test_overwrite_cache():
|
||||
"""Tests that the overwrite feature works correctly."""
|
||||
_random_num.clear_cache()
|
||||
int1 = _random_num()
|
||||
int2 = _random_num()
|
||||
assert int2 == int1
|
||||
int3 = _random_num(overwrite_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num()
|
||||
assert int4 == int3
|
||||
_random_num.clear_cache()
|
||||
|
||||
_random_num_with_arg.clear_cache()
|
||||
int1 = _random_num_with_arg('a')
|
||||
int2 = _random_num_with_arg('a')
|
||||
assert int2 == int1
|
||||
int3 = _random_num_with_arg('a', overwrite_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num_with_arg('a')
|
||||
assert int4 == int3
|
||||
_random_num_with_arg.clear_cache()
|
||||
|
||||
|
||||
def test_ignore_cache():
|
||||
"""Tests that the ignore_cache feature works correctly."""
|
||||
_random_num.clear_cache()
|
||||
int1 = _random_num()
|
||||
int2 = _random_num()
|
||||
assert int2 == int1
|
||||
int3 = _random_num(ignore_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num()
|
||||
assert int4 != int3
|
||||
assert int4 == int1
|
||||
_random_num.clear_cache()
|
||||
|
||||
_random_num_with_arg.clear_cache()
|
||||
int1 = _random_num_with_arg('a')
|
||||
int2 = _random_num_with_arg('a')
|
||||
assert int2 == int1
|
||||
int3 = _random_num_with_arg('a', ignore_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num_with_arg('a')
|
||||
assert int4 != int3
|
||||
assert int4 == int1
|
||||
_random_num_with_arg.clear_cache()
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Non-core-specific tests for cachier."""
|
||||
|
||||
import os
|
||||
|
||||
from cachier.core import (
|
||||
MAX_WORKERS_ENVAR_NAME,
|
||||
DEFAULT_MAX_WORKERS,
|
||||
_max_workers,
|
||||
_set_max_workers,
|
||||
_get_executor
|
||||
)
|
||||
|
||||
|
||||
def test_max_workers():
|
||||
"""Just call this function for coverage."""
|
||||
try:
|
||||
del os.environ[MAX_WORKERS_ENVAR_NAME]
|
||||
except KeyError:
|
||||
pass
|
||||
assert _max_workers() == DEFAULT_MAX_WORKERS
|
||||
|
||||
|
||||
def test_get_executor():
|
||||
"""Just call this function for coverage."""
|
||||
_get_executor()
|
||||
_get_executor(False)
|
||||
_get_executor(True)
|
||||
|
||||
|
||||
def test_set_max_workers():
|
||||
"""Just call this function for coverage."""
|
||||
_set_max_workers(9)
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ def test_mongo_being_calculated():
|
||||
thread2 = threading.Thread(
|
||||
target=_calls_takes_time, kwargs={'res_queue': res_queue})
|
||||
thread1.start()
|
||||
sleep(0.5)
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Test for the Cachier python package."""
|
||||
|
||||
# This file is part of Cachier.
|
||||
# https://github.com/shaypal5/cachier
|
||||
|
||||
# Licensed under the MIT license:
|
||||
# http://www.opensource.org/licenses/MIT-license
|
||||
# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
|
||||
|
||||
# from os.path import (
|
||||
# realpath,
|
||||
# dirname
|
||||
# )
|
||||
import os
|
||||
from time import (
|
||||
time,
|
||||
sleep
|
||||
)
|
||||
from datetime import timedelta
|
||||
from random import random
|
||||
import threading
|
||||
try:
|
||||
import queue
|
||||
except ImportError: # python 2
|
||||
import Queue as queue
|
||||
|
||||
from cachier import cachier
|
||||
from cachier.pickle_core import EXPANDED_CACHIER_DIR
|
||||
|
||||
|
||||
# Pickle core tests
|
||||
|
||||
@cachier(next_time=False)
|
||||
def _takes_5_seconds(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(5)
|
||||
return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
|
||||
|
||||
|
||||
def test_pickle_core():
|
||||
"""Basic Pickle core functionality."""
|
||||
_takes_5_seconds.clear_cache()
|
||||
stringi = _takes_5_seconds('a', 'b')
|
||||
start = time()
|
||||
stringi = _takes_5_seconds('a', 'b', verbose_cache=True)
|
||||
end = time()
|
||||
assert end - start < 1
|
||||
_takes_5_seconds.clear_cache()
|
||||
|
||||
|
||||
DELTA = timedelta(seconds=3)
|
||||
|
||||
@cachier(stale_after=DELTA, next_time=False)
|
||||
def _stale_after_seconds(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
return random()
|
||||
|
||||
|
||||
def test_stale_after():
|
||||
"""Testing the stale_after functionality."""
|
||||
_stale_after_seconds.clear_cache()
|
||||
val1 = _stale_after_seconds(1, 2)
|
||||
val2 = _stale_after_seconds(1, 2)
|
||||
val3 = _stale_after_seconds(1, 3)
|
||||
assert val1 == val2
|
||||
assert val1 != val3
|
||||
sleep(3)
|
||||
val4 = _stale_after_seconds(1, 2)
|
||||
assert val4 != val1
|
||||
_stale_after_seconds.clear_cache()
|
||||
|
||||
|
||||
@cachier(stale_after=DELTA, next_time=True)
|
||||
def _stale_after_next_time(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
return random()
|
||||
|
||||
|
||||
def test_stale_after_next_time():
|
||||
"""Testing the stale_after with next_time functionality."""
|
||||
_stale_after_next_time.clear_cache()
|
||||
val1 = _stale_after_next_time(1, 2)
|
||||
val2 = _stale_after_next_time(1, 2)
|
||||
val3 = _stale_after_next_time(1, 3)
|
||||
assert val1 == val2
|
||||
assert val1 != val3
|
||||
sleep(3)
|
||||
val4 = _stale_after_next_time(1, 2)
|
||||
assert val4 == val1
|
||||
val5 = _stale_after_next_time(1, 2)
|
||||
assert val5 != val1
|
||||
_stale_after_next_time.clear_cache()
|
||||
|
||||
|
||||
|
||||
@cachier()
|
||||
def _random_num():
|
||||
return random()
|
||||
|
||||
|
||||
@cachier()
|
||||
def _random_num_with_arg(a):
|
||||
# print(a)
|
||||
return random()
|
||||
|
||||
|
||||
def test_overwrite_cache():
|
||||
"""Tests that the overwrite feature works correctly."""
|
||||
_random_num.clear_cache()
|
||||
int1 = _random_num()
|
||||
int2 = _random_num()
|
||||
assert int2 == int1
|
||||
int3 = _random_num(overwrite_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num()
|
||||
assert int4 == int3
|
||||
_random_num.clear_cache()
|
||||
|
||||
_random_num_with_arg.clear_cache()
|
||||
int1 = _random_num_with_arg('a')
|
||||
int2 = _random_num_with_arg('a')
|
||||
assert int2 == int1
|
||||
int3 = _random_num_with_arg('a', overwrite_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num_with_arg('a')
|
||||
assert int4 == int3
|
||||
_random_num_with_arg.clear_cache()
|
||||
|
||||
|
||||
def test_ignore_cache():
|
||||
"""Tests that the ignore_cache feature works correctly."""
|
||||
_random_num.clear_cache()
|
||||
int1 = _random_num()
|
||||
int2 = _random_num()
|
||||
assert int2 == int1
|
||||
int3 = _random_num(ignore_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num()
|
||||
assert int4 != int3
|
||||
assert int4 == int1
|
||||
_random_num.clear_cache()
|
||||
|
||||
_random_num_with_arg.clear_cache()
|
||||
int1 = _random_num_with_arg('a')
|
||||
int2 = _random_num_with_arg('a')
|
||||
assert int2 == int1
|
||||
int3 = _random_num_with_arg('a', ignore_cache=True)
|
||||
assert int3 != int1
|
||||
int4 = _random_num_with_arg('a')
|
||||
assert int4 != int3
|
||||
assert int4 == int1
|
||||
_random_num_with_arg.clear_cache()
|
||||
|
||||
@cachier()
|
||||
def _takes_time(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(2) # this has to be enough time for check_calculation to run twice
|
||||
return random() + arg_1 + arg_2
|
||||
|
||||
def _calls_takes_time(res_queue):
|
||||
res = _takes_time(0.13, 0.02)
|
||||
res_queue.put(res)
|
||||
|
||||
def test_pickle_being_calculated():
|
||||
"""Testing pickle core handling of being calculated scenarios."""
|
||||
_takes_time.clear_cache()
|
||||
res_queue = queue.Queue()
|
||||
thread1 = threading.Thread(
|
||||
target=_calls_takes_time, kwargs={'res_queue': res_queue})
|
||||
thread2 = threading.Thread(
|
||||
target=_calls_takes_time, kwargs={'res_queue': res_queue})
|
||||
thread1.start()
|
||||
sleep(0.5)
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
assert res_queue.qsize() == 2
|
||||
res1 = res_queue.get()
|
||||
res2 = res_queue.get()
|
||||
assert res1 == res2
|
||||
|
||||
|
||||
@cachier(stale_after=timedelta(seconds=1), next_time=True)
|
||||
def _being_calc_next_time(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(1)
|
||||
return random() + arg_1 + arg_2
|
||||
|
||||
def _calls_being_calc_next_time(res_queue):
|
||||
res = _being_calc_next_time(0.13, 0.02)
|
||||
res_queue.put(res)
|
||||
|
||||
def test_being_calc_next_time():
|
||||
"""Testing pickle core handling of being calculated scenarios."""
|
||||
_takes_time.clear_cache()
|
||||
_being_calc_next_time(0.13, 0.02)
|
||||
sleep(1.1)
|
||||
res_queue = queue.Queue()
|
||||
thread1 = threading.Thread(
|
||||
target=_calls_being_calc_next_time, kwargs={'res_queue': res_queue})
|
||||
thread2 = threading.Thread(
|
||||
target=_calls_being_calc_next_time, kwargs={'res_queue': res_queue})
|
||||
thread1.start()
|
||||
sleep(0.5)
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
assert res_queue.qsize() == 2
|
||||
res1 = res_queue.get()
|
||||
res2 = res_queue.get()
|
||||
assert res1 == res2
|
||||
|
||||
|
||||
@cachier()
|
||||
def _bad_cache(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(1)
|
||||
return random() + arg_1 + arg_2
|
||||
|
||||
# _BAD_CACHE_FNAME = '.__main__._bad_cache'
|
||||
_BAD_CACHE_FNAME = '.tests.test_pickle_core._bad_cache'
|
||||
_BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME)
|
||||
|
||||
def _calls_bad_cache(res_queue, trash_cache):
|
||||
try:
|
||||
res = _bad_cache(0.13, 0.02)
|
||||
if trash_cache:
|
||||
with open(_BAD_CACHE_FPATH, 'w') as cache_file:
|
||||
cache_file.seek(0)
|
||||
cache_file.truncate()
|
||||
res_queue.put(res)
|
||||
except Exception as exc:
|
||||
res_queue.put(exc)
|
||||
|
||||
def test_bad_cache_file():
|
||||
"""Test pickle core handling of bad cache files."""
|
||||
_bad_cache.clear_cache()
|
||||
res_queue = queue.Queue()
|
||||
thread1 = threading.Thread(
|
||||
target=_calls_bad_cache,
|
||||
kwargs={'res_queue': res_queue, 'trash_cache': True})
|
||||
thread2 = threading.Thread(
|
||||
target=_calls_bad_cache,
|
||||
kwargs={'res_queue': res_queue, 'trash_cache': False})
|
||||
thread1.start()
|
||||
sleep(0.5)
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
assert res_queue.qsize() == 2
|
||||
res1 = res_queue.get()
|
||||
assert isinstance(res1, float)
|
||||
res2 = res_queue.get()
|
||||
assert res2 is None
|
||||
|
||||
|
||||
@cachier()
|
||||
def _delete_cache(arg_1, arg_2):
|
||||
"""Some function."""
|
||||
sleep(1)
|
||||
return random() + arg_1 + arg_2
|
||||
|
||||
# _DEL_CACHE_FNAME = '.__main__._delete_cache'
|
||||
_DEL_CACHE_FNAME = '.tests.test_pickle_core._delete_cache'
|
||||
_DEL_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME)
|
||||
|
||||
def _calls_delete_cache(res_queue, del_cache):
|
||||
try:
|
||||
# print('in')
|
||||
res = _delete_cache(0.13, 0.02)
|
||||
# print('out with {}'.format(res))
|
||||
if del_cache:
|
||||
# print('deleteing!')
|
||||
os.remove(_DEL_CACHE_FPATH)
|
||||
# print(os.path.isfile(_DEL_CACHE_FPATH))
|
||||
res_queue.put(res)
|
||||
except Exception as exc:
|
||||
# print('found')
|
||||
res_queue.put(exc)
|
||||
|
||||
def test_delete_cache_file():
|
||||
"""Test pickle core handling of missing cache files."""
|
||||
_delete_cache.clear_cache()
|
||||
res_queue = queue.Queue()
|
||||
thread1 = threading.Thread(
|
||||
target=_calls_delete_cache,
|
||||
kwargs={'res_queue': res_queue, 'del_cache': True})
|
||||
thread2 = threading.Thread(
|
||||
target=_calls_delete_cache,
|
||||
kwargs={'res_queue': res_queue, 'del_cache': False})
|
||||
thread1.start()
|
||||
sleep(0.5)
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
assert res_queue.qsize() == 2
|
||||
res1 = res_queue.get()
|
||||
# print(res1)
|
||||
assert isinstance(res1, float)
|
||||
res2 = res_queue.get()
|
||||
assert isinstance(res2, KeyError)
|
||||
# print(res2)
|
||||
# print(type(res2))
|
||||
|
||||
def test_clear_being_calculated():
|
||||
"""Test pickle core clear `being calculated` functionality."""
|
||||
_takes_time.clear_being_calculated()
|
||||
|
||||
@cachier(stale_after=timedelta(seconds=1), next_time=True)
|
||||
def _error_throwing_func(arg1):
|
||||
if not hasattr(_error_throwing_func, 'count'):
|
||||
_error_throwing_func.count = 0
|
||||
_error_throwing_func.count += 1
|
||||
if _error_throwing_func.count > 1:
|
||||
raise ValueError("Tiny Rick!")
|
||||
return 7
|
||||
|
||||
|
||||
def test_error_throwing_func():
|
||||
# with
|
||||
res1 = _error_throwing_func(4)
|
||||
sleep(1.5)
|
||||
res2 = _error_throwing_func(4)
|
||||
assert res1 == res2
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_mongo_being_calculated()
|
||||
Reference in New Issue
Block a user