coverage upped to ~93%

This commit is contained in:
Shay Palachy
2017-04-26 11:31:34 +03:00
parent d10d720b3d
commit 88542cf8fb
14 changed files with 1813 additions and 323 deletions
+2 -1
View File
@@ -25,4 +25,5 @@ ignore:
- "versioneer.py"
- "tests"
- "**/_version.py"
- "**/__init__.py"
- "**/__init__.py"
- "cachier/scripts"
+1
View File
@@ -4,6 +4,7 @@ omit =
tests/*
cachier/_version.py
cachier/__init__.py
**/scripts/**
[report]
show_missing = True
# Regexes for lines to exclude from consideration
+1339 -108
View File
File diff suppressed because it is too large Load Diff
+6 -4
View File
@@ -33,19 +33,21 @@ from .pickle_core import _PickleCore
from .mongo_core import _MongoCore
MAX_WORKERS_ENVAR_NAME = 'CACHIER_MAX_WORKERS'
DEFAULT_MAX_WORKERS = 8
def _max_workers():
try:
return int(os.environ['CACHIER_MAX_WORKERS'])
return int(os.environ[MAX_WORKERS_ENVAR_NAME])
except KeyError:
os.environ['CACHIER_MAX_WORKERS'] = str(DEFAULT_MAX_WORKERS)
os.environ[MAX_WORKERS_ENVAR_NAME] = str(DEFAULT_MAX_WORKERS)
return DEFAULT_MAX_WORKERS
def _set_max_workets(max_workers):
os.environ['CACHIER_MAX_WORKERS'] = str(max_workers)
def _set_max_workers(max_workers):
os.environ[MAX_WORKERS_ENVAR_NAME] = str(max_workers)
_get_executor(True)
+75 -62
View File
@@ -9,9 +9,10 @@
import os
import pickle # for local caching
import fcntl # to lock on pickle cache IO
from datetime import datetime
import threading
import portalocker # to lock on pickle cache IO
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
#Altenative: https://github.com/WoLpH/portalocker
@@ -50,14 +51,19 @@ class _PickleCore(_BaseCore):
entry = self.core.get_entry_by_key(self.key, True)[1]
# print(self.key)
# print(entry)
if not entry['being_calculated']:
# print('stoping observer!')
self.value = entry['value']
try:
if not entry['being_calculated']:
# print('stoping observer!')
self.value = entry['value']
self.observer.stop()
# else:
# print('NOT stoping observer... :(')
except TypeError:
self.value = None
self.observer.stop()
# print('NOT stoping observer... :(')
def on_created(self, event):
self._check_calculation()
self._check_calculation() # pragma: no cover
def on_modified(self, event):
self._check_calculation()
@@ -66,6 +72,7 @@ class _PickleCore(_BaseCore):
_BaseCore.__init__(self, stale_after, next_time)
self.cache = None
self.reload = reload
self.lock = threading.RLock()
def _cache_fname(self):
if not hasattr(self, 'cache_fname'):
@@ -85,37 +92,37 @@ class _PickleCore(_BaseCore):
return self.cache_fpath
def _reload_cache(self):
fpath = self._cache_fpath()
try:
with open(fpath, 'rb') as cache_file:
fcntl.flock(cache_file, fcntl.LOCK_SH)
try:
self.cache = pickle.load(cache_file)
except EOFError:
self.cache = {}
fcntl.flock(cache_file, fcntl.LOCK_UN)
except FileNotFoundError:
self.cache = {}
with self.lock:
fpath = self._cache_fpath()
try:
with portalocker.Lock(fpath, mode='rb') as cache_file:
try:
self.cache = pickle.load(cache_file)
except EOFError:
self.cache = {}
except FileNotFoundError:
self.cache = {}
def _get_cache(self):
if not self.cache:
self._reload_cache()
return self.cache
with self.lock:
if not self.cache:
self._reload_cache()
return self.cache
def _save_cache(self, cache):
self.cache = cache
fpath = self._cache_fpath()
with open(fpath, 'wb') as cache_file:
fcntl.flock(cache_file, fcntl.LOCK_EX)
pickle.dump(cache, cache_file)
fcntl.flock(cache_file, fcntl.LOCK_UN)
self._reload_cache()
with self.lock:
self.cache = cache
fpath = self._cache_fpath()
with portalocker.Lock(fpath, mode='wb') as cache_file:
pickle.dump(cache, cache_file)
self._reload_cache()
def get_entry_by_key(self, key, reload=False): # pylint: disable=W0221
# print('{}, {}'.format(self.reload, reload))
if self.reload or reload:
self._reload_cache()
return key, self._get_cache().get(key, None)
with self.lock:
# print('{}, {}'.format(self.reload, reload))
if self.reload or reload:
self._reload_cache()
return key, self._get_cache().get(key, None)
def get_entry(self, args, kwds):
key = args + tuple(sorted(kwds.items()))
@@ -123,40 +130,45 @@ class _PickleCore(_BaseCore):
return self.get_entry_by_key(key)
def set_entry(self, key, func_res):
cache = self._get_cache()
cache[key] = {
'value': func_res,
'time': datetime.now(),
'stale': False,
'being_calculated': False
}
self._save_cache(cache)
def mark_entry_being_calculated(self, key):
cache = self._get_cache()
try:
cache[key]['being_calculated'] = True
except KeyError:
with self.lock:
cache = self._get_cache()
cache[key] = {
'value': None,
'value': func_res,
'time': datetime.now(),
'stale': False,
'being_calculated': True
'being_calculated': False
}
self._save_cache(cache)
self._save_cache(cache)
def mark_entry_being_calculated(self, key):
with self.lock:
cache = self._get_cache()
try:
cache[key]['being_calculated'] = True
except KeyError:
cache[key] = {
'value': None,
'time': datetime.now(),
'stale': False,
'being_calculated': True
}
self._save_cache(cache)
def mark_entry_not_calculated(self, key):
cache = self._get_cache()
try:
cache[key]['being_calculated'] = False
self._save_cache(cache)
except KeyError:
pass # that's ok, we don't need an entry in that case
with self.lock:
cache = self._get_cache()
try:
cache[key]['being_calculated'] = False
self._save_cache(cache)
except KeyError:
pass # that's ok, we don't need an entry in that case
def wait_on_entry_calc(self, key):
entry = self._get_cache()[key]
if not entry['being_calculated']:
return entry['value']
with self.lock:
self._reload_cache()
entry = self._get_cache()[key]
if not entry['being_calculated']:
return entry['value']
event_handler = _PickleCore.CacheChangeHandler(
filename=self._cache_fname(),
core=self,
@@ -170,7 +182,7 @@ class _PickleCore(_BaseCore):
recursive=True
)
observer.start()
observer.join(timeout=2.0)
observer.join(timeout=1.0)
if observer.isAlive():
# print('Timedout waiting. Starting again...')
return self.wait_on_entry_calc(key)
@@ -181,7 +193,8 @@ class _PickleCore(_BaseCore):
self._save_cache({})
def clear_being_calculated(self):
cache = self._get_cache()
for key in cache:
cache[key]['being_calculated'] = False
self._save_cache(cache)
with self.lock:
cache = self._get_cache()
for key in cache:
cache[key]['being_calculated'] = False
self._save_cache(cache)
View File
+18
View File
@@ -0,0 +1,18 @@
"A command-line interface for cachier."
import click
from cachier.core import _set_max_workers
@click.group()
def cli():
"""A command-line interface for cachier."""
pass
@cli.command("Limits the number of worker threads used by cachier.")
@click.argument('max_workers', nargs=1, type=int)
def set_max_workers(max_workers):
"""Limits the number of worker threads used by cachier."""
_set_max_workers(max_workers)
+4
View File
@@ -33,6 +33,10 @@ setup(
author_email='shay.palachy@gmail.com',
url='https://github.com/shaypal5/cachier',
packages=['cachier'],
entry_points='''
[console_scripts]
cachier=cachier.scripts.cli:cli
''',
install_requires=[
'watchdog'
],
-1
View File
@@ -1 +0,0 @@
from .test_cachier import *
+5 -1
View File
@@ -1,7 +1,10 @@
"""Configuration file for pytest."""
import pytest
import shutil
from .test_mongo_core import _test_mongetter
from cachier.pickle_core import EXPANDED_CACHIER_DIR
def mongo_finalizer():
@@ -14,5 +17,6 @@ def mongo_finalizer():
@pytest.fixture(scope="session", autouse=True)
def do_something(request):
"""Sessions-scopre pytest hook."""
"""Session-scope pytest hook."""
shutil.rmtree(EXPANDED_CACHIER_DIR)
request.addfinalizer(mongo_finalizer)
-146
View File
@@ -1,146 +0,0 @@
"""Test for the Cachier python package."""
# This file is part of Cachier.
# https://github.com/shaypal5/cachier
# Licensed under the MIT license:
# http://www.opensource.org/licenses/MIT-license
# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
# from os.path import (
# realpath,
# dirname
# )
from time import (
time,
sleep
)
from datetime import timedelta
from random import random
from cachier import cachier
# Pickle core tests
@cachier(next_time=False)
def _takes_5_seconds(arg_1, arg_2):
"""Some function."""
sleep(5)
return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
def test_pickle_core():
"""Basic Pickle core functionality."""
_takes_5_seconds.clear_cache()
stringi = _takes_5_seconds('a', 'b')
start = time()
stringi = _takes_5_seconds('a', 'b')
end = time()
assert end - start < 1
_takes_5_seconds.clear_cache()
DELTA = timedelta(seconds=3)
@cachier(stale_after=DELTA, next_time=False)
def _stale_after_seconds(arg_1, arg_2):
"""Some function."""
return random()
def test_stale_after():
"""Testing the stale_after functionality."""
_stale_after_seconds.clear_cache()
val1 = _stale_after_seconds(1, 2)
val2 = _stale_after_seconds(1, 2)
val3 = _stale_after_seconds(1, 3)
assert val1 == val2
assert val1 != val3
sleep(3)
val4 = _stale_after_seconds(1, 2)
assert val4 != val1
_stale_after_seconds.clear_cache()
@cachier(stale_after=DELTA, next_time=True)
def _stale_after_next_time(arg_1, arg_2):
"""Some function."""
return random()
def test_stale_after_next_time():
"""Testing the stale_after with next_time functionality."""
_stale_after_next_time.clear_cache()
val1 = _stale_after_next_time(1, 2)
val2 = _stale_after_next_time(1, 2)
val3 = _stale_after_next_time(1, 3)
assert val1 == val2
assert val1 != val3
sleep(3)
val4 = _stale_after_next_time(1, 2)
assert val4 == val1
val5 = _stale_after_next_time(1, 2)
assert val5 != val1
_stale_after_next_time.clear_cache()
@cachier()
def _random_num():
return random()
@cachier()
def _random_num_with_arg(a):
# print(a)
return random()
def test_overwrite_cache():
"""Tests that the overwrite feature works correctly."""
_random_num.clear_cache()
int1 = _random_num()
int2 = _random_num()
assert int2 == int1
int3 = _random_num(overwrite_cache=True)
assert int3 != int1
int4 = _random_num()
assert int4 == int3
_random_num.clear_cache()
_random_num_with_arg.clear_cache()
int1 = _random_num_with_arg('a')
int2 = _random_num_with_arg('a')
assert int2 == int1
int3 = _random_num_with_arg('a', overwrite_cache=True)
assert int3 != int1
int4 = _random_num_with_arg('a')
assert int4 == int3
_random_num_with_arg.clear_cache()
def test_ignore_cache():
"""Tests that the ignore_cache feature works correctly."""
_random_num.clear_cache()
int1 = _random_num()
int2 = _random_num()
assert int2 == int1
int3 = _random_num(ignore_cache=True)
assert int3 != int1
int4 = _random_num()
assert int4 != int3
assert int4 == int1
_random_num.clear_cache()
_random_num_with_arg.clear_cache()
int1 = _random_num_with_arg('a')
int2 = _random_num_with_arg('a')
assert int2 == int1
int3 = _random_num_with_arg('a', ignore_cache=True)
assert int3 != int1
int4 = _random_num_with_arg('a')
assert int4 != int3
assert int4 == int1
_random_num_with_arg.clear_cache()
+34
View File
@@ -0,0 +1,34 @@
"""Non-core-specific tests for cachier."""
import os
from cachier.core import (
MAX_WORKERS_ENVAR_NAME,
DEFAULT_MAX_WORKERS,
_max_workers,
_set_max_workers,
_get_executor
)
def test_max_workers():
"""Just call this function for coverage."""
try:
del os.environ[MAX_WORKERS_ENVAR_NAME]
except KeyError:
pass
assert _max_workers() == DEFAULT_MAX_WORKERS
def test_get_executor():
"""Just call this function for coverage."""
_get_executor()
_get_executor(False)
_get_executor(True)
def test_set_max_workers():
"""Just call this function for coverage."""
_set_max_workers(9)
+1
View File
@@ -117,6 +117,7 @@ def test_mongo_being_calculated():
thread2 = threading.Thread(
target=_calls_takes_time, kwargs={'res_queue': res_queue})
thread1.start()
sleep(0.5)
thread2.start()
thread1.join()
thread2.join()
+328
View File
@@ -0,0 +1,328 @@
"""Test for the Cachier python package."""
# This file is part of Cachier.
# https://github.com/shaypal5/cachier
# Licensed under the MIT license:
# http://www.opensource.org/licenses/MIT-license
# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
# from os.path import (
# realpath,
# dirname
# )
import os
from time import (
time,
sleep
)
from datetime import timedelta
from random import random
import threading
try:
import queue
except ImportError: # python 2
import Queue as queue
from cachier import cachier
from cachier.pickle_core import EXPANDED_CACHIER_DIR
# Pickle core tests
@cachier(next_time=False)
def _takes_5_seconds(arg_1, arg_2):
"""Some function."""
sleep(5)
return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
def test_pickle_core():
"""Basic Pickle core functionality."""
_takes_5_seconds.clear_cache()
stringi = _takes_5_seconds('a', 'b')
start = time()
stringi = _takes_5_seconds('a', 'b', verbose_cache=True)
end = time()
assert end - start < 1
_takes_5_seconds.clear_cache()
DELTA = timedelta(seconds=3)
@cachier(stale_after=DELTA, next_time=False)
def _stale_after_seconds(arg_1, arg_2):
"""Some function."""
return random()
def test_stale_after():
"""Testing the stale_after functionality."""
_stale_after_seconds.clear_cache()
val1 = _stale_after_seconds(1, 2)
val2 = _stale_after_seconds(1, 2)
val3 = _stale_after_seconds(1, 3)
assert val1 == val2
assert val1 != val3
sleep(3)
val4 = _stale_after_seconds(1, 2)
assert val4 != val1
_stale_after_seconds.clear_cache()
@cachier(stale_after=DELTA, next_time=True)
def _stale_after_next_time(arg_1, arg_2):
"""Some function."""
return random()
def test_stale_after_next_time():
"""Testing the stale_after with next_time functionality."""
_stale_after_next_time.clear_cache()
val1 = _stale_after_next_time(1, 2)
val2 = _stale_after_next_time(1, 2)
val3 = _stale_after_next_time(1, 3)
assert val1 == val2
assert val1 != val3
sleep(3)
val4 = _stale_after_next_time(1, 2)
assert val4 == val1
val5 = _stale_after_next_time(1, 2)
assert val5 != val1
_stale_after_next_time.clear_cache()
@cachier()
def _random_num():
return random()
@cachier()
def _random_num_with_arg(a):
# print(a)
return random()
def test_overwrite_cache():
"""Tests that the overwrite feature works correctly."""
_random_num.clear_cache()
int1 = _random_num()
int2 = _random_num()
assert int2 == int1
int3 = _random_num(overwrite_cache=True)
assert int3 != int1
int4 = _random_num()
assert int4 == int3
_random_num.clear_cache()
_random_num_with_arg.clear_cache()
int1 = _random_num_with_arg('a')
int2 = _random_num_with_arg('a')
assert int2 == int1
int3 = _random_num_with_arg('a', overwrite_cache=True)
assert int3 != int1
int4 = _random_num_with_arg('a')
assert int4 == int3
_random_num_with_arg.clear_cache()
def test_ignore_cache():
"""Tests that the ignore_cache feature works correctly."""
_random_num.clear_cache()
int1 = _random_num()
int2 = _random_num()
assert int2 == int1
int3 = _random_num(ignore_cache=True)
assert int3 != int1
int4 = _random_num()
assert int4 != int3
assert int4 == int1
_random_num.clear_cache()
_random_num_with_arg.clear_cache()
int1 = _random_num_with_arg('a')
int2 = _random_num_with_arg('a')
assert int2 == int1
int3 = _random_num_with_arg('a', ignore_cache=True)
assert int3 != int1
int4 = _random_num_with_arg('a')
assert int4 != int3
assert int4 == int1
_random_num_with_arg.clear_cache()
@cachier()
def _takes_time(arg_1, arg_2):
"""Some function."""
sleep(2) # this has to be enough time for check_calculation to run twice
return random() + arg_1 + arg_2
def _calls_takes_time(res_queue):
res = _takes_time(0.13, 0.02)
res_queue.put(res)
def test_pickle_being_calculated():
"""Testing pickle core handling of being calculated scenarios."""
_takes_time.clear_cache()
res_queue = queue.Queue()
thread1 = threading.Thread(
target=_calls_takes_time, kwargs={'res_queue': res_queue})
thread2 = threading.Thread(
target=_calls_takes_time, kwargs={'res_queue': res_queue})
thread1.start()
sleep(0.5)
thread2.start()
thread1.join()
thread2.join()
assert res_queue.qsize() == 2
res1 = res_queue.get()
res2 = res_queue.get()
assert res1 == res2
@cachier(stale_after=timedelta(seconds=1), next_time=True)
def _being_calc_next_time(arg_1, arg_2):
"""Some function."""
sleep(1)
return random() + arg_1 + arg_2
def _calls_being_calc_next_time(res_queue):
res = _being_calc_next_time(0.13, 0.02)
res_queue.put(res)
def test_being_calc_next_time():
"""Testing pickle core handling of being calculated scenarios."""
_takes_time.clear_cache()
_being_calc_next_time(0.13, 0.02)
sleep(1.1)
res_queue = queue.Queue()
thread1 = threading.Thread(
target=_calls_being_calc_next_time, kwargs={'res_queue': res_queue})
thread2 = threading.Thread(
target=_calls_being_calc_next_time, kwargs={'res_queue': res_queue})
thread1.start()
sleep(0.5)
thread2.start()
thread1.join()
thread2.join()
assert res_queue.qsize() == 2
res1 = res_queue.get()
res2 = res_queue.get()
assert res1 == res2
@cachier()
def _bad_cache(arg_1, arg_2):
"""Some function."""
sleep(1)
return random() + arg_1 + arg_2
# _BAD_CACHE_FNAME = '.__main__._bad_cache'
_BAD_CACHE_FNAME = '.tests.test_pickle_core._bad_cache'
_BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME)
def _calls_bad_cache(res_queue, trash_cache):
try:
res = _bad_cache(0.13, 0.02)
if trash_cache:
with open(_BAD_CACHE_FPATH, 'w') as cache_file:
cache_file.seek(0)
cache_file.truncate()
res_queue.put(res)
except Exception as exc:
res_queue.put(exc)
def test_bad_cache_file():
"""Test pickle core handling of bad cache files."""
_bad_cache.clear_cache()
res_queue = queue.Queue()
thread1 = threading.Thread(
target=_calls_bad_cache,
kwargs={'res_queue': res_queue, 'trash_cache': True})
thread2 = threading.Thread(
target=_calls_bad_cache,
kwargs={'res_queue': res_queue, 'trash_cache': False})
thread1.start()
sleep(0.5)
thread2.start()
thread1.join()
thread2.join()
assert res_queue.qsize() == 2
res1 = res_queue.get()
assert isinstance(res1, float)
res2 = res_queue.get()
assert res2 is None
@cachier()
def _delete_cache(arg_1, arg_2):
"""Some function."""
sleep(1)
return random() + arg_1 + arg_2
# _DEL_CACHE_FNAME = '.__main__._delete_cache'
_DEL_CACHE_FNAME = '.tests.test_pickle_core._delete_cache'
_DEL_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME)
def _calls_delete_cache(res_queue, del_cache):
try:
# print('in')
res = _delete_cache(0.13, 0.02)
# print('out with {}'.format(res))
if del_cache:
# print('deleteing!')
os.remove(_DEL_CACHE_FPATH)
# print(os.path.isfile(_DEL_CACHE_FPATH))
res_queue.put(res)
except Exception as exc:
# print('found')
res_queue.put(exc)
def test_delete_cache_file():
"""Test pickle core handling of missing cache files."""
_delete_cache.clear_cache()
res_queue = queue.Queue()
thread1 = threading.Thread(
target=_calls_delete_cache,
kwargs={'res_queue': res_queue, 'del_cache': True})
thread2 = threading.Thread(
target=_calls_delete_cache,
kwargs={'res_queue': res_queue, 'del_cache': False})
thread1.start()
sleep(0.5)
thread2.start()
thread1.join()
thread2.join()
assert res_queue.qsize() == 2
res1 = res_queue.get()
# print(res1)
assert isinstance(res1, float)
res2 = res_queue.get()
assert isinstance(res2, KeyError)
# print(res2)
# print(type(res2))
def test_clear_being_calculated():
"""Test pickle core clear `being calculated` functionality."""
_takes_time.clear_being_calculated()
@cachier(stale_after=timedelta(seconds=1), next_time=True)
def _error_throwing_func(arg1):
if not hasattr(_error_throwing_func, 'count'):
_error_throwing_func.count = 0
_error_throwing_func.count += 1
if _error_throwing_func.count > 1:
raise ValueError("Tiny Rick!")
return 7
def test_error_throwing_func():
# with
res1 = _error_throwing_func(4)
sleep(1.5)
res2 = _error_throwing_func(4)
assert res1 == res2
if __name__ == '__main__':
test_mongo_being_calculated()