diff --git a/.gitignore b/.gitignore index 72364f9..930263f 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,6 @@ ENV/ # Rope project settings .ropeproject + +# added by shaypal5@gmail.com +cachier_test_mongo_cred.yml \ No newline at end of file diff --git a/cachier/core.py b/cachier/core.py index fccfa24..da35a54 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -12,7 +12,18 @@ from functools import wraps import pickle # for local caching import datetime import abc # for the _BaseCore abstract base class -import concurrent.futures # for asynchronous file uploads +try: # for asynchronous file uploads + from concurrent.futures import ThreadPoolExecutor +except ImportError: # we're in python 2.x + import pip + PACKAGES = [ + package.project_name + for package + in pip.get_installed_distributions() + ] + if 'futures' not in PACKAGES: + pip.main(['install', 'futures']) + from concurrent.futures import ThreadPoolExecutor import time # to sleep when waiting on Mongo cache import fcntl # to lock on pickle cache IO @@ -81,7 +92,7 @@ class _BaseCore(object): class _MongoCore(_BaseCore): def __init__(self, mongetter, stale_after, next_time): - _BaseCore.__init__(stale_after, next_time) + _BaseCore.__init__(self, stale_after, next_time) self.mongetter = mongetter self.mongo_collection = None @@ -203,6 +214,7 @@ class _PickleCore(_BaseCore): def __init__(self, filename, core, key): PatternMatchingEventHandler.__init__( + self, patterns=["*" + filename], ignore_patterns=None, ignore_directories=True, @@ -235,7 +247,7 @@ class _PickleCore(_BaseCore): self._check_calculation() def __init__(self, stale_after, next_time, reload): - _BaseCore.__init__(stale_after, next_time) + _BaseCore.__init__(self, stale_after, next_time) self.cache = None self.reload = reload @@ -375,13 +387,11 @@ def _set_max_workets(max_workers): def _get_executor(reset=False): if reset: - _get_executor.executor = concurrent.futures.ThreadPoolExecutor( - _max_workers()) + _get_executor.executor = ThreadPoolExecutor(_max_workers()) try: return _get_executor.executor except AttributeError: - _get_executor.executor = concurrent.futures.ThreadPoolExecutor( - _max_workers()) + _get_executor.executor = ThreadPoolExecutor(_max_workers()) return _get_executor.executor @@ -392,8 +402,7 @@ def _function_thread(core, key, func, args, kwds): except BaseException as exc: # pylint: disable=W0703 print( 'Function call failed with the following exception:\n{}'.format( - exc), - flush=True + exc) ) diff --git a/tests/test_cachier.py b/tests/test_cachier.py index 37eac1d..bc3a519 100644 --- a/tests/test_cachier.py +++ b/tests/test_cachier.py @@ -9,86 +9,158 @@ import time import datetime -from cachier import cachier -from datapy.mongo import get_collection import random +import yaml + +from cachier import cachier +from pymongo.mongo_client import MongoClient +try: + from functools import lru_cache +except ImportError: + from repoze.lru import lru_cache +CRED_FILE_NAME = 'cachier_test_mongo_cred.yml' + +def _get_mongo_cred(): + try: + with open(CRED_FILE_NAME, 'r') as mongo_cred_file: + return yaml.load(mongo_cred_file) + except FileNotFoundError: + msg = 'A MongoDB credentials file is missing. ' + msg += 'Please add a file named cachier_test_mongo_cred.yml to the' + msg += ' tests directory of cachier, pointing to a MongoDB instance' + msg += 'to be used for testing, with the following format:\n' + msg += '---- Format begins below ----\n' + msg += 'host: some_host.com\n' + msg += 'port: 27017\n' + msg += 'username: my_username\n' + msg += 'password: my_password\n' + msg += '---- Format ended above ----\n' + raise FileNotFoundError(msg) + + +def _build_mongo_uri(mongo_cred): + uri = 'mongodb://{username}:{password}@{host}:{port}'.format(**mongo_cred) + print(uri) + return uri + + +def _get_cachier_db_mongo_client(): + # mongo_uri = _build_mongo_uri(_get_mongo_cred()) + # return MongoClient(host=mongo_uri) + mongo_cred = _get_mongo_cred() + client = MongoClient(host=mongo_cred['host'], port=mongo_cred['port']) + client.cachier_test.authenticate( + name=mongo_cred['username'], + password=mongo_cred['password'], + mechanism='SCRAM-SHA-1' + ) + return client + + +@lru_cache(2) def _mongo_getter(): - return get_collection('cachier', server_name='production', mode='writing') + return _get_cachier_db_mongo_client()['cachier_test']['cachier_test'] # Pickle core tests @cachier(next_time=True) -def test_int_pickling(int_1, int_2): +def _test_int_pickling(int_1, int_2): """Add the two given ints.""" return int_1 + int_2 -def test_int_pickling_compare(int_1, int_2): +def _test_int_pickling_compare(int_1, int_2): """Add the two given ints.""" return int_1 + int_2 -def test_speed(): +def test_pickle_speed(): """Test speeds""" - num_of_vals = 10000 + print(" * Comparing speeds of decorated vs non-decorated functions...") + num_of_vals = 1000 times = [] for i in range(1, num_of_vals): tic = time.time() - test_int_pickling_compare(i, i + 1) + _test_int_pickling_compare(i, i + 1) toc = time.time() times.append(toc - tic) - print('Non-decorated average = {:.8f}'.format(sum(times) / num_of_vals)) + print(' - Non-decorated average = {:.8f}'.format( + sum(times) / num_of_vals)) - test_int_pickling.clear_cache() + _test_int_pickling.clear_cache() times = [] for i in range(1, num_of_vals): tic = time.time() - test_int_pickling(i, i + 1) + _test_int_pickling(i, i + 1) toc = time.time() times.append(toc - tic) - print('Decorated average = {:.8f}'.format(sum(times) / num_of_vals)) + print(' - Decorated average = {:.8f}'.format( + sum(times) / num_of_vals)) @cachier(next_time=False) -def takes_30_seconds(arg_1, arg_2): +def _takes_5_seconds(arg_1, arg_2): """Some function.""" - time.sleep(30) + time.sleep(5) return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2) -DELTA = datetime.timedelta(seconds=10) +def test_pickle_core(): + """Basic Pickle core functionality.""" + print(" * Testing basic Pickle core functionality.") + _takes_5_seconds.clear_cache() + stringi = _takes_5_seconds('a', 'b') + start = time.time() + stringi = _takes_5_seconds('a', 'b') + end = time.time() + assert end - start < 1 +DELTA = datetime.timedelta(seconds=3) + @cachier(stale_after=DELTA, next_time=False) -def stale_after_seconds(arg_1, arg_2): +def _stale_after_seconds(arg_1, arg_2): """Some function.""" - return {'arg_1': arg_1, 'arg_2': arg_2} + return random.random() -# Mongo core tests +def test_stale_after(): + """Testing the stale_after functionality.""" + print(" * Testing the stale_after functionality.") + _stale_after_seconds.clear_cache() + val1 = _stale_after_seconds(1, 2) + val2 = _stale_after_seconds(1, 2) + val3 = _stale_after_seconds(1, 3) + assert val1 == val2 + assert val1 != val3 + time.sleep(3) + val4 = _stale_after_seconds(1, 2) + assert val4 != val1 -@cachier(mongetter=_mongo_getter, next_time=True) -def test_mongo_caching(arg_1, arg_2): + +@cachier(stale_after=DELTA, next_time=True) +def _stale_after_next_time(arg_1, arg_2): """Some function.""" - return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2) + return random.random() -@cachier(mongetter=_mongo_getter, next_time=False) -def takes_30_seconds_mongo(arg_1, arg_2): - """Some function.""" - time.sleep(30) - return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2) - -MONGO_DELTA = datetime.timedelta(seconds=30) - - -@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False) -def stale_after_mongo(arg_1, arg_2): - """Some function.""" - return {'arg_1': arg_1, 'arg_2': arg_2} +def test_stale_after_next_time(): + """Testing the stale_after with next_time functionality.""" + print(" * Testing the stale_after with next_time functionality.") + _stale_after_next_time.clear_cache() + val1 = _stale_after_next_time(1, 2) + val2 = _stale_after_next_time(1, 2) + val3 = _stale_after_next_time(1, 3) + assert val1 == val2 + assert val1 != val3 + time.sleep(3) + val4 = _stale_after_next_time(1, 2) + assert val4 == val1 + val5 = _stale_after_next_time(1, 2) + assert val5 != val1 @cachier() @@ -98,12 +170,13 @@ def _random_num(): @cachier() def _random_num_with_arg(a): - print(a) + # print(a) return random.random() def test_overwrite_cache(): """Tests that the overwrite feature works correctly.""" + print(" * Tests that the overwrite feature works correctly.") _random_num.clear_cache() int1 = _random_num() int2 = _random_num() @@ -124,7 +197,8 @@ def test_overwrite_cache(): def test_ignore_cache(): - """Tests that the overwrite feature works correctly.""" + """Tests that the ignore_cache feature works correctly.""" + print(" * Tests that the ignore_cache feature works correctly.") _random_num.clear_cache() int1 = _random_num() int2 = _random_num() @@ -144,3 +218,72 @@ def test_ignore_cache(): int4 = _random_num_with_arg('a') assert int4 != int3 assert int4 == int1 + + +# Mongo core tests + +@cachier(mongetter=_mongo_getter) +def _test_mongo_caching(arg_1, arg_2): + """Some function.""" + return random.random() + arg_1 + arg_2 + + +def test_mongo_core(): + """Basic Mongo core functionality.""" + print(" * Testing basic MongoDB core functionality.") + _test_mongo_caching.clear_cache() + val1 = _test_mongo_caching(1, 2) + val2 = _test_mongo_caching(1, 2) + assert val1 == val2 + val3 = _test_mongo_caching(1, 2, ignore_cache=True) + assert val3 != val1 + val4 = _test_mongo_caching(1, 2) + assert val4 == val1 + val5 = _test_mongo_caching(1, 2, overwrite_cache=True) + assert val5 != val1 + val6 = _test_mongo_caching(1, 2) + assert val6 == val5 + + +MONGO_DELTA = datetime.timedelta(seconds=3) + +@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False) +def _stale_after_mongo(arg_1, arg_2): + """Some function.""" + return random.random() + arg_1 + arg_2 + + +def test_mongo_stale_after(): + """Basic Mongo core functionality.""" + print(" * Testing MongoDB core stale_after functionality.") + _stale_after_mongo.clear_cache() + val1 = _stale_after_mongo(1, 2) + val2 = _stale_after_mongo(1, 2) + assert val1 == val2 + time.sleep(3) + val3 = _stale_after_mongo(1, 2) + assert val3 != val1 + + +# Main + +def main(): + """Calling all tests.""" + print("\nCalling all tests for Cachier.\n") + print("--- Calling all Pickle core tests...") + test_pickle_core() + test_stale_after() + test_stale_after_next_time() + test_overwrite_cache() + test_ignore_cache() + test_pickle_speed() + print("=== All Pickle core tests passed.\n") + print("--- Calling all MongoDB core tests...") + test_mongo_core() + test_mongo_stale_after() + print("=== All MongoDB core tests passed.\n") + print("All tests passed.") + + +if __name__ == "__main__": + main()