tests are now a bit organized

.gitignore

@@ -87,3 +87,6 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+# added by shaypal5@gmail.com
+cachier_test_mongo_cred.yml

cachier/core.py

@@ -12,7 +12,18 @@ from functools import wraps
 import pickle  # for local caching
 import datetime
 import abc  # for the _BaseCore abstract base class
-import concurrent.futures  # for asynchronous file uploads
+try:  # for asynchronous file uploads
+    from concurrent.futures import ThreadPoolExecutor
+except ImportError:  # we're in python 2.x
+    import pip
+    PACKAGES = [
+        package.project_name
+        for package
+        in pip.get_installed_distributions()
+    ]
+    if 'futures' not in PACKAGES:
+        pip.main(['install', 'futures'])
+    from concurrent.futures import ThreadPoolExecutor
 import time   # to sleep when waiting on Mongo cache
 import fcntl  # to lock on pickle cache IO
 
@@ -81,7 +92,7 @@ class _BaseCore(object):
 class _MongoCore(_BaseCore):
 
     def __init__(self, mongetter, stale_after, next_time):
-        _BaseCore.__init__(stale_after, next_time)
+        _BaseCore.__init__(self, stale_after, next_time)
         self.mongetter = mongetter
         self.mongo_collection = None
 
@@ -203,6 +214,7 @@ class _PickleCore(_BaseCore):
 
         def __init__(self, filename, core, key):
             PatternMatchingEventHandler.__init__(
+                self,
                 patterns=["*" + filename],
                 ignore_patterns=None,
                 ignore_directories=True,
@@ -235,7 +247,7 @@ class _PickleCore(_BaseCore):
             self._check_calculation()
 
     def __init__(self, stale_after, next_time, reload):
-        _BaseCore.__init__(stale_after, next_time)
+        _BaseCore.__init__(self, stale_after, next_time)
         self.cache = None
         self.reload = reload
 
@@ -375,13 +387,11 @@ def _set_max_workets(max_workers):
 
 def _get_executor(reset=False):
     if reset:
-        _get_executor.executor = concurrent.futures.ThreadPoolExecutor(
-            _max_workers())
+        _get_executor.executor = ThreadPoolExecutor(_max_workers())
     try:
         return _get_executor.executor
     except AttributeError:
-        _get_executor.executor = concurrent.futures.ThreadPoolExecutor(
-            _max_workers())
+        _get_executor.executor = ThreadPoolExecutor(_max_workers())
         return _get_executor.executor
 
 
@@ -392,8 +402,7 @@ def _function_thread(core, key, func, args, kwds):
     except BaseException as exc:  # pylint: disable=W0703
         print(
             'Function call failed with the following exception:\n{}'.format(
-                exc),
-            flush=True
+                exc)
         )
 
 

tests/test_cachier.py

@@ -9,86 +9,158 @@
 
 import time
 import datetime
-from cachier import cachier
-from datapy.mongo import get_collection
 import random
+import yaml
+
+from cachier import cachier
+from pymongo.mongo_client import MongoClient
+try:
+    from functools import lru_cache
+except ImportError:
+    from repoze.lru import lru_cache
 
 
+CRED_FILE_NAME = 'cachier_test_mongo_cred.yml'
+
+def _get_mongo_cred():
+    try:
+        with open(CRED_FILE_NAME, 'r') as mongo_cred_file:
+            return yaml.load(mongo_cred_file)
+    except FileNotFoundError:
+        msg = 'A MongoDB credentials file is missing. '
+        msg += 'Please add a file named cachier_test_mongo_cred.yml to the'
+        msg += ' tests directory of cachier, pointing to a MongoDB instance'
+        msg += 'to be used for testing, with the following format:\n'
+        msg += '---- Format begins below ----\n'
+        msg += 'host: some_host.com\n'
+        msg += 'port: 27017\n'
+        msg += 'username: my_username\n'
+        msg += 'password: my_password\n'
+        msg += '---- Format ended above ----\n'
+        raise FileNotFoundError(msg)
+
+
+def _build_mongo_uri(mongo_cred):
+    uri = 'mongodb://{username}:{password}@{host}:{port}'.format(**mongo_cred)
+    print(uri)
+    return uri
+
+
+def _get_cachier_db_mongo_client():
+    # mongo_uri = _build_mongo_uri(_get_mongo_cred())
+    # return MongoClient(host=mongo_uri)
+    mongo_cred = _get_mongo_cred()
+    client = MongoClient(host=mongo_cred['host'], port=mongo_cred['port'])
+    client.cachier_test.authenticate(
+        name=mongo_cred['username'],
+        password=mongo_cred['password'],
+        mechanism='SCRAM-SHA-1'
+    )
+    return client
+
+
+@lru_cache(2)
 def _mongo_getter():
-    return get_collection('cachier', server_name='production', mode='writing')
+    return _get_cachier_db_mongo_client()['cachier_test']['cachier_test']
 
 
 # Pickle core tests
 
 @cachier(next_time=True)
-def test_int_pickling(int_1, int_2):
+def _test_int_pickling(int_1, int_2):
     """Add the two given ints."""
     return int_1 + int_2
 
 
-def test_int_pickling_compare(int_1, int_2):
+def _test_int_pickling_compare(int_1, int_2):
     """Add the two given ints."""
     return int_1 + int_2
 
 
-def test_speed():
+def test_pickle_speed():
     """Test speeds"""
-    num_of_vals = 10000
+    print("    * Comparing speeds of decorated vs non-decorated functions...")
+    num_of_vals = 1000
     times = []
     for i in range(1, num_of_vals):
         tic = time.time()
-        test_int_pickling_compare(i, i + 1)
+        _test_int_pickling_compare(i, i + 1)
         toc = time.time()
         times.append(toc - tic)
-    print('Non-decorated average = {:.8f}'.format(sum(times) / num_of_vals))
+    print('      - Non-decorated average = {:.8f}'.format(
+        sum(times) / num_of_vals))
 
-    test_int_pickling.clear_cache()
+    _test_int_pickling.clear_cache()
     times = []
     for i in range(1, num_of_vals):
         tic = time.time()
-        test_int_pickling(i, i + 1)
+        _test_int_pickling(i, i + 1)
         toc = time.time()
         times.append(toc - tic)
-    print('Decorated average = {:.8f}'.format(sum(times) / num_of_vals))
+    print('      - Decorated average = {:.8f}'.format(
+        sum(times) / num_of_vals))
 
 
 @cachier(next_time=False)
-def takes_30_seconds(arg_1, arg_2):
+def _takes_5_seconds(arg_1, arg_2):
     """Some function."""
-    time.sleep(30)
+    time.sleep(5)
     return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
 
 
-DELTA = datetime.timedelta(seconds=10)
+def test_pickle_core():
+    """Basic Pickle core functionality."""
+    print("    * Testing basic Pickle core functionality.")
+    _takes_5_seconds.clear_cache()
+    stringi = _takes_5_seconds('a', 'b')
+    start = time.time()
+    stringi = _takes_5_seconds('a', 'b')
+    end = time.time()
+    assert end - start < 1
 
 
+DELTA = datetime.timedelta(seconds=3)
+
 @cachier(stale_after=DELTA, next_time=False)
-def stale_after_seconds(arg_1, arg_2):
+def _stale_after_seconds(arg_1, arg_2):
     """Some function."""
-    return {'arg_1': arg_1, 'arg_2': arg_2}
+    return random.random()
 
 
-# Mongo core tests
+def test_stale_after():
+    """Testing the stale_after functionality."""
+    print("    * Testing the stale_after functionality.")
+    _stale_after_seconds.clear_cache()
+    val1 = _stale_after_seconds(1, 2)
+    val2 = _stale_after_seconds(1, 2)
+    val3 = _stale_after_seconds(1, 3)
+    assert val1 == val2
+    assert val1 != val3
+    time.sleep(3)
+    val4 = _stale_after_seconds(1, 2)
+    assert val4 != val1
 
-@cachier(mongetter=_mongo_getter, next_time=True)
-def test_mongo_caching(arg_1, arg_2):
+
+@cachier(stale_after=DELTA, next_time=True)
+def _stale_after_next_time(arg_1, arg_2):
     """Some function."""
-    return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
+    return random.random()
 
 
-@cachier(mongetter=_mongo_getter, next_time=False)
-def takes_30_seconds_mongo(arg_1, arg_2):
-    """Some function."""
-    time.sleep(30)
-    return 'arg_1:{}, arg_2:{}'.format(arg_1, arg_2)
-
-MONGO_DELTA = datetime.timedelta(seconds=30)
-
-
-@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False)
-def stale_after_mongo(arg_1, arg_2):
-    """Some function."""
-    return {'arg_1': arg_1, 'arg_2': arg_2}
+def test_stale_after_next_time():
+    """Testing the stale_after with next_time functionality."""
+    print("    * Testing the stale_after with next_time functionality.")
+    _stale_after_next_time.clear_cache()
+    val1 = _stale_after_next_time(1, 2)
+    val2 = _stale_after_next_time(1, 2)
+    val3 = _stale_after_next_time(1, 3)
+    assert val1 == val2
+    assert val1 != val3
+    time.sleep(3)
+    val4 = _stale_after_next_time(1, 2)
+    assert val4 == val1
+    val5 = _stale_after_next_time(1, 2)
+    assert val5 != val1
 
 
 @cachier()
@@ -98,12 +170,13 @@ def _random_num():
 
 @cachier()
 def _random_num_with_arg(a):
-    print(a)
+    # print(a)
     return random.random()
 
 
 def test_overwrite_cache():
     """Tests that the overwrite feature works correctly."""
+    print("    * Tests that the overwrite feature works correctly.")
     _random_num.clear_cache()
     int1 = _random_num()
     int2 = _random_num()
@@ -124,7 +197,8 @@ def test_overwrite_cache():
 
 
 def test_ignore_cache():
-    """Tests that the overwrite feature works correctly."""
+    """Tests that the ignore_cache feature works correctly."""
+    print("    * Tests that the ignore_cache feature works correctly.")
     _random_num.clear_cache()
     int1 = _random_num()
     int2 = _random_num()
@@ -144,3 +218,72 @@ def test_ignore_cache():
     int4 = _random_num_with_arg('a')
     assert int4 != int3
     assert int4 == int1
+
+
+# Mongo core tests
+
+@cachier(mongetter=_mongo_getter)
+def _test_mongo_caching(arg_1, arg_2):
+    """Some function."""
+    return random.random() + arg_1 + arg_2
+
+
+def test_mongo_core():
+    """Basic Mongo core functionality."""
+    print("    * Testing basic MongoDB core functionality.")
+    _test_mongo_caching.clear_cache()
+    val1 = _test_mongo_caching(1, 2)
+    val2 = _test_mongo_caching(1, 2)
+    assert val1 == val2
+    val3 = _test_mongo_caching(1, 2, ignore_cache=True)
+    assert val3 != val1
+    val4 = _test_mongo_caching(1, 2)
+    assert val4 == val1
+    val5 = _test_mongo_caching(1, 2, overwrite_cache=True)
+    assert val5 != val1
+    val6 = _test_mongo_caching(1, 2)
+    assert val6 == val5
+
+
+MONGO_DELTA = datetime.timedelta(seconds=3)
+
+@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False)
+def _stale_after_mongo(arg_1, arg_2):
+    """Some function."""
+    return random.random() + arg_1 + arg_2
+
+
+def test_mongo_stale_after():
+    """Basic Mongo core functionality."""
+    print("    * Testing MongoDB core stale_after functionality.")
+    _stale_after_mongo.clear_cache()
+    val1 = _stale_after_mongo(1, 2)
+    val2 = _stale_after_mongo(1, 2)
+    assert val1 == val2
+    time.sleep(3)
+    val3 = _stale_after_mongo(1, 2)
+    assert val3 != val1
+
+
+# Main
+
+def main():
+    """Calling all tests."""
+    print("\nCalling all tests for Cachier.\n")
+    print("--- Calling all Pickle core tests...")
+    test_pickle_core()
+    test_stale_after()
+    test_stale_after_next_time()
+    test_overwrite_cache()
+    test_ignore_cache()
+    test_pickle_speed()
+    print("=== All Pickle core tests passed.\n")
+    print("--- Calling all MongoDB core tests...")
+    test_mongo_core()
+    test_mongo_stale_after()
+    print("=== All MongoDB core tests passed.\n")
+    print("All tests passed.")
+
+
+if __name__ == "__main__":
+    main()