From 0985483e7c69c236afa63c1e5359db286cc99c06 Mon Sep 17 00:00:00 2001 From: mira Date: Sun, 14 Oct 2018 19:19:40 +0200 Subject: [PATCH 1/3] Added RecalculationNeeded Exception --- cachier/core.py | 14 +++++++++----- cachier/mongo_core.py | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cachier/core.py b/cachier/core.py index aba3148..fe3c3fb 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -30,7 +30,7 @@ except ImportError: # we're in python 2.x from concurrent.futures import ThreadPoolExecutor from .pickle_core import _PickleCore -from .mongo_core import _MongoCore +from .mongo_core import _MongoCore, RecalculationNeeded @@ -82,8 +82,6 @@ def _calc_entry(core, key, func, args, kwds): finally: core.mark_entry_not_calculated(key) - - def cachier(stale_after=None, next_time=False, pickle_reload=True, mongetter=None): """A persistent, stale-free memoization decorator. @@ -156,7 +154,10 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, _print('Returning stale.') return entry['value'] # return stale val _print('Already calc. Waiting on change.') - return core.wait_on_entry_calc(key) + try: + return core.wait_on_entry_calc(key) + except RecalculationNeeded: + return _calc_entry(core, key, func, args, kwds) if next_time: _print('Async calc and return stale') try: @@ -173,7 +174,10 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, return entry['value'] if entry['being_calculated']: _print('No value but being calculated. Waiting.') - return core.wait_on_entry_calc(key) + try: + return core.wait_on_entry_calc(key) + except RecalculationNeeded: + return _calc_entry(core, key, func, args, kwds) _print('No entry found. No current calc. Calling like a boss.') return _calc_entry(core, key, func, args, kwds) diff --git a/cachier/mongo_core.py b/cachier/mongo_core.py index 1bf3ebb..7c36ee5 100644 --- a/cachier/mongo_core.py +++ b/cachier/mongo_core.py @@ -28,6 +28,8 @@ from .base_core import _BaseCore MONGO_SLEEP_DURATION_IN_SEC = 1 +class RecalculationNeeded(Exception): + pass class _MongoCore(_BaseCore): @@ -126,6 +128,8 @@ class _MongoCore(_BaseCore): while True: time.sleep(MONGO_SLEEP_DURATION_IN_SEC) key, entry = self.get_entry_by_key(key) + if entry is None: + raise RecalculationNeeded() if entry is not None and not entry['being_calculated']: return entry['value'] From 8be1c81205d10b8d279a65410c6166e0b7b7ea07 Mon Sep 17 00:00:00 2001 From: mira Date: Tue, 16 Oct 2018 23:00:58 +0200 Subject: [PATCH 2/3] added tests --- tests/test_mongo_core.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py index 4255a95..8fbc5e0 100644 --- a/tests/test_mongo_core.py +++ b/tests/test_mongo_core.py @@ -1,5 +1,4 @@ """Testing the MongoDB core of cachier.""" - import sys from random import random from datetime import timedelta @@ -15,8 +14,7 @@ from pymongo.mongo_client import MongoClient from pymongo.errors import OperationFailure from cachier import cachier -from cachier.mongo_core import _MongoCore - +from cachier.mongo_core import _MongoCore, RecalculationNeeded _TEST_HOST = 'ds119508.mlab.com' _TEST_PORT = 19508 @@ -163,3 +161,22 @@ def test_mongo_write_failure(): def test_mongo_clear_being_calculated(): """Testing MongoDB core clear_being_calculated.""" _func_w_bad_mongo.clear_being_calculated() + +@cachier(mongetter=_test_mongetter) +def _stalled_func(): + """ Testing stalled function""" + return 1 + +def test_stalled_mongo_db_cache(): + core = _MongoCore(_test_mongetter, None, False) + core.set_func(_stalled_func) + _stalled_func.clear_cache() + with pytest.raises(RecalculationNeeded): + core.wait_on_entry_calc(key=None) + +def test_stalled_mong_db_core(): + core = _MongoCore(_test_mongetter, None, False) + core.get_entry = lambda: {'being_calculated':True} + res = _stalled_func() + assert res == 1 + From 5210a0d12fe093ae841caf5f1d876de8cad285c6 Mon Sep 17 00:00:00 2001 From: mira Date: Tue, 16 Oct 2018 23:54:17 +0200 Subject: [PATCH 3/3] added tests for mongo_core pending calculation --- tests/test_mongo_core.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py index 8fbc5e0..b2585fe 100644 --- a/tests/test_mongo_core.py +++ b/tests/test_mongo_core.py @@ -1,4 +1,6 @@ """Testing the MongoDB core of cachier.""" + +import datetime import sys from random import random from datetime import timedelta @@ -162,21 +164,42 @@ def test_mongo_clear_being_calculated(): """Testing MongoDB core clear_being_calculated.""" _func_w_bad_mongo.clear_being_calculated() -@cachier(mongetter=_test_mongetter) -def _stalled_func(): - """ Testing stalled function""" - return 1 def test_stalled_mongo_db_cache(): + @cachier(mongetter=_test_mongetter) + def _stalled_func(): + return 1 core = _MongoCore(_test_mongetter, None, False) core.set_func(_stalled_func) - _stalled_func.clear_cache() + core.clear_cache() with pytest.raises(RecalculationNeeded): core.wait_on_entry_calc(key=None) -def test_stalled_mong_db_core(): - core = _MongoCore(_test_mongetter, None, False) - core.get_entry = lambda: {'being_calculated':True} +def test_stalled_mong_db_core(monkeypatch): + def mock_get_entry(self, args, kwargs): + return "key", {'being_calculated': True} + def mock_get_entry_by_key(self, key): + return "key", None + monkeypatch.setattr("cachier.mongo_core._MongoCore.get_entry", mock_get_entry ) + monkeypatch.setattr("cachier.mongo_core._MongoCore.get_entry_by_key", mock_get_entry_by_key ) + @cachier(mongetter=_test_mongetter) + def _stalled_func(): + return 1 res = _stalled_func() assert res == 1 + def mock_get_entry_2(self, args, kwargs): + return "key", {'being_calculated': True, + "value": 1, + "time": datetime.datetime.now() - datetime.timedelta(seconds=10)} + monkeypatch.setattr("cachier.mongo_core._MongoCore.get_entry", mock_get_entry_2 ) + + stale_after = datetime.timedelta(seconds=1) + @cachier(mongetter=_test_mongetter,stale_after=stale_after) + def _stalled_func_2(): + """ Testing stalled function""" + return 2 + res = _stalled_func_2() + assert res == 2 + +