From 7cf96f8726138043d144a1230e0437c7ee713227 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Thu, 20 Apr 2017 11:22:26 +0300 Subject: [PATCH] shaping things up --- .coveragerc | 8 +++ .travis.yml | 32 ++++++++++ README.rst | 84 ++++++++++++++++--------- cachier/core.py | 2 +- codecov.yml | 12 ++++ setup.cfg | 9 +-- setup.py | 38 +++++++++-- tests/test_cachier.py | 133 ++------------------------------------- tests/test_mongo_core.py | 75 ++++++++++++++++++++++ 9 files changed, 226 insertions(+), 167 deletions(-) create mode 100644 .coveragerc create mode 100644 .travis.yml create mode 100644 codecov.yml create mode 100644 tests/test_mongo_core.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..7b4a138 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +branch = True +omit = + tests/* + cachier/_version.py + cachier/__init__.py +[report] +show_missing = True \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f6de007 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +language: python +python: +- '2.7' +- '3.5' +- '3.6' +notifications: + email: + on_success: change + on_failure: always +before_install: + # coverage submission packages + - pip install codecov +install: + - python setup.py test +script: nosetests --cover-erase --with-coverage --cover-package=cachier -d +# --ignore-files="tests_perf\.py" -coveralls +# submit coverage +after_success: + # - coveralls + - codecov +deploy: + provider: pypi + user: shaypal5 + password: + secure: RaTkue2YoXAkT/byvecFuOYdrJpHchCSMYnV4xqUFhQgT8qbyv7/EL+pm6sQ75Ni/JKAhSBdW8l4faKY9X1vW3yc5E2YHBf/81VJo1+JAbD9vGX0RLZbDzIsEVht+hL2Xxvab5Xh3fI7Pcr+cozKKMZxcOvEEwGo5DENM4CquGLeYyUxQOyhwSVjA+54bucFm+u+BA0QxUSHSyFy0cPKJVi8jNMSK/XAvs+Zk26o/MOU3udSj25FBtTkqPTBphaUNkt2EPGK1ZWkS9uhqs+hrETMWj6n6k49WblXvMDJiUtCIM+36Q+GBH/9UxCWuMOjL+uRFjVwwKQEcend9YMvp4+jRZ1HsLAWjMMhazxgkZ6M7bErHRxyabb7om+5IMptPdIo31gw8S1dKktGyuiSVYqs9X2mGA//SoItoDjIAUI81TQ3s2QLw5SfyEARfAdN+QS2aRaZHc0RPcLb10xNu7d7cy/4I88W+dptNqVABCP26Jlg4xsrRGMnONMSt/kWg1enXfgSXJcyrqqAaZzQUs/5QmVGtU4DBl3C8pUiHs6eGdpcqO3vpYoHveNT0WzoONkqZBvzBOjUWf8oMJzI3LcmCruUtktyd9cnlzM49pndnCyXPIKcpqPSGtR0FS3I/QhP7RBB+5xjP39SIuFsEW6ciakOs+AdMePl8tAq01o= + distributions: sdist bdist_wheel + on: + all_branches: true + tags: true + repo: shaypal5/cachier + condition: $TRAVIS_PYTHON_VERSION = "3.5" + skip_upload_docs: true diff --git a/README.rst b/README.rst index af42adf..c8afe1b 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,7 @@ Cachier -======= +####### + +|PyPI-Status| |PyPI-Versions| |LICENCE| Persistent, stale-free cache / memoization decorators for Python. @@ -7,35 +9,40 @@ Persistent, stale-free cache / memoization decorators for Python. from cachier import cachier import datetime - - SHELF_LIFE = datetime.timedelta(days=3) - - @cachier(stale_after=SHELF_LIFE) + + @cachier(stale_after=datetime.timedelta(days=3)) def foo(arg1, arg2): - """foo now has a persistent cache, trigerring recalculation for values stored more than 3 days!""" + """foo now has a persistent cache, trigerring recalculation for values stored more than 3 days.""" return {'arg1': arg1, 'arg2': arg2} .. role:: python(code) :language: python -Dependencies and Setup ----------------------- +.. contents:: -Cachier uses the following packages: +.. section-numbering: -* pymongo_ -* watchdog_ -You can install cachier using: + +Installation +============ + +.. Cachier uses the following packages: + +.. * pymongo_ +.. * watchdog_ + +Install ``cachier`` with: .. code-block:: python pip install cachier Features ----------------------- +======== +* Tested on Linux and OS X systems. Does not support windows. * Compatible with Python 2.7+ and Python 3.5+. * A simple interface. * Defining "shelf life" for cached values. @@ -49,34 +56,33 @@ Cachier is not: * Especially fast. It is meant to replace function calls that take more than... a second, say (overhead is around 1 millisecond). Future features: -~~~~~~~~~~~~~~~~ +---------------- * S3 core. * Multi-core caching. Use ---- +=== The positional and keyword arguments to the wrapped function must be hashable (i.e. Python's immutable built-in objects, not mutable containers). Also, notice that since objects which are instances of user-defined classes are hashable but all compare unequal (their hash value is their id), equal objects across different sessions will not yield identical keys. Setting up a Cache -~~~~~~~~~~~~~~~~~~ +------------------ You can add a deafult, pickle-based, persistent cache to your function - meaning it will last across different Python kernels calling the wrapped function - by decorating it with the ``cachier`` decorator (notice the ``()``!). .. code-block:: python from cachier import cachier - + @cachier() def foo(arg1, arg2): """Your function now has a persistent cache mapped by argument values!""" return {'arg1': arg1, 'arg2': arg2} - - -Resetting a Cache -~~~~~~~~~~~~~~~~~ + +Resetting a Cache +----------------- The Cachier wrapper adds a ``clear_cache()`` function to each wrapped function. To reset the cache of the wrapped function simply call this method: .. code-block:: python @@ -85,23 +91,23 @@ The Cachier wrapper adds a ``clear_cache()`` function to each wrapped function. Setting Shelf Live -~~~~~~~~~~~~~~~~~~ +------------------ You can set any duration as the shelf life of cached return values of a function by providing a corresponding ``timedelta`` object to the ``stale_after`` parameter: .. code-block:: python import datetime - + @cachier(stale_after=datetime.timedelta(weeks=2)) def bar(arg1, arg2): return {'arg1': arg1, 'arg2': arg2} - + Now when a cached value matching the given arguments is found the time of its calculation is checked; if more than ``stale_after`` time has since passed, the function will be run again for the same arguments and the new value will be cached and returned. This is usefull for lengthy calculations that depend on a dynamic data source. Fuzzy Shelf Live -~~~~~~~~~~~~~~~~ +---------------- Sometimes you may want your function to trigger a calculation when it encounters a stale result, but still not wait on it if it's not that critical. In that case you can set ``next_time`` to ``True`` to have your function trigger a recalculation **in a separate thread**, but return the currently cached stale value: .. code-block:: python @@ -112,10 +118,10 @@ Further function calls made while the calculation is being performed will not tr Cachier Cores -------------- +============= Pickle Core -~~~~~~~~~~~~ +----------- The default core for Cachier is pickle based, meaning each function will store its cache is a seperate pickle file in the ``~/.cachier`` directory. Naturally, this kind of cache is both machine-specific and user-specific. @@ -129,7 +135,7 @@ This will prevent reading the cache file on each cache read, speeding things up MongoDB Core -~~~~~~~~~~~~ +------------ You can set a MongoDB-based cache by assigning ``mongetter`` with a callable that returns a ``pymongo.Collection`` object with writing permission: .. code-block:: python @@ -138,6 +144,28 @@ You can set a MongoDB-based cache by assigning ``mongetter`` with a callable tha This allows you to have a cross-machine, albeit slower, cache. + +Credits +======= +Created by Shay Palachy (shay.palachy@gmail.com). + + + +.. |PyPI-Status| image:: https://img.shields.io/pypi/v/cachier.svg + :target: https://pypi.python.org/pypi/cachier + +.. |PyPI-Versions| image:: https://img.shields.io/pypi/pyversions/cachier.svg + :target: https://pypi.python.org/pypi/cachier + +.. |Build-Status| image:: https://travis-ci.org/shaypal5/cachier.svg?branch=master + :target: https://travis-ci.org/shaypal5/cachier + +.. |LICENCE| image:: https://img.shields.io/pypi/l/cachier.svg + :target: https://pypi.python.org/pypi/cachier + +.. |Codecov| image:: https://codecov.io/github/shaypal5/cachier/coverage.svg?branch=master + :target: https://codecov.io/github/shaypal5/cachier?branch=master + .. links: .. _pymongo: https://api.mongodb.com/python/current/ .. _watchdog: https://github.com/gorakhargosh/watchdog diff --git a/cachier/core.py b/cachier/core.py index 6f0915c..c7c4141 100644 --- a/cachier/core.py +++ b/cachier/core.py @@ -102,7 +102,7 @@ def cachier(stale_after=None, next_time=False, pickle_reload=True, pickle_reload (optional) : bool If set to True, in-memory cache will be reloaded on each cache read, enabling different threads to share cache. Should be set to False for - faster reads in single-read programs. Defaults to True. + faster reads in single-thread programs. Defaults to True. mongetter (optional) : callable A callable that takes no arguments and returns a pymongo.Collection object with writing permissions. If unset a local pickle cache is used diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..9a6118d --- /dev/null +++ b/codecov.yml @@ -0,0 +1,12 @@ +comment: + layout: header, changes, diff +coverage: + status: + patch: + default: + target: '80' + project: false +ignore: + - "tests" + - "**/_version.py" + - "**/__init__.py" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 0bfd687..41bd74a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,9 @@ -[bumpversion] -current_version = 0.1.18 +# [bumpversion] +# current_version = 0.1.18 -[metadata] -description-file = README.md +[bdist_wheel] +# Use this option if your package is pure-python +universal = [versioneer] VCS = git diff --git a/setup.py b/setup.py index 4f3c3b6..b75696e 100644 --- a/setup.py +++ b/setup.py @@ -7,23 +7,51 @@ # http://www.opensource.org/licenses/MIT-license # Copyright (c) 2016, Shay Palachy -from setuptools import setup, find_packages +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + import versioneer +README_RST = '' +with open('README.rst') as f: + README_RST = f.read( + setup( - name='Cachier', + name='cachier', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), description='Persistent, stale-free memoization decorators for Python.', + long_description=README_RST, license='MIT', author='Shay Palachy', - author_email='shaypal5@gmail.com', + author_email='shay.palachy@gmail.com', url='https://github.com/shaypal5/cachier', - packages=find_packages(), + packages=['cachier'], install_requires=[ 'pymongo', 'watchdog' ], + setup_requires=['nose', 'coverage'], + test_suite='nose.collector', + platforms=['linux', 'osx'], keywords=['cache', 'persistence', 'mongo', 'memoization', 'decorator'], - classifiers=[], + classifiers=[ + # Trove classifiers + # (https://pypi.python.org/pypi?%3Aaction=list_classifiers) + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Utilities', + 'Topic :: Other/Nonlisted Topic', + 'Intended Audience :: Developers', + ] ) diff --git a/tests/test_cachier.py b/tests/test_cachier.py index 2ef07ae..f94714b 100644 --- a/tests/test_cachier.py +++ b/tests/test_cachier.py @@ -7,70 +7,18 @@ # http://www.opensource.org/licenses/MIT-license # Copyright (c) 2016, Shay Palachy -from os.path import ( - realpath, - dirname -) +# from os.path import ( +# realpath, +# dirname +# ) from time import ( time, sleep ) from datetime import timedelta from random import random -import yaml from cachier import cachier -from pymongo.mongo_client import MongoClient -try: - from functools import lru_cache -except ImportError: - from repoze.lru import lru_cache - - -CRED_FILE_NAME = 'cachier_test_mongo_cred.yml' - -def _get_mongo_cred(): - try: - current_dir = dirname(realpath(__file__)) - cred_file_path = current_dir + '/' + CRED_FILE_NAME - with open(cred_file_path, 'r') as mongo_cred_file: - return yaml.load(mongo_cred_file) - except FileNotFoundError: - msg = 'A MongoDB credentials file is missing. ' - msg += 'Please add a file named cachier_test_mongo_cred.yml to the' - msg += ' tests directory of cachier, pointing to a MongoDB instance' - msg += 'to be used for testing, with the following format:\n' - msg += '---- Format begins below ----\n' - msg += 'host: some_host.com\n' - msg += 'port: 27017\n' - msg += 'username: my_username\n' - msg += 'password: my_password\n' - msg += '---- Format ended above ----\n' - raise FileNotFoundError(msg) - - -def _build_mongo_uri(mongo_cred): - uri = 'mongodb://{username}:{password}@{host}:{port}'.format(**mongo_cred) - print(uri) - return uri - - -def _get_cachier_db_mongo_client(): - # mongo_uri = _build_mongo_uri(_get_mongo_cred()) - # return MongoClient(host=mongo_uri) - mongo_cred = _get_mongo_cred() - client = MongoClient(host=mongo_cred['host'], port=mongo_cred['port']) - client.cachier_test.authenticate( - name=mongo_cred['username'], - password=mongo_cred['password'], - mechanism='SCRAM-SHA-1' - ) - return client - - -@lru_cache(2) -def _mongo_getter(): - return _get_cachier_db_mongo_client()['cachier_test']['cachier_test'] # Pickle core tests @@ -119,7 +67,6 @@ def _takes_5_seconds(arg_1, arg_2): def test_pickle_core(): """Basic Pickle core functionality.""" - print(" * Testing basic Pickle core functionality.") _takes_5_seconds.clear_cache() stringi = _takes_5_seconds('a', 'b') start = time() @@ -138,7 +85,6 @@ def _stale_after_seconds(arg_1, arg_2): def test_stale_after(): """Testing the stale_after functionality.""" - print(" * Testing the stale_after functionality.") _stale_after_seconds.clear_cache() val1 = _stale_after_seconds(1, 2) val2 = _stale_after_seconds(1, 2) @@ -158,7 +104,6 @@ def _stale_after_next_time(arg_1, arg_2): def test_stale_after_next_time(): """Testing the stale_after with next_time functionality.""" - print(" * Testing the stale_after with next_time functionality.") _stale_after_next_time.clear_cache() val1 = _stale_after_next_time(1, 2) val2 = _stale_after_next_time(1, 2) @@ -185,7 +130,6 @@ def _random_num_with_arg(a): def test_overwrite_cache(): """Tests that the overwrite feature works correctly.""" - print(" * Tests that the overwrite feature works correctly.") _random_num.clear_cache() int1 = _random_num() int2 = _random_num() @@ -207,7 +151,6 @@ def test_overwrite_cache(): def test_ignore_cache(): """Tests that the ignore_cache feature works correctly.""" - print(" * Tests that the ignore_cache feature works correctly.") _random_num.clear_cache() int1 = _random_num() int2 = _random_num() @@ -228,71 +171,3 @@ def test_ignore_cache(): assert int4 != int3 assert int4 == int1 - -# Mongo core tests - -@cachier(mongetter=_mongo_getter) -def _test_mongo_caching(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - -def test_mongo_core(): - """Basic Mongo core functionality.""" - print(" * Testing basic MongoDB core functionality.") - _test_mongo_caching.clear_cache() - val1 = _test_mongo_caching(1, 2) - val2 = _test_mongo_caching(1, 2) - assert val1 == val2 - val3 = _test_mongo_caching(1, 2, ignore_cache=True) - assert val3 != val1 - val4 = _test_mongo_caching(1, 2) - assert val4 == val1 - val5 = _test_mongo_caching(1, 2, overwrite_cache=True) - assert val5 != val1 - val6 = _test_mongo_caching(1, 2) - assert val6 == val5 - - -MONGO_DELTA = timedelta(seconds=3) - -@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False) -def _stale_after_mongo(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - -def test_mongo_stale_after(): - """Basic Mongo core functionality.""" - print(" * Testing MongoDB core stale_after functionality.") - _stale_after_mongo.clear_cache() - val1 = _stale_after_mongo(1, 2) - val2 = _stale_after_mongo(1, 2) - assert val1 == val2 - sleep(3) - val3 = _stale_after_mongo(1, 2) - assert val3 != val1 - - -# Main - -def main(): - """Calling all tests.""" - print("\nCalling all tests for Cachier.\n") - print("--- Calling all Pickle core tests...") - test_pickle_core() - test_stale_after() - test_stale_after_next_time() - test_overwrite_cache() - test_ignore_cache() - test_pickle_speed() - print("=== All Pickle core tests passed.\n") - print("--- Calling all MongoDB core tests...") - test_mongo_core() - test_mongo_stale_after() - print("=== All MongoDB core tests passed.\n") - print("All tests passed.") - - -if __name__ == "__main__": - main() diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py new file mode 100644 index 0000000..f4aeaca --- /dev/null +++ b/tests/test_mongo_core.py @@ -0,0 +1,75 @@ +"""Testing the MongoDB core of cachier.""" + +from random import random +from datetime import timedelta +from time import sleep + +from cachier import cachier +from pymongo.mongo_client import MongoClient +try: + from functools import lru_cache +except ImportError: + from repoze.lru import lru_cache + +_TEST_HOST = 'ds119508.mlab.com' +_TEST_PORT = 19508 +_TEST_USERNAME = 'cachier_test' +_TEST_PWD = 'ZGhjO5CQESYJ69U4z65G79YG' + + +def _get_cachier_db_mongo_client(): + client = MongoClient(host=_TEST_HOST, port=_TEST_PORT) + client.cachier_test.authenticate( + name=_TEST_USERNAME, + password=_TEST_PWD, + mechanism='SCRAM-SHA-1' + ) + return client + + +@lru_cache(2) +def _mongo_getter(): + return _get_cachier_db_mongo_client()['cachier_test']['cachier_test'] + + +# Mongo core tests + +@cachier(mongetter=_mongo_getter) +def _test_mongo_caching(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + +def test_mongo_core(): + """Basic Mongo core functionality.""" + _test_mongo_caching.clear_cache() + val1 = _test_mongo_caching(1, 2) + val2 = _test_mongo_caching(1, 2) + assert val1 == val2 + val3 = _test_mongo_caching(1, 2, ignore_cache=True) + assert val3 != val1 + val4 = _test_mongo_caching(1, 2) + assert val4 == val1 + val5 = _test_mongo_caching(1, 2, overwrite_cache=True) + assert val5 != val1 + val6 = _test_mongo_caching(1, 2) + assert val6 == val5 + + +MONGO_DELTA = timedelta(seconds=3) + +@cachier(mongetter=_mongo_getter, stale_after=MONGO_DELTA, next_time=False) +def _stale_after_mongo(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + +def test_mongo_stale_after(): + """Testing MongoDB core stale_after functionality..""" + _stale_after_mongo.clear_cache() + val1 = _stale_after_mongo(1, 2) + val2 = _stale_after_mongo(1, 2) + assert val1 == val2 + sleep(3) + val3 = _stale_after_mongo(1, 2) + assert val3 != val1