From 0f78de46e5f7fbd3b45d6511e7731a3a3f9461d7 Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Sat, 7 Dec 2013 21:18:44 -0600 Subject: [PATCH 1/6] Add dynamic creation of imread_collection function --- skimage/io/collection.py | 4 +-- skimage/io/manage_plugins.py | 57 ++++++++++++++++++++++++++++++----- skimage/io/tests/test_null.py | 11 +++++++ 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/skimage/io/collection.py b/skimage/io/collection.py index 5708983a..972bc55b 100644 --- a/skimage/io/collection.py +++ b/skimage/io/collection.py @@ -2,14 +2,13 @@ from __future__ import with_statement -__all__ = ['MultiImage', 'ImageCollection', 'imread', 'concatenate_images'] +__all__ = ['MultiImage', 'ImageCollection', 'concatenate_images'] from glob import glob import re from copy import copy import numpy as np -from ._io import imread import six @@ -312,6 +311,7 @@ class ImageCollection(object): self._cached = None if load_func is None: + from ._io import imread self.load_func = imread else: self.load_func = load_func diff --git a/skimage/io/manage_plugins.py b/skimage/io/manage_plugins.py index 08bf1703..1da75a0d 100644 --- a/skimage/io/manage_plugins.py +++ b/skimage/io/manage_plugins.py @@ -10,6 +10,8 @@ except ImportError: import os.path from glob import glob +from .collection import ImageCollection + __all__ = ['use_plugin', 'call_plugin', 'plugin_info', 'plugin_order', 'reset_plugins', 'find_available_plugins', 'available_plugins'] @@ -105,7 +107,14 @@ def _scan_plugins(): print("Plugin `%s` wants to provide non-existent `%s`." \ " Ignoring." % (name, p)) + # Add plugins that provide 'imread' as provider of 'imread_collection'. + need_to_add_collection = ('imread_collection' not in valid_provides and + 'imread' in valid_provides) + if need_to_add_collection: + valid_provides.append('imread_collection') + plugin_provides[name] = valid_provides + plugin_module_name[name] = os.path.basename(filename)[:-4] _scan_plugins() @@ -135,7 +144,7 @@ def find_available_plugins(loaded=False): d = {} for plugin in plugin_provides: if not loaded or plugin in active_plugins: - d[plugin] = [f for f in plugin_provides[plugin] \ + d[plugin] = [f for f in plugin_provides[plugin] if not f.startswith('_')] return d @@ -240,6 +249,32 @@ def use_plugin(name, kind=None): plugin_store[k] = funcs +def _imread_collection_wrapper(imread): + def imread_collection(load_pattern, conserve_memory=True): + """Return an `ImageCollection` from files matching the given pattern. + + Note that files are always stored in alphabetical order. Also note that + slicing returns a new ImageCollection, *not* a view into the data. + + See `skimage.io.ImageCollection` for details. + + Parameters + ---------- + load_pattern : str or list + Pattern glob or filenames to load. The path can be absolute or + relative. Multiple patterns should be separated by a colon, + e.g. '/tmp/work/*.png:/tmp/other/*.jpg'. Also see + implementation notes below. + conserve_memory : bool, optional + If True, never keep more than one in memory at a specific + time. Otherwise, images will be cached once they are loaded. + + """ + return ImageCollection(load_pattern, conserve_memory=conserve_memory, + load_func=imread) + return imread_collection + + def _load(plugin): """Load the given plugin. @@ -264,14 +299,22 @@ def _load(plugin): provides = plugin_provides[plugin] for p in provides: - if not hasattr(plugin_module, p): + if p == 'imread_collection': + add_plugin = (not hasattr(plugin_module, 'imread_collection') and + hasattr(plugin_module, 'imread')) + if add_plugin: + imread = getattr(plugin_module, 'imread') + func = _imread_collection_wrapper(imread) + setattr(plugin_module, 'imread_collection', func) + elif not hasattr(plugin_module, p): print("Plugin %s does not provide %s as advertised. Ignoring." % \ (plugin, p)) - else: - store = plugin_store[p] - func = getattr(plugin_module, p) - if not (plugin, func) in store: - store.append((plugin, func)) + continue + + store = plugin_store[p] + func = getattr(plugin_module, p) + if not (plugin, func) in store: + store.append((plugin, func)) def plugin_info(plugin): diff --git a/skimage/io/tests/test_null.py b/skimage/io/tests/test_null.py index 6d3913d9..8895f77c 100644 --- a/skimage/io/tests/test_null.py +++ b/skimage/io/tests/test_null.py @@ -30,6 +30,17 @@ def test_null_imshow(): io.imshow(np.zeros((3, 3)), plugin='null') +@raises(Warning) +def test_null_imread_collection(): + # Note that the null plugin doesn't define an `imread_collection` plugin + # but this function dynamically added by the plugin manager. + path = os.path.join(data_dir, '*.png') + with warnings.catch_warnings(): # Temporarily set warnings as errors. + warnings.filterwarnings('error') + collection = io.imread_collection(path, plugin='null') + collection[0] + + if __name__ == '__main__': from numpy.testing import run_module_suite run_module_suite() From 8a4c5ce354302c49d2a715d1654c83a420835609 Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Sat, 7 Dec 2013 21:41:53 -0600 Subject: [PATCH 2/6] Add comments about how plugins are managed. --- skimage/io/manage_plugins.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/skimage/io/manage_plugins.py b/skimage/io/manage_plugins.py index 1da75a0d..7ce74b1d 100644 --- a/skimage/io/manage_plugins.py +++ b/skimage/io/manage_plugins.py @@ -1,5 +1,19 @@ """Handle image reading, writing and plotting plugins. +To improve performance, plugins are only loaded as needed. As a result, there +can be multiple states for a given plugin: + + available: Defined in an *ini file located in `skimage.io._plugins`. + See also `skimage.io.available_plugins`. + partial definition: Specified in an *ini file, but not defined in the + corresponding plugin module. This will raise an error when loaded. + available but not on this system: Defined in `skimage.io._plugins`, but + a dependent library (e.g. Qt, PIL) is not available on your system. + This will raise an error when loaded. + loaded: The real availability is determined when it's explicitly loaded, + either because it's one of the default plugins, or because it's + loaded explicitly by the user. + """ try: @@ -20,11 +34,14 @@ __all__ = ['use_plugin', 'call_plugin', 'plugin_info', 'plugin_order', # The plugin store will save a list of *loaded* io functions for each io type # (e.g. 'imread', 'imsave', etc.). Plugins are loaded as requested. plugin_store = None - +# Dictionary mapping plugin names to a list of functions they provide. plugin_provides = {} +# The module names for the plugins in `skimage.io._plugins`. plugin_module_name = {} +# Meta-data about plugins provided by *.ini files. plugin_meta_data = {} - +# For each plugin type, default to the first available plugin as defined by +# the following preferences. preferred_plugins = { # Default plugins for all types (overridden by specific types below). 'all': ['matplotlib', 'pil', 'qt', 'freeimage', 'null'], From 19102510cec81406c3d7005791d83aa98b5977df Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Sun, 8 Dec 2013 10:05:32 -0600 Subject: [PATCH 3/6] Move wrapper function --- skimage/io/collection.py | 33 ++++++++++++++++++++++++++++++--- skimage/io/manage_plugins.py | 30 ++---------------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/skimage/io/collection.py b/skimage/io/collection.py index 972bc55b..96d29d4a 100644 --- a/skimage/io/collection.py +++ b/skimage/io/collection.py @@ -2,17 +2,18 @@ from __future__ import with_statement -__all__ = ['MultiImage', 'ImageCollection', 'concatenate_images'] - from glob import glob import re from copy import copy import numpy as np - import six +__all__ = ['MultiImage', 'ImageCollection', 'concatenate_images', + 'imread_collection_wrapper'] + + def concatenate_images(ic): """Concatenate all images in the image collection into an array. @@ -430,3 +431,29 @@ class ImageCollection(object): If images in the `ImageCollection` don't have identical shapes. """ return concatenate_images(self) + + +def imread_collection_wrapper(imread): + def imread_collection(load_pattern, conserve_memory=True): + """Return an `ImageCollection` from files matching the given pattern. + + Note that files are always stored in alphabetical order. Also note that + slicing returns a new ImageCollection, *not* a view into the data. + + See `skimage.io.ImageCollection` for details. + + Parameters + ---------- + load_pattern : str or list + Pattern glob or filenames to load. The path can be absolute or + relative. Multiple patterns should be separated by a colon, + e.g. '/tmp/work/*.png:/tmp/other/*.jpg'. Also see + implementation notes below. + conserve_memory : bool, optional + If True, never keep more than one in memory at a specific + time. Otherwise, images will be cached once they are loaded. + + """ + return ImageCollection(load_pattern, conserve_memory=conserve_memory, + load_func=imread) + return imread_collection diff --git a/skimage/io/manage_plugins.py b/skimage/io/manage_plugins.py index 7ce74b1d..f80b38c1 100644 --- a/skimage/io/manage_plugins.py +++ b/skimage/io/manage_plugins.py @@ -24,7 +24,7 @@ except ImportError: import os.path from glob import glob -from .collection import ImageCollection +from .collection import imread_collection_wrapper __all__ = ['use_plugin', 'call_plugin', 'plugin_info', 'plugin_order', @@ -266,32 +266,6 @@ def use_plugin(name, kind=None): plugin_store[k] = funcs -def _imread_collection_wrapper(imread): - def imread_collection(load_pattern, conserve_memory=True): - """Return an `ImageCollection` from files matching the given pattern. - - Note that files are always stored in alphabetical order. Also note that - slicing returns a new ImageCollection, *not* a view into the data. - - See `skimage.io.ImageCollection` for details. - - Parameters - ---------- - load_pattern : str or list - Pattern glob or filenames to load. The path can be absolute or - relative. Multiple patterns should be separated by a colon, - e.g. '/tmp/work/*.png:/tmp/other/*.jpg'. Also see - implementation notes below. - conserve_memory : bool, optional - If True, never keep more than one in memory at a specific - time. Otherwise, images will be cached once they are loaded. - - """ - return ImageCollection(load_pattern, conserve_memory=conserve_memory, - load_func=imread) - return imread_collection - - def _load(plugin): """Load the given plugin. @@ -321,7 +295,7 @@ def _load(plugin): hasattr(plugin_module, 'imread')) if add_plugin: imread = getattr(plugin_module, 'imread') - func = _imread_collection_wrapper(imread) + func = imread_collection_wrapper(imread) setattr(plugin_module, 'imread_collection', func) elif not hasattr(plugin_module, p): print("Plugin %s does not provide %s as advertised. Ignoring." % \ From 07f2656975c660be30a92e09aa17fa5def0e770a Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Sun, 8 Dec 2013 10:14:00 -0600 Subject: [PATCH 4/6] Factor out function injection --- skimage/io/collection.py | 10 ++++++++++ skimage/io/manage_plugins.py | 15 +++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/skimage/io/collection.py b/skimage/io/collection.py index 96d29d4a..28a9632d 100644 --- a/skimage/io/collection.py +++ b/skimage/io/collection.py @@ -457,3 +457,13 @@ def imread_collection_wrapper(imread): return ImageCollection(load_pattern, conserve_memory=conserve_memory, load_func=imread) return imread_collection + + +def inject_imread_collection_if_needed(module): + """Add `imread_collection` to module if not already present.""" + add_plugin = (not hasattr(module, 'imread_collection') and + hasattr(module, 'imread')) + if add_plugin: + imread = getattr(module, 'imread') + func = imread_collection_wrapper(imread) + setattr(module, 'imread_collection', func) diff --git a/skimage/io/manage_plugins.py b/skimage/io/manage_plugins.py index f80b38c1..d31e6dbe 100644 --- a/skimage/io/manage_plugins.py +++ b/skimage/io/manage_plugins.py @@ -266,6 +266,14 @@ def use_plugin(name, kind=None): plugin_store[k] = funcs +def _inject_imread_collection_if_needed(module): + """Add `imread_collection` to module if not already present.""" + if not hasattr(module, 'imread_collection') and hasattr(module, 'imread'): + imread = getattr(module, 'imread') + func = imread_collection_wrapper(imread) + setattr(module, 'imread_collection', func) + + def _load(plugin): """Load the given plugin. @@ -291,12 +299,7 @@ def _load(plugin): provides = plugin_provides[plugin] for p in provides: if p == 'imread_collection': - add_plugin = (not hasattr(plugin_module, 'imread_collection') and - hasattr(plugin_module, 'imread')) - if add_plugin: - imread = getattr(plugin_module, 'imread') - func = imread_collection_wrapper(imread) - setattr(plugin_module, 'imread_collection', func) + _inject_imread_collection_if_needed(plugin_module) elif not hasattr(plugin_module, p): print("Plugin %s does not provide %s as advertised. Ignoring." % \ (plugin, p)) From 25e349d416e720e7c1750c71f3bbe21fa542aa46 Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Tue, 14 Jan 2014 23:15:45 -0600 Subject: [PATCH 5/6] Remove outdated function --- skimage/io/collection.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/skimage/io/collection.py b/skimage/io/collection.py index 28a9632d..96d29d4a 100644 --- a/skimage/io/collection.py +++ b/skimage/io/collection.py @@ -457,13 +457,3 @@ def imread_collection_wrapper(imread): return ImageCollection(load_pattern, conserve_memory=conserve_memory, load_func=imread) return imread_collection - - -def inject_imread_collection_if_needed(module): - """Add `imread_collection` to module if not already present.""" - add_plugin = (not hasattr(module, 'imread_collection') and - hasattr(module, 'imread')) - if add_plugin: - imread = getattr(module, 'imread') - func = imread_collection_wrapper(imread) - setattr(module, 'imread_collection', func) From e0dafcf004f07f7aa333d16252f86b2fc5f70d0b Mon Sep 17 00:00:00 2001 From: Tony S Yu Date: Tue, 14 Jan 2014 23:27:23 -0600 Subject: [PATCH 6/6] Clean up test --- skimage/io/tests/test_null.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/skimage/io/tests/test_null.py b/skimage/io/tests/test_null.py index 8895f77c..56f5df89 100644 --- a/skimage/io/tests/test_null.py +++ b/skimage/io/tests/test_null.py @@ -1,5 +1,6 @@ import os import warnings +from contextlib import contextmanager import numpy as np from numpy.testing import raises @@ -8,35 +9,38 @@ from skimage import io from skimage import data_dir +@contextmanager +def warnings_as_errors(): + # Temporarily set warnings as errors so we can test the warning is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('error') + yield + @raises(Warning) def test_null_imread(): path = os.path.join(data_dir, 'color.png') - with warnings.catch_warnings(): # Temporarily set warnings as errors. - warnings.filterwarnings('error') + with warnings_as_errors(): io.imread(path, plugin='null') @raises(Warning) def test_null_imsave(): - with warnings.catch_warnings(): # Temporarily set warnings as errors. - warnings.filterwarnings('error') + with warnings_as_errors(): io.imsave('dummy.png', np.zeros((3, 3)), plugin='null') @raises(Warning) def test_null_imshow(): - with warnings.catch_warnings(): # Temporarily set warnings as errors. - warnings.filterwarnings('error') + with warnings_as_errors(): io.imshow(np.zeros((3, 3)), plugin='null') @raises(Warning) def test_null_imread_collection(): # Note that the null plugin doesn't define an `imread_collection` plugin - # but this function dynamically added by the plugin manager. + # but this function is dynamically added by the plugin manager. path = os.path.join(data_dir, '*.png') - with warnings.catch_warnings(): # Temporarily set warnings as errors. - warnings.filterwarnings('error') + with warnings_as_errors(): collection = io.imread_collection(path, plugin='null') collection[0]