From 42e62e7a0b6fc1d39a15c4331b7531e395d72645 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 24 Apr 2015 14:59:34 -0500 Subject: [PATCH 01/20] Add process_chunks function --- skimage/util/process.py | 71 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 skimage/util/process.py diff --git a/skimage/util/process.py b/skimage/util/process.py new file mode 100644 index 00000000..e0463fec --- /dev/null +++ b/skimage/util/process.py @@ -0,0 +1,71 @@ +from multiprocessing import cpu_count + +import dask.array as da + +__all__ = ['process_chunks'] + + +def _get_chunks(shape, ncpu): + """ + Split the array into equal sized chunks based on the number of + available processors. The last chunk in each dimension absorbs the + remainder array elements if the number of cpus does not divide evenly into + the number of array elements. + """ + chunks = [] + + for i in shape: + regular_chunk = i // ncpu + remainder_chunk = regular_chunk + (i % ncpu) + + if regular_chunk == 0: + chunk_lens = (remainder_chunk,) + else: + chunk_lens = (regular_chunk,) * (ncpu - 1) + (remainder_chunk,) + + chunks.append(chunk_lens) + return tuple(chunks) + + +def process_chunks(function, array, args=(), kwargs={} chunks=None, depth=0, mode=None): + """Map a function in parallel across an array. + + Split an array into possibly overlapping chunks of a given depth and + boundary type, call the given function in parallel on the chunks, combine + the chunks and return the resulting array. + + Parameters + ---------- + function : function + Function to be mapped which takes an array as an argument. + array : numpy array + array which the function will be applied to. + chunks : int, tuple, or tuple of tuples + One tuple of length array.ndim or a list of tuples of length ndim. + Where each subtuple adds to the size of the array in the corresponding + dimension. If None, the array is broken up into chunks based on the + number of available cpus. + depth : int + integer equal to the depth of the internal external padding + mode : 'reflect', 'periodic', 'wrap', 'nearest' + type of external boundary padding + + Notes + ----- + Be careful choosing the depth so that it is never larger than the length of + a chunk. + + """ + if chunks == None: + shape = array.shape + ncpu = cpu_count() + chunks = _get_chunks(shape, ncpu) + + if mode == 'wrap': + mode = 'periodic' + + def wrapped_func(arr): + return function(arr, *args, **kwargs) + + darr = da.from_array(array, chunks=chunks) + return darr.map_overlap(wrapped_func, depth, boundary=mode).compute() From 43bd8242ff6df1502d4d2e246023796e859c8d40 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 24 Apr 2015 17:37:41 -0500 Subject: [PATCH 02/20] load process chunks into namespace --- skimage/util/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skimage/util/__init__.py b/skimage/util/__init__.py index 4739bd5b..06895e64 100644 --- a/skimage/util/__init__.py +++ b/skimage/util/__init__.py @@ -2,6 +2,7 @@ from .dtype import (img_as_float, img_as_int, img_as_uint, img_as_ubyte, img_as_bool, dtype_limits) from .shape import view_as_blocks, view_as_windows from .noise import random_noise +from .process import process_chunks from .arraypad import pad, crop from ._regular_grid import regular_grid @@ -20,4 +21,5 @@ __all__ = ['img_as_float', 'crop', 'random_noise', 'regular_grid', + 'process_chunks', 'unique_rows'] From 72091a88188959a1cd40d37e9708f9986886569c Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 24 Apr 2015 17:38:24 -0500 Subject: [PATCH 03/20] Tests for process chunks function. --- skimage/util/tests/test_process.py | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 skimage/util/tests/test_process.py diff --git a/skimage/util/tests/test_process.py b/skimage/util/tests/test_process.py new file mode 100644 index 00000000..3710cece --- /dev/null +++ b/skimage/util/tests/test_process.py @@ -0,0 +1,62 @@ +import numpy as np +from numpy.testing import assert_array_almost_equal + +from skimage.filters import threshold_adaptive, gaussian_filter +from skimage.util import process_chunks + +def test_process_chunks(): + # data + a = np.arange(144).reshape(12, 12).astype(float) + + # wrapp the function we're applying + def wrapped_thresh(arr): + return threshold_adaptive(arr, 3, mode='reflect') + + # apply the filter + expected1 = threshold_adaptive(a, 3) + result1 = process_chunks(wrapped_thresh, a, chunks=(6, 6), + depth=5) + + assert_array_almost_equal(result1, expected1) + + def wrapped_gauss(arr): + return gaussian_filter(arr, 1, mode='reflect') + + expected2 = gaussian_filter(a, 1, mode='reflect') + result2 = process_chunks(wrapped_gauss, a, chunks=(6, 6), + depth=5) + + assert_array_almost_equal(result2, expected2) + + +def test_no_chunks(): + a = np.ones(1 * 4 * 8 * 9).reshape(1, 4, 8, 9) + def add_42(arr): + return arr + 42 + + expected = add_42(a) + result = process_chunks(add_42, a) + + assert_array_almost_equal(result, expected) + + +def test_process_chunks_wrap(): + def wrapped(arr): + return gaussian_filter(arr, 1, mode='wrap') + a = np.arange(144).reshape(12, 12).astype(float) + expected = gaussian_filter(a, 1, mode='wrap') + result = process_chunks(wrapped, a, chunks=(6, 6), + depth=5, mode='wrap') + + assert_array_almost_equal(result, expected) + + +def test_process_chunks_nearest(): + def wrapped(arr): + return gaussian_filter(arr, 1, mode='nearest') + a = np.arange(144).reshape(12, 12).astype(float) + expected = gaussian_filter(a, 1, mode='nearest') + result = process_chunks(wrapped, a, chunks=(6, 6), + depth={0: 5, 1: 5}, mode='nearest') + + assert_array_almost_equal(result, expected) From bf3bb8d0450f6995dc4b2ee312e23864e31edb26 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 5 May 2015 11:07:46 -0500 Subject: [PATCH 04/20] Add latest dask to .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 14dd7c30..ae56bcda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ before_install: - tools/build_versions.py install: + - pip install git+https://github.com/ContinuumIO/dask --upgrade - python setup.py build_ext --inplace - python setup.py install From 54e95cff9b768278bd85da269f6d266308d7f07c Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 5 May 2015 11:09:37 -0500 Subject: [PATCH 05/20] Add latest dask to appveyor --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 0bd10075..2d1f4761 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,6 +39,7 @@ install: # Install the build and runtime dependencies of the project. - "%CMD_IN_ENV% pip install -v %WHEELHOUSE% -r tools/appveyor/requirements.txt" - "%CMD_IN_ENV% pip install -v -r requirements.txt" + - "%CMD_IN_ENV% pip install git+https://github.com/ContinuumIO/dask --upgrade" - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst" - ps: "ls dist" From 4a47127871d9e408c9c1b4796604317fe3ac5cc8 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 5 May 2015 17:55:21 -0500 Subject: [PATCH 06/20] pep8 --- skimage/util/process.py | 5 +++-- skimage/util/tests/test_process.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/skimage/util/process.py b/skimage/util/process.py index e0463fec..c03b1c5b 100644 --- a/skimage/util/process.py +++ b/skimage/util/process.py @@ -27,7 +27,8 @@ def _get_chunks(shape, ncpu): return tuple(chunks) -def process_chunks(function, array, args=(), kwargs={} chunks=None, depth=0, mode=None): +def process_chunks(function, array, args=(), kwargs={}, chunks=None, depth=0, + mode=None): """Map a function in parallel across an array. Split an array into possibly overlapping chunks of a given depth and @@ -56,7 +57,7 @@ def process_chunks(function, array, args=(), kwargs={} chunks=None, depth=0, mod a chunk. """ - if chunks == None: + if chunks is None: shape = array.shape ncpu = cpu_count() chunks = _get_chunks(shape, ncpu) diff --git a/skimage/util/tests/test_process.py b/skimage/util/tests/test_process.py index 3710cece..52fe248c 100644 --- a/skimage/util/tests/test_process.py +++ b/skimage/util/tests/test_process.py @@ -4,6 +4,7 @@ from numpy.testing import assert_array_almost_equal from skimage.filters import threshold_adaptive, gaussian_filter from skimage.util import process_chunks + def test_process_chunks(): # data a = np.arange(144).reshape(12, 12).astype(float) @@ -31,6 +32,7 @@ def test_process_chunks(): def test_no_chunks(): a = np.ones(1 * 4 * 8 * 9).reshape(1, 4, 8, 9) + def add_42(arr): return arr + 42 From 6be2b1bfa265932b0d6dda5035097ab5dc1ccd53 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 6 May 2015 15:35:36 -0500 Subject: [PATCH 07/20] Smarter chunk selection. --- skimage/util/process.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/skimage/util/process.py b/skimage/util/process.py index c03b1c5b..177651ff 100644 --- a/skimage/util/process.py +++ b/skimage/util/process.py @@ -1,3 +1,4 @@ +from math import ceil from multiprocessing import cpu_count import dask.array as da @@ -11,19 +12,35 @@ def _get_chunks(shape, ncpu): available processors. The last chunk in each dimension absorbs the remainder array elements if the number of cpus does not divide evenly into the number of array elements. + + >>> _get_chunks((4, 4), 4) + ((2, 2), (2, 2)) + >>> _get_chunks((4, 4), 2) + ((2, 2), (4,)) + >>> _get_chunks((5, 5), 2) + ((2, 3), (5,)) + >>> _get_chunks((2, 4), 2) + ((1, 1), (4,)) """ chunks = [] + nchunks_per_dim = int(ceil(ncpu ** (1./len(shape)))) + used_chunks = 1 for i in shape: - regular_chunk = i // ncpu - remainder_chunk = regular_chunk + (i % ncpu) + if used_chunks < ncpu: + regular_chunk = i // nchunks_per_dim + remainder_chunk = regular_chunk + (i % nchunks_per_dim) - if regular_chunk == 0: - chunk_lens = (remainder_chunk,) + if regular_chunk == 0: + chunk_lens = (remainder_chunk,) + else: + chunk_lens = ((regular_chunk,) * (nchunks_per_dim - 1) + + (remainder_chunk,)) else: - chunk_lens = (regular_chunk,) * (ncpu - 1) + (remainder_chunk,) + chunk_lens = (i,) chunks.append(chunk_lens) + used_chunks *= nchunks_per_dim return tuple(chunks) From 08dcf4a4e6728d68c5f2284459a94e9a9a1f8bea Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 7 May 2015 15:51:26 -0500 Subject: [PATCH 08/20] Move the function's args and kws to the end of the signature renamed them to extra_arguments and extra_keywords suggested by @jni --- skimage/util/process.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/skimage/util/process.py b/skimage/util/process.py index 177651ff..3a97343c 100644 --- a/skimage/util/process.py +++ b/skimage/util/process.py @@ -44,8 +44,8 @@ def _get_chunks(shape, ncpu): return tuple(chunks) -def process_chunks(function, array, args=(), kwargs={}, chunks=None, depth=0, - mode=None): +def process_chunks(function, array, chunks=None, depth=0, + mode=None, extra_arguments=(), extra_keywords={}): """Map a function in parallel across an array. Split an array into possibly overlapping chunks of a given depth and @@ -67,6 +67,10 @@ def process_chunks(function, array, args=(), kwargs={}, chunks=None, depth=0, integer equal to the depth of the internal external padding mode : 'reflect', 'periodic', 'wrap', 'nearest' type of external boundary padding + extra_arguments : tuple + Tuple of arguments to be passed to the function. + extra_keywords : dictionary + Dictionary of keyword arguments to be passed to the function. Notes ----- @@ -83,7 +87,7 @@ def process_chunks(function, array, args=(), kwargs={}, chunks=None, depth=0, mode = 'periodic' def wrapped_func(arr): - return function(arr, *args, **kwargs) + return function(arr, *extra_arguments, **extra_keywords) darr = da.from_array(array, chunks=chunks) return darr.map_overlap(wrapped_func, depth, boundary=mode).compute() From ae73d03f5f0d4b62e0d5fa4d89b67004ecd3b39c Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 11 May 2015 17:22:49 -0500 Subject: [PATCH 09/20] process -> apply, more docs --- skimage/util/__init__.py | 4 +- skimage/util/{process.py => apply.py} | 38 +++++++++---------- .../tests/{test_process.py => test_apply.py} | 29 ++++++-------- 3 files changed, 33 insertions(+), 38 deletions(-) rename skimage/util/{process.py => apply.py} (66%) rename skimage/util/tests/{test_process.py => test_apply.py} (61%) diff --git a/skimage/util/__init__.py b/skimage/util/__init__.py index 06895e64..40f17955 100644 --- a/skimage/util/__init__.py +++ b/skimage/util/__init__.py @@ -2,7 +2,7 @@ from .dtype import (img_as_float, img_as_int, img_as_uint, img_as_ubyte, img_as_bool, dtype_limits) from .shape import view_as_blocks, view_as_windows from .noise import random_noise -from .process import process_chunks +from .apply import apply_chunks from .arraypad import pad, crop from ._regular_grid import regular_grid @@ -21,5 +21,5 @@ __all__ = ['img_as_float', 'crop', 'random_noise', 'regular_grid', - 'process_chunks', + 'apply_chunks', 'unique_rows'] diff --git a/skimage/util/process.py b/skimage/util/apply.py similarity index 66% rename from skimage/util/process.py rename to skimage/util/apply.py index 3a97343c..495be032 100644 --- a/skimage/util/process.py +++ b/skimage/util/apply.py @@ -3,7 +3,7 @@ from multiprocessing import cpu_count import dask.array as da -__all__ = ['process_chunks'] +__all__ = ['apply_chunks'] def _get_chunks(shape, ncpu): @@ -44,8 +44,8 @@ def _get_chunks(shape, ncpu): return tuple(chunks) -def process_chunks(function, array, chunks=None, depth=0, - mode=None, extra_arguments=(), extra_keywords={}): +def apply_chunks(function, array, chunks=None, depth=0, mode=None, + extra_arguments=(), extra_keywords={}): """Map a function in parallel across an array. Split an array into possibly overlapping chunks of a given depth and @@ -57,26 +57,26 @@ def process_chunks(function, array, chunks=None, depth=0, function : function Function to be mapped which takes an array as an argument. array : numpy array - array which the function will be applied to. - chunks : int, tuple, or tuple of tuples - One tuple of length array.ndim or a list of tuples of length ndim. - Where each subtuple adds to the size of the array in the corresponding - dimension. If None, the array is broken up into chunks based on the - number of available cpus. - depth : int - integer equal to the depth of the internal external padding - mode : 'reflect', 'periodic', 'wrap', 'nearest' + Array which the function will be applied to. + chunks : int, tuple, or tuple of tuples, optional + A single integer is interpreted as the length of one side of a square + chunk that should be tiled across the array. One tuple of length + ``array.ndim`` represents the shape of a chunk, and it is tiled across + the array. A list of tuples of length ``ndim``, where each sub-tuple + is a sequence of chunk sizes along the corresponding dimension. If + None, the array is broken up into chunks based on the number of + available cpus. More information about chunks is in the documentation + `here `_. + depth : int, optional + Integer equal to the depth of the added boundary cells. Defaults to + zero. + mode : 'reflect', 'periodic', 'wrap', 'nearest', optional type of external boundary padding - extra_arguments : tuple + extra_arguments : tuple, optional Tuple of arguments to be passed to the function. - extra_keywords : dictionary + extra_keywords : dictionary, optional Dictionary of keyword arguments to be passed to the function. - Notes - ----- - Be careful choosing the depth so that it is never larger than the length of - a chunk. - """ if chunks is None: shape = array.shape diff --git a/skimage/util/tests/test_process.py b/skimage/util/tests/test_apply.py similarity index 61% rename from skimage/util/tests/test_process.py rename to skimage/util/tests/test_apply.py index 52fe248c..561f3c10 100644 --- a/skimage/util/tests/test_process.py +++ b/skimage/util/tests/test_apply.py @@ -2,21 +2,18 @@ import numpy as np from numpy.testing import assert_array_almost_equal from skimage.filters import threshold_adaptive, gaussian_filter -from skimage.util import process_chunks +from skimage.util import apply_chunks -def test_process_chunks(): +def test_apply_chunks(): # data a = np.arange(144).reshape(12, 12).astype(float) - # wrapp the function we're applying - def wrapped_thresh(arr): - return threshold_adaptive(arr, 3, mode='reflect') - # apply the filter expected1 = threshold_adaptive(a, 3) - result1 = process_chunks(wrapped_thresh, a, chunks=(6, 6), - depth=5) + result1 = apply_chunks(threshold_adaptive, a, chunks=(6, 6), depth=5, + extra_arguments=(3,), + extra_keywords={'mode': 'reflect'}) assert_array_almost_equal(result1, expected1) @@ -24,8 +21,7 @@ def test_process_chunks(): return gaussian_filter(arr, 1, mode='reflect') expected2 = gaussian_filter(a, 1, mode='reflect') - result2 = process_chunks(wrapped_gauss, a, chunks=(6, 6), - depth=5) + result2 = apply_chunks(wrapped_gauss, a, chunks=(6, 6), depth=5) assert_array_almost_equal(result2, expected2) @@ -37,28 +33,27 @@ def test_no_chunks(): return arr + 42 expected = add_42(a) - result = process_chunks(add_42, a) + result = apply_chunks(add_42, a) assert_array_almost_equal(result, expected) -def test_process_chunks_wrap(): +def test_apply_chunks_wrap(): def wrapped(arr): return gaussian_filter(arr, 1, mode='wrap') a = np.arange(144).reshape(12, 12).astype(float) expected = gaussian_filter(a, 1, mode='wrap') - result = process_chunks(wrapped, a, chunks=(6, 6), - depth=5, mode='wrap') + result = apply_chunks(wrapped, a, chunks=(6, 6), depth=5, mode='wrap') assert_array_almost_equal(result, expected) -def test_process_chunks_nearest(): +def test_apply_chunks_nearest(): def wrapped(arr): return gaussian_filter(arr, 1, mode='nearest') a = np.arange(144).reshape(12, 12).astype(float) expected = gaussian_filter(a, 1, mode='nearest') - result = process_chunks(wrapped, a, chunks=(6, 6), - depth={0: 5, 1: 5}, mode='nearest') + result = apply_chunks(wrapped, a, chunks=(6, 6), depth={0: 5, 1: 5}, + mode='nearest') assert_array_almost_equal(result, expected) From b2e4468e554ceec219f34b84793b1f8c6c33b857 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 12 May 2015 11:29:47 -0500 Subject: [PATCH 10/20] Use my dask branch instead of master --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae56bcda..56909d28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_install: - tools/build_versions.py install: - - pip install git+https://github.com/ContinuumIO/dask --upgrade + - pip install git+https://github.com/cowlicks/dask@skimage --upgrade - python setup.py build_ext --inplace - python setup.py install diff --git a/appveyor.yml b/appveyor.yml index 2d1f4761..4efeca11 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,7 +39,7 @@ install: # Install the build and runtime dependencies of the project. - "%CMD_IN_ENV% pip install -v %WHEELHOUSE% -r tools/appveyor/requirements.txt" - "%CMD_IN_ENV% pip install -v -r requirements.txt" - - "%CMD_IN_ENV% pip install git+https://github.com/ContinuumIO/dask --upgrade" + - "%CMD_IN_ENV% pip install git+https://github.com/cowlicks/dask@skimage --upgrade" - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst" - ps: "ls dist" From 7ac7e1078b0e7b37906f7bbacabd7345a57f5e7a Mon Sep 17 00:00:00 2001 From: arve0 Date: Sat, 16 May 2015 12:07:11 +0200 Subject: [PATCH 11/20] cpu_count may raise NotImplementedError --- skimage/util/apply.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skimage/util/apply.py b/skimage/util/apply.py index 495be032..a63be1c1 100644 --- a/skimage/util/apply.py +++ b/skimage/util/apply.py @@ -80,7 +80,10 @@ def apply_chunks(function, array, chunks=None, depth=0, mode=None, """ if chunks is None: shape = array.shape - ncpu = cpu_count() + try: + ncpu = cpu_count() + except NotImplementedError: + ncpu = 4 chunks = _get_chunks(shape, ncpu) if mode == 'wrap': From a258fc7b3bea17223744a6f8b61f5af1e3334b19 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Sat, 16 May 2015 19:36:59 -0500 Subject: [PATCH 12/20] Rename apply and apply_chunks to apply_parallel --- skimage/util/__init__.py | 4 ++-- skimage/util/{apply.py => apply_parallel.py} | 4 ++-- .../{test_apply.py => test_apply_parallel.py} | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) rename skimage/util/{apply.py => apply_parallel.py} (97%) rename skimage/util/tests/{test_apply.py => test_apply_parallel.py} (73%) diff --git a/skimage/util/__init__.py b/skimage/util/__init__.py index 40f17955..2500469b 100644 --- a/skimage/util/__init__.py +++ b/skimage/util/__init__.py @@ -2,7 +2,7 @@ from .dtype import (img_as_float, img_as_int, img_as_uint, img_as_ubyte, img_as_bool, dtype_limits) from .shape import view_as_blocks, view_as_windows from .noise import random_noise -from .apply import apply_chunks +from .apply_parallel import apply_parallel from .arraypad import pad, crop from ._regular_grid import regular_grid @@ -21,5 +21,5 @@ __all__ = ['img_as_float', 'crop', 'random_noise', 'regular_grid', - 'apply_chunks', + 'apply_parallel', 'unique_rows'] diff --git a/skimage/util/apply.py b/skimage/util/apply_parallel.py similarity index 97% rename from skimage/util/apply.py rename to skimage/util/apply_parallel.py index a63be1c1..d9142b91 100644 --- a/skimage/util/apply.py +++ b/skimage/util/apply_parallel.py @@ -3,7 +3,7 @@ from multiprocessing import cpu_count import dask.array as da -__all__ = ['apply_chunks'] +__all__ = ['apply_parallel'] def _get_chunks(shape, ncpu): @@ -44,7 +44,7 @@ def _get_chunks(shape, ncpu): return tuple(chunks) -def apply_chunks(function, array, chunks=None, depth=0, mode=None, +def apply_parallel(function, array, chunks=None, depth=0, mode=None, extra_arguments=(), extra_keywords={}): """Map a function in parallel across an array. diff --git a/skimage/util/tests/test_apply.py b/skimage/util/tests/test_apply_parallel.py similarity index 73% rename from skimage/util/tests/test_apply.py rename to skimage/util/tests/test_apply_parallel.py index 561f3c10..c0d3f44d 100644 --- a/skimage/util/tests/test_apply.py +++ b/skimage/util/tests/test_apply_parallel.py @@ -2,16 +2,16 @@ import numpy as np from numpy.testing import assert_array_almost_equal from skimage.filters import threshold_adaptive, gaussian_filter -from skimage.util import apply_chunks +from skimage.util import apply_parallel -def test_apply_chunks(): +def test_apply_parallel(): # data a = np.arange(144).reshape(12, 12).astype(float) # apply the filter expected1 = threshold_adaptive(a, 3) - result1 = apply_chunks(threshold_adaptive, a, chunks=(6, 6), depth=5, + result1 = apply_parallel(threshold_adaptive, a, chunks=(6, 6), depth=5, extra_arguments=(3,), extra_keywords={'mode': 'reflect'}) @@ -21,7 +21,7 @@ def test_apply_chunks(): return gaussian_filter(arr, 1, mode='reflect') expected2 = gaussian_filter(a, 1, mode='reflect') - result2 = apply_chunks(wrapped_gauss, a, chunks=(6, 6), depth=5) + result2 = apply_parallel(wrapped_gauss, a, chunks=(6, 6), depth=5) assert_array_almost_equal(result2, expected2) @@ -33,27 +33,27 @@ def test_no_chunks(): return arr + 42 expected = add_42(a) - result = apply_chunks(add_42, a) + result = apply_parallel(add_42, a) assert_array_almost_equal(result, expected) -def test_apply_chunks_wrap(): +def test_apply_parallel_wrap(): def wrapped(arr): return gaussian_filter(arr, 1, mode='wrap') a = np.arange(144).reshape(12, 12).astype(float) expected = gaussian_filter(a, 1, mode='wrap') - result = apply_chunks(wrapped, a, chunks=(6, 6), depth=5, mode='wrap') + result = apply_parallel(wrapped, a, chunks=(6, 6), depth=5, mode='wrap') assert_array_almost_equal(result, expected) -def test_apply_chunks_nearest(): +def test_apply_parallel_nearest(): def wrapped(arr): return gaussian_filter(arr, 1, mode='nearest') a = np.arange(144).reshape(12, 12).astype(float) expected = gaussian_filter(a, 1, mode='nearest') - result = apply_chunks(wrapped, a, chunks=(6, 6), depth={0: 5, 1: 5}, + result = apply_parallel(wrapped, a, chunks=(6, 6), depth={0: 5, 1: 5}, mode='nearest') assert_array_almost_equal(result, expected) From 9be6424b4b5c7db8bbd4f1cb17bfdcf2eed9b180 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 18 May 2015 14:10:01 -0500 Subject: [PATCH 13/20] Install dask[array] from pypi for travis and appveyor --- .travis.yml | 1 - appveyor.yml | 1 - tools/appveyor/requirements.txt | 1 + tools/travis_script.sh | 2 ++ 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56909d28..14dd7c30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ before_install: - tools/build_versions.py install: - - pip install git+https://github.com/cowlicks/dask@skimage --upgrade - python setup.py build_ext --inplace - python setup.py install diff --git a/appveyor.yml b/appveyor.yml index 4efeca11..0bd10075 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,7 +39,6 @@ install: # Install the build and runtime dependencies of the project. - "%CMD_IN_ENV% pip install -v %WHEELHOUSE% -r tools/appveyor/requirements.txt" - "%CMD_IN_ENV% pip install -v -r requirements.txt" - - "%CMD_IN_ENV% pip install git+https://github.com/cowlicks/dask@skimage --upgrade" - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst" - ps: "ls dist" diff --git a/tools/appveyor/requirements.txt b/tools/appveyor/requirements.txt index 9cc4022f..58a4ecfa 100644 --- a/tools/appveyor/requirements.txt +++ b/tools/appveyor/requirements.txt @@ -13,5 +13,6 @@ scipy==0.14.0 cython==0.20.2 matplotlib==1.4.2 pillow==2.6.1 +dask[array]>=0.5.0 wheel nose diff --git a/tools/travis_script.sh b/tools/travis_script.sh index 8dc2bb4e..30777285 100755 --- a/tools/travis_script.sh +++ b/tools/travis_script.sh @@ -67,6 +67,8 @@ fi retry pip install -q tifffile +pip install 'dask[array]>=0.5.0' + section_end "Install.optional.dependencies" From 0e573f71af8de5a265f3a8ef06df6f8fd29b15e8 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 18 May 2015 14:27:55 -0500 Subject: [PATCH 14/20] Use absolute import --- skimage/util/tests/test_apply_parallel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/skimage/util/tests/test_apply_parallel.py b/skimage/util/tests/test_apply_parallel.py index c0d3f44d..a43f4338 100644 --- a/skimage/util/tests/test_apply_parallel.py +++ b/skimage/util/tests/test_apply_parallel.py @@ -1,8 +1,10 @@ +from __future__ import absolute_import + import numpy as np from numpy.testing import assert_array_almost_equal from skimage.filters import threshold_adaptive, gaussian_filter -from skimage.util import apply_parallel +from skimage.util.apply_parallel import apply_parallel def test_apply_parallel(): From c885a9ef729412d98c2be5daf9c1716c42bfcf97 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 19 May 2015 13:00:07 -0500 Subject: [PATCH 15/20] Add dask to DEPENDS.txt --- DEPENDS.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DEPENDS.txt b/DEPENDS.txt index c7f91df5..13e56598 100644 --- a/DEPENDS.txt +++ b/DEPENDS.txt @@ -52,6 +52,9 @@ functionality is only available with the following installed: * `imread `__ Optional io plugin providing most standard `formats `__. +* `dask array`__ + Package used for ``apply_parallel`` out-of-core and parallel computations. + Testing requirements -------------------- From 9d11b3854a2814a93876f3b5755faa59ddaa8e61 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 19 May 2015 13:11:13 -0500 Subject: [PATCH 16/20] Add to release notes. --- doc/release/release_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release/release_dev.txt b/doc/release/release_dev.txt index 1123eb77..039274f6 100644 --- a/doc/release/release_dev.txt +++ b/doc/release/release_dev.txt @@ -15,6 +15,7 @@ http://scikit-image.org New Features ------------ +- ``skimage.util.apply_parallel`` (#1493) From a8d49950aa7c259c092eca378e56f8d3811c154f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 19 May 2015 13:55:55 -0500 Subject: [PATCH 17/20] Defer import errors of dask array until runtime. --- skimage/util/apply_parallel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skimage/util/apply_parallel.py b/skimage/util/apply_parallel.py index d9142b91..fd50ba56 100644 --- a/skimage/util/apply_parallel.py +++ b/skimage/util/apply_parallel.py @@ -1,8 +1,6 @@ from math import ceil from multiprocessing import cpu_count -import dask.array as da - __all__ = ['apply_parallel'] @@ -78,6 +76,8 @@ def apply_parallel(function, array, chunks=None, depth=0, mode=None, Dictionary of keyword arguments to be passed to the function. """ + import dask.array as da + if chunks is None: shape = array.shape try: From d6c028a9b34d842726885de3e6cba93d3e384928 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 21 May 2015 12:12:10 -0500 Subject: [PATCH 18/20] Move dask intall to travis_befor_install.sh --- tools/travis_before_install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index f98e48de..55e55c03 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -54,6 +54,8 @@ fi retry pip install $WHEELHOUSE -r requirements.txt +pip install 'dask[array]>=0.5.0' + # clean up disk space sudo apt-get clean sudo rm -rf /tmp/* From 14a185810eccce94e874f6ead32f46918f99edf9 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 21 May 2015 12:12:38 -0500 Subject: [PATCH 19/20] Remove dask install from travis_script.sh --- tools/travis_script.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/travis_script.sh b/tools/travis_script.sh index 30777285..8dc2bb4e 100755 --- a/tools/travis_script.sh +++ b/tools/travis_script.sh @@ -67,8 +67,6 @@ fi retry pip install -q tifffile -pip install 'dask[array]>=0.5.0' - section_end "Install.optional.dependencies" From 2ebcadbb96de7f4314c92a11aaa6cc08576736fe Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 21 May 2015 14:15:07 -0500 Subject: [PATCH 20/20] Correct link in DEPENDS.txt --- DEPENDS.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/DEPENDS.txt b/DEPENDS.txt index 13e56598..e66d53cc 100644 --- a/DEPENDS.txt +++ b/DEPENDS.txt @@ -16,6 +16,7 @@ Runtime requirements * `NetworkX `__ * `Pillow `__ (or `PIL `__) +* `dask array `__ Known build errors ------------------ @@ -52,9 +53,6 @@ functionality is only available with the following installed: * `imread `__ Optional io plugin providing most standard `formats `__. -* `dask array`__ - Package used for ``apply_parallel`` out-of-core and parallel computations. - Testing requirements --------------------