From 79d698ee112eb305e0d8843d128a4489f62246ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Boulogne?= Date: Fri, 27 Jun 2014 21:40:37 -0400 Subject: [PATCH 1/4] add user guide parallelization --- doc/source/user_guide/parallelization.txt | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 doc/source/user_guide/parallelization.txt diff --git a/doc/source/user_guide/parallelization.txt b/doc/source/user_guide/parallelization.txt new file mode 100755 index 00000000..d4ffe59b --- /dev/null +++ b/doc/source/user_guide/parallelization.txt @@ -0,0 +1,60 @@ +========================= +How to parallelize loops? +========================= + +In image processing, we frequently apply the same algorithm +on a large batch of images. Let us define an example. + +.. code-block:: python + + from skimage import data, color + from skimage.restoration import denoise_tv_chambolle + from skimage.feature import hog + + def tasks(image): + """ + Apply some functions. + """ + image = denoise_tv_chambolle(image, weight=0.1, multichannel=True) + fd, hog_image = hog(color.rgb2gray(image), orientations=8, + pixels_per_cell=(16, 16), cells_per_block=(1, 1), + visualise=True) + + + # Prepare images + hubble = data.hubble_deep_field() + width = 10 + pics = [hubble[:,slice:slice+width] for slice in range(0, 1000, width)] + +To call the function ``tasks`` on each element of the list ``pics``, it is +usual to write a for loop. To measure the execution time of this loop, a function +is defined and called with ``timeit``. + +.. code-block:: python + + def classic_loop(): + for image in pics: + tasks(image) + + import timeit + print("classic_loop():", timeit.timeit("classic_loop()", setup="from __main__ import (classic_loop, tasks, pics)", number=1)) + +Another equivalent way to code this loop is to use a comprehension list which has the same efficiency. + +.. code-block:: python + + def comprehension_loop(): + [tasks(image) for image in pics] + + print("comprehension_loop():", timeit.timeit("comprehension_loop()", setup="from __main__ import (comprehension_loop, tasks, pics)", number=1)) + +``joblib`` is a library providing an easy way to parallelize for loops once we have a comprehension list. +The number of jobs can be specified. + +.. code-block:: python + + from joblib import Parallel, delayed + def joblib_loop(): + Parallel(n_jobs=4)(delayed(tasks)(i) for i in pics) + + print("joblib_loop():", timeit.timeit("joblib_loop()", setup="from __main__ import (joblib_loop, tasks, pics)", number=1)) From 26e0348f3cf76d68f9afc043c7928bc25152a0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Boulogne?= Date: Mon, 25 Aug 2014 08:55:53 -0400 Subject: [PATCH 2/4] Comments of the PR --- doc/source/user_guide/parallelization.txt | 25 +++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/doc/source/user_guide/parallelization.txt b/doc/source/user_guide/parallelization.txt index d4ffe59b..20517de5 100755 --- a/doc/source/user_guide/parallelization.txt +++ b/doc/source/user_guide/parallelization.txt @@ -11,14 +11,15 @@ on a large batch of images. Let us define an example. from skimage.restoration import denoise_tv_chambolle from skimage.feature import hog - def tasks(image): + def task(image): """ - Apply some functions. + Apply some functions and return an image. """ image = denoise_tv_chambolle(image, weight=0.1, multichannel=True) fd, hog_image = hog(color.rgb2gray(image), orientations=8, pixels_per_cell=(16, 16), cells_per_block=(1, 1), visualise=True) + return hog_image # Prepare images @@ -26,7 +27,7 @@ on a large batch of images. Let us define an example. width = 10 pics = [hubble[:,slice:slice+width] for slice in range(0, 1000, width)] -To call the function ``tasks`` on each element of the list ``pics``, it is +To call the function ``task`` on each element of the list ``pics``, it is usual to write a for loop. To measure the execution time of this loop, a function is defined and called with ``timeit``. @@ -34,19 +35,25 @@ is defined and called with ``timeit``. def classic_loop(): for image in pics: - tasks(image) + task(image) import timeit - print("classic_loop():", timeit.timeit("classic_loop()", setup="from __main__ import (classic_loop, tasks, pics)", number=1)) + print("classic_loop():", timeit.timeit("classic_loop()", setup="from __main__ import (classic_loop, task, pics)", number=1)) + +Alternatively, you can use ipython and measure the execution time with ``%timeit``. + +.. code-block:: python + + %timeit classic_loop() Another equivalent way to code this loop is to use a comprehension list which has the same efficiency. .. code-block:: python def comprehension_loop(): - [tasks(image) for image in pics] + [task(image) for image in pics] - print("comprehension_loop():", timeit.timeit("comprehension_loop()", setup="from __main__ import (comprehension_loop, tasks, pics)", number=1)) + %timeit comprehension_loop() ``joblib`` is a library providing an easy way to parallelize for loops once we have a comprehension list. The number of jobs can be specified. @@ -55,6 +62,6 @@ The number of jobs can be specified. from joblib import Parallel, delayed def joblib_loop(): - Parallel(n_jobs=4)(delayed(tasks)(i) for i in pics) + Parallel(n_jobs=4)(delayed(task)(i) for i in pics) - print("joblib_loop():", timeit.timeit("joblib_loop()", setup="from __main__ import (joblib_loop, tasks, pics)", number=1)) + %timeit joblib_loop() From ffe7156f317bd3bdfa2d3395e21d8fc1a52abd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Boulogne?= Date: Tue, 2 Sep 2014 09:44:33 -0400 Subject: [PATCH 3/4] Stephan's comments --- doc/source/user_guide/parallelization.txt | 28 +++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/doc/source/user_guide/parallelization.txt b/doc/source/user_guide/parallelization.txt index 20517de5..f0323c7d 100755 --- a/doc/source/user_guide/parallelization.txt +++ b/doc/source/user_guide/parallelization.txt @@ -1,13 +1,13 @@ -========================= -How to parallelize loops? -========================= +======================== +How to parallelize loops +======================== In image processing, we frequently apply the same algorithm -on a large batch of images. Let us define an example. +on a large batch of images. Here is an example: .. code-block:: python - from skimage import data, color + from skimage import data, color, util from skimage.restoration import denoise_tv_chambolle from skimage.feature import hog @@ -25,24 +25,18 @@ on a large batch of images. Let us define an example. # Prepare images hubble = data.hubble_deep_field() width = 10 - pics = [hubble[:,slice:slice+width] for slice in range(0, 1000, width)] + pics = util.view_as_windows(hubble, (width, hubble.shape[1], hubble.shape[2]), step=width) To call the function ``task`` on each element of the list ``pics``, it is -usual to write a for loop. To measure the execution time of this loop, a function -is defined and called with ``timeit``. +usual to write a for loop. To measure the execution time of this loop, you can +use ipython and measure the execution time with ``%timeit``. .. code-block:: python def classic_loop(): for image in pics: - task(image) + task(image[0][0]) - import timeit - print("classic_loop():", timeit.timeit("classic_loop()", setup="from __main__ import (classic_loop, task, pics)", number=1)) - -Alternatively, you can use ipython and measure the execution time with ``%timeit``. - -.. code-block:: python %timeit classic_loop() @@ -51,7 +45,7 @@ Another equivalent way to code this loop is to use a comprehension list which ha .. code-block:: python def comprehension_loop(): - [task(image) for image in pics] + [task(image[0][0]) for image in pics] %timeit comprehension_loop() @@ -62,6 +56,6 @@ The number of jobs can be specified. from joblib import Parallel, delayed def joblib_loop(): - Parallel(n_jobs=4)(delayed(task)(i) for i in pics) + Parallel(n_jobs=4)(delayed(task)(i[0][0]) for i in pics) %timeit joblib_loop() From 5920d6788d560e6d1bd216fbbd08f13eca670615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Boulogne?= Date: Thu, 4 Sep 2014 18:00:56 -0400 Subject: [PATCH 4/4] simplify notations --- doc/source/user_guide/parallelization.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/user_guide/parallelization.txt b/doc/source/user_guide/parallelization.txt index f0323c7d..e083cbd2 100755 --- a/doc/source/user_guide/parallelization.txt +++ b/doc/source/user_guide/parallelization.txt @@ -15,7 +15,7 @@ on a large batch of images. Here is an example: """ Apply some functions and return an image. """ - image = denoise_tv_chambolle(image, weight=0.1, multichannel=True) + image = denoise_tv_chambolle(image[0][0], weight=0.1, multichannel=True) fd, hog_image = hog(color.rgb2gray(image), orientations=8, pixels_per_cell=(16, 16), cells_per_block=(1, 1), visualise=True) @@ -35,7 +35,7 @@ use ipython and measure the execution time with ``%timeit``. def classic_loop(): for image in pics: - task(image[0][0]) + task(image) %timeit classic_loop() @@ -45,7 +45,7 @@ Another equivalent way to code this loop is to use a comprehension list which ha .. code-block:: python def comprehension_loop(): - [task(image[0][0]) for image in pics] + [task(image) for image in pics] %timeit comprehension_loop() @@ -56,6 +56,6 @@ The number of jobs can be specified. from joblib import Parallel, delayed def joblib_loop(): - Parallel(n_jobs=4)(delayed(task)(i[0][0]) for i in pics) + Parallel(n_jobs=4)(delayed(task)(i) for i in pics) %timeit joblib_loop()