From 9557c8dde1495709b25ddb89ec5602c9cbd4b4f1 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Tue, 11 Mar 2014 21:49:21 +0530 Subject: [PATCH 01/10] added a functionality to fetch images from URL in _novice.py --- skimage/novice/_novice.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index 27721f9b..0e3a9fd1 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -4,10 +4,17 @@ from collections import namedtuple import numpy as np from skimage import io +from io import BytesIO from skimage import img_as_ubyte from skimage.transform import resize from skimage.color import color_dict +import re + +from six.moves.urllib_parse import urlparse +from six.moves.urllib import request +urlopen = request.urlopen + import six # Convert colors from `skimage.color` to uint8 and allow access through @@ -19,7 +26,7 @@ colors = namedtuple('colors', color_dict.keys())(**color_dict) def open(path): """Return Picture object from the given image path.""" - return Picture(path=os.path.abspath(path)) + return Picture(path) def _verify_picture_index(index): @@ -210,9 +217,24 @@ class Picture(object): msg = "Must provide a single keyword arg (path, array, xy_array)." ValueError(msg) elif path is not None: - self.array = img_as_ubyte(io.imread(path)) self._path = path - self._format = imghdr.what(path) + regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain + r'localhost|' #localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + matchObj = re.match(regex, path) + if matchObj: + data = urlopen(path).read() + self.array = img_as_ubyte(io.imread(BytesIO(data))) + self._format = imghdr.what("", h=data) + else: + path = os.path.abspath(path) + self.array = img_as_ubyte(io.imread(path)) + self._path = path + self._format = imghdr.what(path) elif array is not None: self.array = array elif xy_array is not None: From 1fd34b47515b6c5d462df2c16d4060877a3f9c5f Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Tue, 11 Mar 2014 21:53:47 +0530 Subject: [PATCH 02/10] added description and exmaple for new functionality of fetching images of url --- skimage/novice/_novice.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index 0e3a9fd1..f3cce638 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -169,7 +169,7 @@ class Picture(object): Attributes ---------- path : str - Path to an image file to load. + Path to an image file to load / URL of an image array : array Raw RGB image data [0-255], with origin at top-left. xy_array : array @@ -181,6 +181,8 @@ class Picture(object): >>> from skimage import novice >>> from skimage import data >>> picture = novice.open(data.data_dir + '/chelsea.png') + >>> picture = novice.open('http://static3.businessinsider.com/image/ + 52a0bbfd6bb3f7961363819e/the-most-amazing-satellite-images-of-the-year.jpg') Create a blank 100 pixel wide, 200 pixel tall white image >>> pic = Picture.from_size((100, 200), color=(255, 255, 255)) From 654642c986e664e21bc2a71f406ff0f1217349d1 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Tue, 11 Mar 2014 22:35:07 +0530 Subject: [PATCH 03/10] changed test url in example for the Picture(object) --- skimage/novice/_novice.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index f3cce638..eec1941a 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -181,8 +181,7 @@ class Picture(object): >>> from skimage import novice >>> from skimage import data >>> picture = novice.open(data.data_dir + '/chelsea.png') - >>> picture = novice.open('http://static3.businessinsider.com/image/ - 52a0bbfd6bb3f7961363819e/the-most-amazing-satellite-images-of-the-year.jpg') + >>> picture = novice.open('http://scikit-image.org/_static/img/logo.png') Create a blank 100 pixel wide, 200 pixel tall white image >>> pic = Picture.from_size((100, 200), color=(255, 255, 255)) From 15893a3ed078b84fc42e3384cb920555e5c76dc4 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 00:29:19 +0530 Subject: [PATCH 04/10] changed the method of checking if a string is URL 1. removed regex code which was used earlier to check if a string is url. (regex code taken from django) 2. Now checking if a string starts with http(s) or ftp(s). --- skimage/novice/_novice.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index eec1941a..c1f0dce5 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -10,13 +10,12 @@ from skimage.transform import resize from skimage.color import color_dict import re - +import six from six.moves.urllib_parse import urlparse from six.moves.urllib import request +from six.moves.urllib import error urlopen = request.urlopen -import six - # Convert colors from `skimage.color` to uint8 and allow access through # dict or a named tuple. color_dict = dict((name, tuple(int(255 * c + 0.5) for c in rgb)) @@ -181,6 +180,8 @@ class Picture(object): >>> from skimage import novice >>> from skimage import data >>> picture = novice.open(data.data_dir + '/chelsea.png') + + Load an image from a URL. URL must start with http(s):// or ftp(s):// >>> picture = novice.open('http://scikit-image.org/_static/img/logo.png') Create a blank 100 pixel wide, 200 pixel tall white image @@ -218,19 +219,19 @@ class Picture(object): msg = "Must provide a single keyword arg (path, array, xy_array)." ValueError(msg) elif path is not None: - self._path = path - regex = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain - r'localhost|' #localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip - r'(?::\d+)?' # optional port - r'(?:/?|[/?]\S+)$', re.IGNORECASE) - matchObj = re.match(regex, path) - if matchObj: - data = urlopen(path).read() - self.array = img_as_ubyte(io.imread(BytesIO(data))) - self._format = imghdr.what("", h=data) + urlObj = urlparse(path) + if (urlObj.scheme == 'http') or (urlObj.scheme == 'https') or (urlObj.scheme == 'ftp') or (urlObj.scheme == 'ftps'): + try: + data = urlopen(path).read() + self.array = img_as_ubyte(io.imread(BytesIO(data))) + self._format = imghdr.what("", h=data) + self._path = path + except error.HTTPError, e: + print 'HTTP Error ', e.code + except error.URLError, e: + print 'URL Error\n', e.args + except error.ContentTooShortError: + print 'Content too short' else: path = os.path.abspath(path) self.array = img_as_ubyte(io.imread(path)) From 2f3bd464b8bcc0a75460e749c0ec3e8677c91aa5 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 08:00:53 +0530 Subject: [PATCH 05/10] added a dummy useragent in Picture class while fetching image using URL. --- skimage/novice/_novice.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index c1f0dce5..a5dbef51 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -11,6 +11,7 @@ from skimage.color import color_dict import re import six +import six.moves.urllib from six.moves.urllib_parse import urlparse from six.moves.urllib import request from six.moves.urllib import error @@ -222,7 +223,10 @@ class Picture(object): urlObj = urlparse(path) if (urlObj.scheme == 'http') or (urlObj.scheme == 'https') or (urlObj.scheme == 'ftp') or (urlObj.scheme == 'ftps'): try: - data = urlopen(path).read() + req = request.Request(path) + opener = request.build_opener() + req.add_header('User-Agent', 'Mozilla/5.0') + data = opener.open(req).read() self.array = img_as_ubyte(io.imread(BytesIO(data))) self._format = imghdr.what("", h=data) self._path = path From 70ca1495cb88e169dfffde5a51c4294b28291278 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 08:48:23 +0530 Subject: [PATCH 06/10] changed urlparse library from six.moves.urllib_parse to six.moves.urllib.parse --- skimage/novice/_novice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index a5dbef51..4cbc35ed 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -12,7 +12,7 @@ from skimage.color import color_dict import re import six import six.moves.urllib -from six.moves.urllib_parse import urlparse +from six.moves.urllib.parse import urlparse from six.moves.urllib import request from six.moves.urllib import error urlopen = request.urlopen From dc85186e19b89cca994eef0e452da263d9365f37 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 13:05:53 +0530 Subject: [PATCH 07/10] corrected syntax for error handling while fetching image from a URL in class Picture --- skimage/novice/_novice.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index 4cbc35ed..0b63fcca 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -230,12 +230,12 @@ class Picture(object): self.array = img_as_ubyte(io.imread(BytesIO(data))) self._format = imghdr.what("", h=data) self._path = path - except error.HTTPError, e: - print 'HTTP Error ', e.code - except error.URLError, e: - print 'URL Error\n', e.args + except error.HTTPError as err: + print('HTTP Error: {0}'.format(err)) + except error.URLError as err: + print('URL Error: {0}'.format(err)) except error.ContentTooShortError: - print 'Content too short' + print('Content too short') else: path = os.path.abspath(path) self.array = img_as_ubyte(io.imread(path)) From 7880453d42ac0a4f7625f9ed09995db68d9fd6d3 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 15:57:32 +0530 Subject: [PATCH 08/10] changed the method of fetching images in Picture class using file_or_url_context to get context and calling imread and imghdr on returned context --- skimage/novice/_novice.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index 0b63fcca..ac0754e8 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -8,13 +8,14 @@ from io import BytesIO from skimage import img_as_ubyte from skimage.transform import resize from skimage.color import color_dict +from skimage.io.util import file_or_url_context +from skimage.io.util import is_url import re import six import six.moves.urllib from six.moves.urllib.parse import urlparse from six.moves.urllib import request -from six.moves.urllib import error urlopen = request.urlopen # Convert colors from `skimage.color` to uint8 and allow access through @@ -220,27 +221,14 @@ class Picture(object): msg = "Must provide a single keyword arg (path, array, xy_array)." ValueError(msg) elif path is not None: - urlObj = urlparse(path) - if (urlObj.scheme == 'http') or (urlObj.scheme == 'https') or (urlObj.scheme == 'ftp') or (urlObj.scheme == 'ftps'): - try: - req = request.Request(path) - opener = request.build_opener() - req.add_header('User-Agent', 'Mozilla/5.0') - data = opener.open(req).read() - self.array = img_as_ubyte(io.imread(BytesIO(data))) - self._format = imghdr.what("", h=data) - self._path = path - except error.HTTPError as err: - print('HTTP Error: {0}'.format(err)) - except error.URLError as err: - print('URL Error: {0}'.format(err)) - except error.ContentTooShortError: - print('Content too short') + if is_url(path): + self._path = path else: path = os.path.abspath(path) - self.array = img_as_ubyte(io.imread(path)) self._path = path - self._format = imghdr.what(path) + with file_or_url_context(path) as context: + self.array = io.imread(context) + self._format = imghdr.what(context) elif array is not None: self.array = array elif xy_array is not None: From 0bfdda58e53b77024340830260fff6a8124c0139 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Wed, 12 Mar 2014 17:55:54 +0530 Subject: [PATCH 09/10] removed unnecessary imports from _novice.py --- skimage/novice/_novice.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index ac0754e8..597faca3 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -4,19 +4,13 @@ from collections import namedtuple import numpy as np from skimage import io -from io import BytesIO from skimage import img_as_ubyte from skimage.transform import resize from skimage.color import color_dict from skimage.io.util import file_or_url_context from skimage.io.util import is_url -import re import six -import six.moves.urllib -from six.moves.urllib.parse import urlparse -from six.moves.urllib import request -urlopen = request.urlopen # Convert colors from `skimage.color` to uint8 and allow access through # dict or a named tuple. From 1d133e5c17f1c3aaa5ed955914fc18b2acbf2408 Mon Sep 17 00:00:00 2001 From: Neeraj Gangwar Date: Thu, 13 Mar 2014 18:42:12 +0530 Subject: [PATCH 10/10] made minor changes in syntax 1. merged is_url and file_or_url_context in one import statement 2. changed syntax to check if path is a URL. now more compact --- skimage/novice/_novice.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/skimage/novice/_novice.py b/skimage/novice/_novice.py index 597faca3..e58f1b78 100644 --- a/skimage/novice/_novice.py +++ b/skimage/novice/_novice.py @@ -7,8 +7,7 @@ from skimage import io from skimage import img_as_ubyte from skimage.transform import resize from skimage.color import color_dict -from skimage.io.util import file_or_url_context -from skimage.io.util import is_url +from skimage.io.util import file_or_url_context, is_url import six @@ -215,11 +214,9 @@ class Picture(object): msg = "Must provide a single keyword arg (path, array, xy_array)." ValueError(msg) elif path is not None: - if is_url(path): - self._path = path - else: + if not is_url(path): path = os.path.abspath(path) - self._path = path + self._path = path with file_or_url_context(path) as context: self.array = io.imread(context) self._format = imghdr.what(context)