From 49a94d29869145e1e64b3a17dc734a56140902bc Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 1 Jun 2015 13:35:45 -0400 Subject: [PATCH 01/12] ADD: Endpoint for multi api requests --- indicoio/__init__.py | 15 ++--- indicoio/config.py | 15 +++++ indicoio/utils/__init__.py | 1 - indicoio/utils/multi.py | 121 +++++++++++++++++++++++++++++++++++++ tests/test_remote.py | 46 ++++++++++++++ 5 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 indicoio/utils/multi.py diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 58b92c8..6cbaab4 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -16,18 +16,11 @@ from indicoio.text.tagging import text_tags from indicoio.images.fer import fer from indicoio.images.features import facial_features from indicoio.images.features import image_features +from indicoio.utils.multi import predict_image, predict_text -apis = [ - 'political', - 'posneg', - 'sentiment', - 'language', - 'fer', - 'facial_features', - 'image_features', - 'text_tags' -] -apis = dict((api, globals().get(api)) for api in apis) +from indicoio.config import API_NAMES + +apis = dict((api, globals().get(api)) for api in API_NAMES) for api in apis: globals()[api] = partial(apis[api]) diff --git a/indicoio/config.py b/indicoio/config.py index 7f271e8..895c76a 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -45,11 +45,26 @@ class Settings(ConfigParser.ConfigParser): None ) +TEXT_APIS = [ + 'text_tags', + 'political', + 'sentiment', + 'language' +] + +IMAGE_APIS = [ + 'fer', + 'facial_features', + 'image_features' +] + +API_NAMES = IMAGE_APIS + TEXT_APIS + ["predict_text", "predict_image"] SETTINGS = Settings(files=[ os.path.expanduser("~/.indicorc"), os.path.join(os.getcwd(), '.indicorc') ]) + api_key = SETTINGS.api_key() cloud = SETTINGS.cloud() PUBLIC_API_HOST = 'apiv2.indico.io' diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index b7fb5eb..86cfa96 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -28,7 +28,6 @@ def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): url = url + "/batch" if batch else url url += "?key=%s" % api_key - response = requests.post(url, data=json_data, headers=JSON_HEADERS) if response.status_code == 503 and cloud != None: raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api)) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py new file mode 100644 index 0000000..fab8098 --- /dev/null +++ b/indicoio/utils/multi.py @@ -0,0 +1,121 @@ +from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES +from indicoio.utils import api_handler + + +CLIENT_SERVER_MAP = dict((api, api.strip().replace("_", "").lower()) for api in API_NAMES) +SERVER_CLIENT_MAP = dict((v, k) for k, v in CLIENT_SERVER_MAP.iteritems()) + +def multi(data, type, apis, available, batch=False, **kwargs): + """ + Helper to make multi requests of different types. + + :param data: data to be sent in JSON. + :param type: String type of API request + :param apis: List of apis to use. + :param apis: List of apis available for use. + :type data: str or image + :type type: str or unicode + :type apis: list of str + :type available: list of str + :rtype: Dictionary of api responses + """ + # Client side api name checking - strictly only accept func name api + invalid_apis = [api for api in apis if api not in available] + if invalid_apis: + raise ValueError("%s are not valid %s APIs. Please reference the available APIs below:\n%s" + % (", ".join(invalid_apis), type, ", ".join(available)) + ) + # Convert client api names to server names before sending request + apis = map(CLIENT_SERVER_MAP.get, apis) + result = api_handler(data, apis=apis, batch=batch, **kwargs) + + if batch: + return [handle_response(each) for each in result] + + return handle_response(result) + +def handle_response(result): + try: + # Parse out the results to a dicionary of api: result + return dict((SERVER_CLIENT_MAP[api], parsed_response(res)) + for api, res in result.iteritems()) + except KeyError: + for api in result: + if "error" in result[api]: + raise ValueError(result[api]["error"]) + raise Exception("Sorry, %s API returned an unexpected response:\n%s" % (api, result[api])) + + +def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwargs): + """ + Given input text, returns the results of specified text apis. Possible apis + include: [ 'text_tags', 'political', 'sentiment', 'language' ] + + Example usage: + + .. code-block:: python + + >>> import indicoio + >>> text = 'Monday: Delightful with mostly sunny skies. Highs in the low 70s.' + >>> results = indicoio.text(data = text, apis = ["language", "sentiment"]) + >>> language_results = results["langauge"] + >>> sentiment_results = results["sentiment"] + + :param text: The text to be analyzed. + :param apis: List of apis to use. + :type text: str or unicode + :type apis: list of str + :rtype: Dictionary of api responses + """ + + return multi( + api="apis", + data=input_text, + type="text", + available = TEXT_APIS, + cloud=cloud, + batch=batch, + api_key=api_key, + apis=apis, + **kwargs) + + +def predict_image(image, apis, cloud=None, batch=False, api_key=None, **kwargs): + """ + Given input image, returns the results of specified image apis. Possible apis + include: ['fer', 'facial_features', 'image_features'] + + Example usage: + + .. code-block:: python + + >>> import indicoio + >>> import numpy as np + >>> face = np.zeros((48,48)).tolist() + >>> results = indicoio.image(image = face, apis = ["fer", "facial_features"]) + >>> fer = results["fer"] + >>> facial_features = results["facial_features"] + + :param text: The text to be analyzed. + :param apis: List of apis to use. + :type text: str or unicode + :type apis: list of str + :rtype: Dictionary of api responses + """ + + return multi( + api="apis", + data=image, + type="image", + available=IMAGE_APIS, + cloud=cloud, + batch=batch, + api_key=api_key, + apis=apis, + **kwargs) + +def parsed_response(response): + result = response.get('results') or response.get('error', False) + if result: + return result + raise KeyError diff --git a/tests/test_remote.py b/tests/test_remote.py index 32f4b3f..b9f85bd 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -9,6 +9,7 @@ from indicoio import config from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags +from indicoio import predict_image, predict_text, batch_predict_image, batch_predict_text DIR = os.path.dirname(os.path.realpath(__file__)) @@ -107,6 +108,51 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) + def test_multi_api_image(self): + test_data = generate_array((48,48)) + response = predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) + + def test_multi_api_text(self): + test_data = 'clearly an english sentence' + response = predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) + + def test_batch_multi_api_image(self): + test_data = [generate_array((48,48))] + response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.IMAGE_APIS)) + + def test_batch_multi_api_text(self): + test_data = ['clearly an english sentence'] + response = batch_predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + + def test_multi_api_bad_api(self): + self.assertRaises(ValueError, + batch_predict_text, + "this shouldn't work", + apis=["sentiment", "somethingbad"]) + + def test_multi_bad_mixed_api(self): + self.assertRaises(ValueError, + predict_text, + "this shouldn't work", + apis=["fer", "sentiment", "facial_features"]) + def test_batch_multi_bad_mixed_api(self): + self.assertRaises(ValueError, + batch_predict_text, + ["this shouldn't work"], + apis=["fer", "sentiment", "facial_features"]) + def test_batch_set_cloud(self): test_data = ['clearly an english sentence'] self.assertRaises(ConnectionError, From adf8295f82adee9e679c3ad342d6d39cb5dbe258 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 3 Jun 2015 15:37:01 -0400 Subject: [PATCH 02/12] ADD: default apis for text/image with tests --- indicoio/utils/multi.py | 4 ++-- tests/test_remote.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index fab8098..a947e05 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -46,7 +46,7 @@ def handle_response(result): raise Exception("Sorry, %s API returned an unexpected response:\n%s" % (api, result[api])) -def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwargs): +def predict_text(input_text, apis=TEXT_APIS, cloud=None, batch=False, api_key=None, **kwargs): """ Given input text, returns the results of specified text apis. Possible apis include: [ 'text_tags', 'political', 'sentiment', 'language' ] @@ -80,7 +80,7 @@ def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwar **kwargs) -def predict_image(image, apis, cloud=None, batch=False, api_key=None, **kwargs): +def predict_image(image, apis=IMAGE_APIS, cloud=None, batch=False, api_key=None, **kwargs): """ Given input image, returns the results of specified image apis. Possible apis include: ['fer', 'facial_features', 'image_features'] diff --git a/tests/test_remote.py b/tests/test_remote.py index b9f85bd..3b50c55 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -136,6 +136,13 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + def test_default_multi_api_text(self): + test_data = ['clearly an english sentence'] + response = batch_predict_text(test_data, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + def test_multi_api_bad_api(self): self.assertRaises(ValueError, batch_predict_text, From 5623f6de9068af35ab7cd15d6e92224f100204c8 Mon Sep 17 00:00:00 2001 From: Madison May Date: Fri, 5 Jun 2015 04:06:14 -0400 Subject: [PATCH 03/12] Updated format for batch --- indicoio/utils/multi.py | 4 ---- tests/test_remote.py | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index a947e05..c023e62 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -28,10 +28,6 @@ def multi(data, type, apis, available, batch=False, **kwargs): # Convert client api names to server names before sending request apis = map(CLIENT_SERVER_MAP.get, apis) result = api_handler(data, apis=apis, batch=batch, **kwargs) - - if batch: - return [handle_response(each) for each in result] - return handle_response(result) def handle_response(result): diff --git a/tests/test_remote.py b/tests/test_remote.py index 3b50c55..62ae094 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -126,22 +126,22 @@ class BatchAPIRun(unittest.TestCase): test_data = [generate_array((48,48))] response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.IMAGE_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) def test_batch_multi_api_text(self): test_data = ['clearly an english sentence'] response = batch_predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) def test_default_multi_api_text(self): test_data = ['clearly an english sentence'] response = batch_predict_text(test_data, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) def test_multi_api_bad_api(self): self.assertRaises(ValueError, From 6e0ac4cd5d466452f3eb3c5a5cbec3153de05d92 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 5 Jun 2015 13:50:13 -0400 Subject: [PATCH 04/12] ADD: url params for apis for multi api req --- indicoio/images/features.py | 4 ++-- indicoio/images/fer.py | 2 +- indicoio/text/lang.py | 2 +- indicoio/text/sentiment.py | 4 ++-- indicoio/text/tagging.py | 2 +- indicoio/utils/__init__.py | 11 +++++------ indicoio/utils/multi.py | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 3c20010..fc2fd10 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -25,7 +25,7 @@ def facial_features(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: List containing feature responses """ image = image_preprocess(image, batch=batch) - return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="facialfeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs) def image_features(image, cloud=None, batch=False, api_key=None, **kwargs): """ @@ -58,4 +58,4 @@ def image_features(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: List containing features """ image = image_preprocess(image, batch=batch, size=(64,64)) - return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="imagefeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 642f086..9a85266 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -27,4 +27,4 @@ def fer(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary containing emotion probability pairs """ image = image_preprocess(image, batch=batch) - return api_handler(image, cloud=cloud, api="fer", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="fer", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 547c18b..d4c42d2 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -24,4 +24,4 @@ def language(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of language probability pairs """ - return api_handler(text, cloud=cloud, api="language", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="language", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 1464943..d61f584 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -26,7 +26,7 @@ def political(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, cloud=cloud, api="political", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="political", url_params={"batch":batch, "api_key":api_key}, **kwargs) def posneg(text, cloud=None, batch=False, api_key=None, **kwargs): """ @@ -49,4 +49,4 @@ def posneg(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Float """ - return api_handler(text, cloud=cloud, api="sentiment", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="sentiment", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 13a7246..8be1f75 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -23,4 +23,4 @@ def text_tags(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of class probability pairs """ - return api_handler(text, cloud=cloud, api="texttags", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="texttags", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index 86cfa96..89f9a3e 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -8,7 +8,7 @@ from indicoio import config B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)") -def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): +def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, **kwargs): data = {'data': arg} data.update(**kwargs) json_data = json.dumps(data) @@ -21,12 +21,11 @@ def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): # default to indico public cloud host = config.PUBLIC_API_HOST - if not api_key: - api_key = config.api_key - url = config.url_protocol + "//%s/%s" % (host, api) - url = url + "/batch" if batch else url - url += "?key=%s" % api_key + url = url + "/batch" if url_params.get("batch", False) else url + url += "?key=%s" % (url_params.get("api_key", None) or config.api_key) + if "apis" in url_params: + url += "&apis=%s" % ",".join(url_params["apis"]) response = requests.post(url, data=json_data, headers=JSON_HEADERS) if response.status_code == 503 and cloud != None: diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index c023e62..e8f033f 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -27,7 +27,7 @@ def multi(data, type, apis, available, batch=False, **kwargs): ) # Convert client api names to server names before sending request apis = map(CLIENT_SERVER_MAP.get, apis) - result = api_handler(data, apis=apis, batch=batch, **kwargs) + result = api_handler(data, url_params = {"apis":apis, "batch":batch}, **kwargs) return handle_response(result) def handle_response(result): From d803846cf0498eb603eea8e0ba7b5ef61f40ad43 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 5 Jun 2015 17:18:46 -0400 Subject: [PATCH 05/12] FIX: accept PIL Image as input --- indicoio/utils/__init__.py | 2 ++ tests/test_remote.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index b7fb5eb..31abcc5 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -107,6 +107,8 @@ def image_preprocess(image, size=(48,48), batch=False): DeprecationWarning ) outImage = process_list_image(image) + elif isinstance(image, Image.Image): + outImage = image elif type(image).__name__ == "ndarray": # image is from numpy/scipy out_image = Image.fromarray(image) else: diff --git a/tests/test_remote.py b/tests/test_remote.py index 32f4b3f..749034e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -62,6 +62,12 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(isinstance(response[0], dict)) + def test_batch_fer_pil_image(self): + test_data = [Image.open(os.path.normpath(os.path.join(DIR, "data/fear.png")))] + response = batch_fer(test_data, api_key=self.api_key) + self.assertTrue(isinstance(response, list)) + self.assertTrue(isinstance(response[0], dict)) + def test_batch_fer_nonexistant_filepath(self): test_data = ["data/unhappy.png"] self.assertRaises(ValueError, batch_fer, test_data, api_key=self.api_key) @@ -184,6 +190,12 @@ class FullAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Happy'] > 0.5) + def test_happy_fer_pil(self): + test_face = Image.open(os.path.normpath(os.path.join(DIR, "data/happy.png"))).convert('L'); + response = fer(test_face) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(response['Happy'] > 0.5) + def test_fear_fer(self): test_face = self.load_image("data/fear.png", as_grey=True) response = fer(test_face) From 5c78d5c9e93a3ea356ab49a0d8a22f8458ffdfc0 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 9 Jun 2015 18:31:31 -0400 Subject: [PATCH 06/12] FIX: incomplete tests testing multi batch apis --- tests/test_remote.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_remote.py b/tests/test_remote.py index ce4c707..2d8c9b7 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -134,6 +134,8 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) + self.assertTrue("results" in response["fer"]) + def test_batch_multi_api_text(self): test_data = ['clearly an english sentence'] @@ -141,6 +143,7 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) + self.assertTrue("results" in response["sentiment"]) def test_default_multi_api_text(self): test_data = ['clearly an english sentence'] From 610577fc703915e62ac17f3fc0b5f51d7c5217a1 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 9 Jun 2015 18:35:55 -0400 Subject: [PATCH 07/12] FIX: Version number update 0.7.0 --- CHANGES.txt | 2 +- README.md | 4 ++-- indicoio/__init__.py | 4 ++-- setup.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 66d2843..f35f802 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,4 +22,4 @@ v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc f v0.5.1, Friday Feb 27 -- More README updates, fixed rst formatting issue, added classifiers v0.5.2, Tue March 7 -- Required API keys, configuration settings v0.5.3, Wed Apr 15 -- Added scipy to requirements, edited Readme to not break pypi page -v0.6.0, Thu May 29 -- Remove numpy / scipy dependency in favor of Pillow \ No newline at end of file +v0.6.0, Thu May 29 -- Remove numpy / scipy dependency in favor of Pillow diff --git a/README.md b/README.md index 55202ad..beb5c2f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ pip install indicoio From source: ```bash -git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git +git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git python setup.py install ``` @@ -62,7 +62,7 @@ Examples >>> text_tags(test_text, top_n=1) # return only keys with top_n values {u'startups_and_entrepreneurship': 0.21888586688354486} ->>> import numpy as np +>>> import numpy as np >>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 6cbaab4..f2abfb9 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -4,10 +4,10 @@ JSON_HEADERS = { 'Content-type': 'application/json', 'Accept': 'application/json', 'client-lib': 'python', - 'version-number': '0.6.0' + 'version-number': '0.7.0' } -Version, version, __version__, VERSION = ('0.6.0',) * 4 +Version, version, __version__, VERSION = ('0.7.0',) * 4 from indicoio.text.sentiment import political, posneg from indicoio.text.sentiment import posneg as sentiment diff --git a/setup.py b/setup.py index 808a286..b00d99a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: setup( name="IndicoIo", - version='0.6.0', + version='0.7.0', packages=[ "indicoio", "indicoio.text", From 0ccd07a6518846b7be8cb2da958a76ae3acb4ee4 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 9 Jun 2015 19:26:23 -0400 Subject: [PATCH 08/12] gs --- indicoio/utils/__init__.py | 9 +++++---- indicoio/utils/errors.py | 2 ++ indicoio/utils/multi.py | 24 +++++++++++------------- tests/test_remote.py | 2 -- 4 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 indicoio/utils/errors.py diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index fefa922..657a2e6 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -2,6 +2,7 @@ import inspect, json, getpass, os.path, base64, StringIO, re, warnings import requests from PIL import Image +from indicoio.utils.errors import IndicoError from indicoio import JSON_HEADERS from indicoio import config @@ -29,13 +30,13 @@ def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, * response = requests.post(url, data=json_data, headers=JSON_HEADERS) if response.status_code == 503 and cloud != None: - raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api)) + raise IndicoError("Private cloud '%s' does not include api '%s'" % (cloud, api)) json_results = response.json() results = json_results.get('results', False) if results is False: error = json_results.get('error') - raise ValueError(error) + raise IndicoError(error) return results @@ -97,7 +98,7 @@ def image_preprocess(image, size=(48,48), batch=False): elif B64_PATTERN.match(b64_str) is not None: return b64_str else: - raise ValueError("Snose tring provided must be a valid filepath or base64 encoded string") + raise IndicoError("Snose tring provided must be a valid filepath or base64 encoded string") elif isinstance(image, list): # image passed in is a list and not np.array warnings.warn( @@ -110,7 +111,7 @@ def image_preprocess(image, size=(48,48), batch=False): elif type(image).__name__ == "ndarray": # image is from numpy/scipy out_image = Image.fromarray(image) else: - raise ValueError("Image must be a filepath, base64 encoded string, or a numpy array") + raise IndicoError("Image must be a filepath, base64 encoded string, or a numpy array") # image resizing outImage = outImage.resize(size) diff --git a/indicoio/utils/errors.py b/indicoio/utils/errors.py new file mode 100644 index 0000000..e7d0d5d --- /dev/null +++ b/indicoio/utils/errors.py @@ -0,0 +1,2 @@ +class IndicoError(ValueError): + pass diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index e8f033f..b60c624 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -1,4 +1,5 @@ from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES +from indicoio.utils.errors import IndicoError from indicoio.utils import api_handler @@ -22,7 +23,7 @@ def multi(data, type, apis, available, batch=False, **kwargs): # Client side api name checking - strictly only accept func name api invalid_apis = [api for api in apis if api not in available] if invalid_apis: - raise ValueError("%s are not valid %s APIs. Please reference the available APIs below:\n%s" + raise IndicoError("%s are not valid %s APIs. Please reference the available APIs below:\n%s" % (", ".join(invalid_apis), type, ", ".join(available)) ) # Convert client api names to server names before sending request @@ -31,15 +32,9 @@ def multi(data, type, apis, available, batch=False, **kwargs): return handle_response(result) def handle_response(result): - try: - # Parse out the results to a dicionary of api: result - return dict((SERVER_CLIENT_MAP[api], parsed_response(res)) - for api, res in result.iteritems()) - except KeyError: - for api in result: - if "error" in result[api]: - raise ValueError(result[api]["error"]) - raise Exception("Sorry, %s API returned an unexpected response:\n%s" % (api, result[api])) + # Parse out the results to a dicionary of api: result + return dict((SERVER_CLIENT_MAP[api], parsed_response(res)) + for api, res in result.iteritems()) def predict_text(input_text, apis=TEXT_APIS, cloud=None, batch=False, api_key=None, **kwargs): @@ -110,8 +105,11 @@ def predict_image(image, apis=IMAGE_APIS, cloud=None, batch=False, api_key=None, apis=apis, **kwargs) -def parsed_response(response): - result = response.get('results') or response.get('error', False) +def parsed_response(api, response): + result = response.get('results', False) if result: return result - raise KeyError + raise IndicoError( + "Sorry, the %s API returned an unexpected response.\n\t%s" + % (api, response.get('error', "")) + ) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2d8c9b7..7ef7a58 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -134,7 +134,6 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) - self.assertTrue("results" in response["fer"]) def test_batch_multi_api_text(self): @@ -143,7 +142,6 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) - self.assertTrue("results" in response["sentiment"]) def test_default_multi_api_text(self): test_data = ['clearly an english sentence'] From b1575a2e9fcdde0e09045a846648ccc1b8d7beab Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 9 Jun 2015 19:27:59 -0400 Subject: [PATCH 09/12] FIX: Added version changes and IndicoError --- CHANGES.txt | 1 + indicoio/__init__.py | 6 +++--- indicoio/utils/multi.py | 2 +- setup.py | 4 +++- tests/test_remote.py | 15 ++++++++------- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f35f802..d026e5d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,3 +23,4 @@ v0.5.1, Friday Feb 27 -- More README updates, fixed rst formatting issue, added v0.5.2, Tue March 7 -- Required API keys, configuration settings v0.5.3, Wed Apr 15 -- Added scipy to requirements, edited Readme to not break pypi page v0.6.0, Thu May 29 -- Remove numpy / scipy dependency in favor of Pillow +v0.7.0, Tue Jun 9 -- Added support for calling multiple APIs in a single function and accepting filenames as image API inputs diff --git a/indicoio/__init__.py b/indicoio/__init__.py index f2abfb9..4b0af6b 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -1,14 +1,14 @@ from functools import partial +Version, version, __version__, VERSION = ('0.7.0',) * 4 + JSON_HEADERS = { 'Content-type': 'application/json', 'Accept': 'application/json', 'client-lib': 'python', - 'version-number': '0.7.0' + 'version-number': VERSION } -Version, version, __version__, VERSION = ('0.7.0',) * 4 - from indicoio.text.sentiment import political, posneg from indicoio.text.sentiment import posneg as sentiment from indicoio.text.lang import language diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index b60c624..ae7b704 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -33,7 +33,7 @@ def multi(data, type, apis, available, batch=False, **kwargs): def handle_response(result): # Parse out the results to a dicionary of api: result - return dict((SERVER_CLIENT_MAP[api], parsed_response(res)) + return dict((SERVER_CLIENT_MAP[api], parsed_response(api, res)) for api, res in result.iteritems()) diff --git a/setup.py b/setup.py index b00d99a..5dbf095 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ """ Setup for indico apis """ +from indicoio import VERSION + try: from setuptools import setup except ImportError: @@ -8,7 +10,7 @@ except ImportError: setup( name="IndicoIo", - version='0.7.0', + version=VERSION, packages=[ "indicoio", "indicoio.text", diff --git a/tests/test_remote.py b/tests/test_remote.py index 7ef7a58..32736db 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -10,6 +10,7 @@ from indicoio import political, sentiment, fer, facial_features, language, image from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags from indicoio import predict_image, predict_text, batch_predict_image, batch_predict_text +from indicoio.utils.errors import IndicoError DIR = os.path.dirname(os.path.realpath(__file__)) @@ -49,7 +50,7 @@ class BatchAPIRun(unittest.TestCase): def test_batch_fer_bad_b64(self): test_data = ["$bad#FI jeaf9(#0"] - self.assertRaises(ValueError, batch_fer, test_data, api_key=self.api_key) + self.assertRaises(IndicoError, batch_fer, test_data, api_key=self.api_key) def test_batch_fer_good_b64(self): test_data = ["iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAg5JREFUeNrEV4uNgzAMpegGyAgZgQ3KBscIjMAGx03QEdqbgG5AOwG3AWwAnSCXqLZkuUkwhfYsvaLm5xc7sZ1dIhdtUVjsLZRFTvp+LSaLq8UZ/s+KMSbZCcY5RV9E4QQKHG7QtgeCGv4PFt8WpzkCcztu3TiL0eJgkQmsVFn0MK+LzYkRKEGpG1GDyZdKRdaolhAoJewXnJsO1jtKCFDlChZAFxyJj2PnBRU20KZg7oMlOAENijpi8hwmGkKkZW2GzONtVLA/DxHAhTO2I7MCVBSQ6nGDlEBJDhyVYiUBHXBxzQm0wE4FzPYsGs856dA9SAAP2oENzFYqR6iAFQpHIAUzO/nxnOgthF/lM3w/3U8KYXTwxG/1IgIulF+wPQUXDMl75UoJZIHstRWpaGb8IGYqwBoKlG/lgpzoUEBoj50p8QtVrmHgaaXyC/H3BFC+e9kGFlCB0CtBF7FifQ8D9zjQQHj0pdOM3F1pUBoFKdxtqkMClScHJCSDlSxhHSNRT5K+FaZnHglrz+AGoxZLKNLYH6s3CkkuyJlp58wviZ4PuSCWDXl5hmjZtxcSCGbDUD3gK7EMOZBLCETrgVBF5K0lI5bIZ0wfrYh8NWHIAiNTPHpuTOKpCes1VTFaiNaFdGwPfdmaqlj6LmjJbgoSSfUW74K3voz+/W0oIeB7HWu2s+dfx3N+eLX8CTAAwUmKjK/dHS4AAAAASUVORK5CYII="] @@ -71,7 +72,7 @@ class BatchAPIRun(unittest.TestCase): def test_batch_fer_nonexistant_filepath(self): test_data = ["data/unhappy.png"] - self.assertRaises(ValueError, batch_fer, test_data, api_key=self.api_key) + self.assertRaises(IndicoError, batch_fer, test_data, api_key=self.api_key) def test_batch_facial_features(self): @@ -151,18 +152,18 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) def test_multi_api_bad_api(self): - self.assertRaises(ValueError, + self.assertRaises(IndicoError, batch_predict_text, "this shouldn't work", apis=["sentiment", "somethingbad"]) def test_multi_bad_mixed_api(self): - self.assertRaises(ValueError, + self.assertRaises(IndicoError, predict_text, "this shouldn't work", apis=["fer", "sentiment", "facial_features"]) def test_batch_multi_bad_mixed_api(self): - self.assertRaises(ValueError, + self.assertRaises(IndicoError, batch_predict_text, ["this shouldn't work"], apis=["fer", "sentiment", "facial_features"]) @@ -363,7 +364,7 @@ class FullAPIRun(unittest.TestCase): def test_set_api_key(self): test_data = 'clearly an english sentence' - self.assertRaises(ValueError, + self.assertRaises(IndicoError, language, test_data, api_key ='invalid_api_key') @@ -372,7 +373,7 @@ class FullAPIRun(unittest.TestCase): config.api_key = 'invalid_api_key' self.assertEqual(config.api_key, 'invalid_api_key') - self.assertRaises(ValueError, + self.assertRaises(IndicoError, language, test_data) From f902eec858efa6018a859694c7691f7b0505eac8 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 10 Jun 2015 10:33:06 -0400 Subject: [PATCH 10/12] ADD: Testing for image processing in multi image predict --- indicoio/utils/multi.py | 4 +-- tests/test_remote.py | 56 +++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index ae7b704..78ca691 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -1,6 +1,6 @@ from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES from indicoio.utils.errors import IndicoError -from indicoio.utils import api_handler +from indicoio.utils import api_handler, image_preprocess CLIENT_SERVER_MAP = dict((api, api.strip().replace("_", "").lower()) for api in API_NAMES) @@ -96,7 +96,7 @@ def predict_image(image, apis=IMAGE_APIS, cloud=None, batch=False, api_key=None, return multi( api="apis", - data=image, + data=image_preprocess(image, batch=batch), type="image", available=IMAGE_APIS, cloud=cloud, diff --git a/tests/test_remote.py b/tests/test_remote.py index 32736db..dcb0f81 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -115,27 +115,13 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) - def test_multi_api_image(self): - test_data = generate_array((48,48)) - response = predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) - - self.assertTrue(isinstance(response, dict)) - self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) - - def test_multi_api_text(self): - test_data = 'clearly an english sentence' - response = predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) - - self.assertTrue(isinstance(response, dict)) - self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) - def test_batch_multi_api_image(self): - test_data = [generate_array((48,48))] + test_data = [generate_array((48,48)), generate_int_array((48,48))] response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) - + self.assertTrue(isinstance(response["fer"], list)) def test_batch_multi_api_text(self): test_data = ['clearly an english sentence'] @@ -179,6 +165,9 @@ class BatchAPIRun(unittest.TestCase): class FullAPIRun(unittest.TestCase): + def setUp(self): + self.api_key = config.api_key + def load_image(self, relpath, as_grey=False): im = Image.open(os.path.normpath(os.path.join(DIR, relpath))).convert('L'); pixels = list(im.getdata()) @@ -239,6 +228,14 @@ class FullAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, dict)) self.assertEqual(fer_set, set(response.keys())) + def test_good_int_array_fer(self): + fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) + test_face = generate_int_array((48,48)) + response = fer(test_face) + + self.assertTrue(isinstance(response, dict)) + self.assertEqual(fer_set, set(response.keys())) + def test_happy_fer(self): test_face = self.load_image("data/happy.png", as_grey=True) response = fer(test_face) @@ -273,6 +270,15 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(len(response), 48) self.check_range(response) + def test_good_int_array_facial_features(self): + fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) + test_face = generate_int_array((48,48)) + response = facial_features(test_face) + + self.assertTrue(isinstance(response, list)) + self.assertEqual(len(response), 48) + self.check_range(response) + # TODO: uncomment this test once the remote server is updated to # deal with image_urls # def test_image_url(self): @@ -299,6 +305,21 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(len(response), 2048) self.check_range(response) + def test_multi_api_image(self): + test_data = generate_array((48,48)) + response = predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) + + def test_multi_api_text(self): + test_data = 'clearly an english sentence' + response = predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) + + def test_language(self): language_set = set([ 'English', @@ -390,6 +411,9 @@ def flatten(container): def generate_array(size): return [[random.random() for _ in xrange(size[0])] for _ in xrange(size[1])] +def generate_int_array(size): + return [[random.randint(0, 50) for _ in xrange(size[0])] for _ in xrange(size[1])] + if __name__ == "__main__": unittest.main() From 585fd16fae982d7a3db33423601344b1da9ccdc7 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 10 Jun 2015 10:52:30 -0400 Subject: [PATCH 11/12] UPDATE: README --- README.md | 29 +++++++++++++++++++++++++++++ README.rst | 51 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index beb5c2f..508c6d3 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,32 @@ Each `indicoio` function has a corresponding batch function for analyzing many e >>> batch_sentiment(['Best day ever', 'Worst day ever']) [0.9899001220871786, 0.005709885173415242] ``` + + +Calling multiple APIs with a single function +--------- +There are two multiple API functions `predict_text` and `predict_image`. These functions are similar to the existing api functions, but take in an additional `apis` argument as a list of strings of API names (defaults to all existing apis). `predict_text` accepts a list of existing text APIs and vice versa for `predict_image`. These functions also support batch as the other functions do. + +Accepted text API names: `text_tags, political, sentiment, language` + +Accepted image API names: `fer, facial_features, image_features` + +```python +>>> from indicoio import predict_text, predict_image, batch_predict_text, batch_predict_image + +>>> predict_text('Best day ever', apis=["sentiment", "language"]) +{'sentiment': 0.9899001220871786, 'language': {u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, ...}} + +>>> batch_predict_text(['Best day ever', 'Worst day ever'], apis=["sentiment", "language"]) +{'sentiment': [0.9899001220871786, 0.005709885173415242], 'language': [{u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, u'Romanian': 0.00010661175919993216, ...}, {u'Swedish': 0.4924352805804646, u'Vietnamese': 0.028574824174911372, u'Romanian': 0.004185623723173551, u'Dutch': 0.000717033819689362, u'Korean': 0.0030093489153785826, ...}]} + +>>> import numpy as np + +>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() + +>>> predict_image(test_face, apis=["fer", "facial_features"]) +{'facial_features': [0.0, -0.026176479280200796, 0.20707644777495776, ...], 'fer': {u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}} + +>>> batch_predict_image([test_face, test_face], apis=["fer", "facial_features"]) +{'facial_features': [[0.0, -0.026176479280200796, 0.20707644777495776, ...], [0.0, -0.026176479280200796, 0.20707644777495776, ...]], 'fer': [{u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}, { u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}]} +``` diff --git a/README.rst b/README.rst index a31c07d..7b692de 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,7 @@ From source: .. code:: bash - git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git + git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git python setup.py install API Keys + Setup @@ -54,8 +54,7 @@ Examples >>> indicoio.config.api_key = "YOUR_API_KEY" >>> political("Guns don't kill people. People kill people.") - {u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, - u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} + {u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} >>> sentiment('Worst movie ever.') 0.07062467665597527 @@ -71,23 +70,21 @@ Examples >>> text_tags(test_text, top_n=1) # return only keys with top_n values {u'startups_and_entrepreneurship': 0.21888586688354486} - >>> import numpy as np + >>> import numpy as np >>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() >>> fer(test_face) - {u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, - u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} + {u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} >>> facial_features(test_face) [0.0, -0.02568680526917187, 0.21645604230056517, ..., 3.0342637531932777] >>> language('Quis custodiet ipsos custodes') - {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, - u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} + {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} -Batch API Access ----------------- +Batch API +--------- Each ``indicoio`` function has a corresponding batch function for analyzing many examples with a single request. Simply pass in a list of @@ -100,3 +97,37 @@ inputs and receive a list of results in return. >>> batch_sentiment(['Best day ever', 'Worst day ever']) [0.9899001220871786, 0.005709885173415242] +Calling multiple APIs with a single function +-------------------------------------------- + +There are two multiple API functions ``predict_text`` and +``predict_image``. These functions are similar to the existing api +functions, but take in an additional ``apis`` argument as a list of +strings of API names (defaults to all existing apis). ``predict_text`` +accepts a list of existing text APIs and vice versa for +``predict_image``. These functions also support batch as the other +functions do. + +Accepted text API names: ``text_tags, political, sentiment, language`` + +Accepted image API names: ``fer, facial_features, image_features`` + +.. code:: python + + >>> from indicoio import predict_text, predict_image, batch_predict_text, batch_predict_image + + >>> predict_text('Best day ever', apis=["sentiment", "language"]) + {'sentiment': 0.9899001220871786, 'language': {u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, ...}} + + >>> batch_predict_text(['Best day ever', 'Worst day ever'], apis=["sentiment", "language"]) + {'sentiment': [0.9899001220871786, 0.005709885173415242], 'language': [{u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, u'Romanian': 0.00010661175919993216, ...}, {u'Swedish': 0.4924352805804646, u'Vietnamese': 0.028574824174911372, u'Romanian': 0.004185623723173551, u'Dutch': 0.000717033819689362, u'Korean': 0.0030093489153785826, ...}]} + + >>> import numpy as np + + >>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() + + >>> predict_image(test_face, apis=["fer", "facial_features"]) + {'facial_features': [0.0, -0.026176479280200796, 0.20707644777495776, ...], 'fer': {u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}} + + >>> batch_predict_image([test_face, test_face], apis=["fer", "facial_features"]) + {'facial_features': [[0.0, -0.026176479280200796, 0.20707644777495776, ...], [0.0, -0.026176479280200796, 0.20707644777495776, ...]], 'fer': [{u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}, { u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}]} From 074a15c5624cb8513d3d3b715ca5e0375e9eeed7 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 10 Jun 2015 12:52:02 -0400 Subject: [PATCH 12/12] REFACTOR: indicio.utils.__init__.py to multiple utils modules --- indicoio/images/features.py | 3 +- indicoio/images/fer.py | 3 +- indicoio/text/lang.py | 2 +- indicoio/text/sentiment.py | 2 +- indicoio/text/tagging.py | 2 +- indicoio/utils/__init__.py | 159 ++---------------------------------- indicoio/utils/api.py | 39 +++++++++ indicoio/utils/errors.py | 18 ++++ indicoio/utils/image.py | 110 +++++++++++++++++++++++++ indicoio/utils/multi.py | 3 +- 10 files changed, 181 insertions(+), 160 deletions(-) create mode 100644 indicoio/utils/api.py create mode 100644 indicoio/utils/image.py diff --git a/indicoio/images/features.py b/indicoio/images/features.py index fc2fd10..e1fc18b 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -1,6 +1,7 @@ import requests -from indicoio.utils import image_preprocess, api_handler +from indicoio.utils.image import image_preprocess +from indicoio.utils.api import api_handler def facial_features(image, cloud=None, batch=False, api_key=None, **kwargs): """ diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 9a85266..b36d6ff 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -1,6 +1,7 @@ import requests -from indicoio.utils import api_handler, image_preprocess +from indicoio.utils.api import api_handler +from indicoio.utils.image import image_preprocess import indicoio.config as config def fer(image, cloud=None, batch=False, api_key=None, **kwargs): diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index d4c42d2..9f0871f 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,4 +1,4 @@ -from indicoio.utils import api_handler +from indicoio.utils.api import api_handler import indicoio.config as config def language(text, cloud=None, batch=False, api_key=None, **kwargs): diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index d61f584..182908a 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -1,4 +1,4 @@ -from indicoio.utils import api_handler +from indicoio.utils.api import api_handler def political(text, cloud=None, batch=False, api_key=None, **kwargs): """ diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 8be1f75..e0f58de 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,4 +1,4 @@ -from indicoio.utils import api_handler +from indicoio.utils.api import api_handler import indicoio.config as config def text_tags(text, cloud=None, batch=False, api_key=None, **kwargs): diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index 657a2e6..3d5e122 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -1,44 +1,9 @@ -import inspect, json, getpass, os.path, base64, StringIO, re, warnings -import requests -from PIL import Image - -from indicoio.utils.errors import IndicoError -from indicoio import JSON_HEADERS -from indicoio import config - -B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)") - - -def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, **kwargs): - data = {'data': arg} - data.update(**kwargs) - json_data = json.dumps(data) - if not cloud: - cloud=config.cloud - - if cloud: - host = "%s.indico.domains" % cloud - else: - # default to indico public cloud - host = config.PUBLIC_API_HOST - - url = config.url_protocol + "//%s/%s" % (host, api) - url = url + "/batch" if url_params.get("batch", False) else url - url += "?key=%s" % (url_params.get("api_key", None) or config.api_key) - if "apis" in url_params: - url += "&apis=%s" % ",".join(url_params["apis"]) - - response = requests.post(url, data=json_data, headers=JSON_HEADERS) - if response.status_code == 503 and cloud != None: - raise IndicoError("Private cloud '%s' does not include api '%s'" % (cloud, api)) - - json_results = response.json() - results = json_results.get('results', False) - if results is False: - error = json_results.get('error') - raise IndicoError(error) - return results +""" +Basic utility classes and functions +""" +import inspect +from indicoio.utils.errors import DataStructureException class TypeCheck(object): """ @@ -67,120 +32,6 @@ class TypeCheck(object): return check_args -class DataStructureException(Exception): - """ - If a non-accepted datastructure is passed, throws an exception - """ - def __init__(self, callback, passed_structure, accepted_structures): - self.callback = callback.__name__ - self.structure = str(type(passed_structure)) - self.accepted = [str(structure) for structure in accepted_structures] - - def __str__(self): - return """ - function %s does not accept %s, accepted types are: %s - """ % (self.callback, self.structure, str(self.accepted)) - - -def image_preprocess(image, size=(48,48), batch=False): - """ - Takes an image and prepares it for sending to the api including - resizing and image data/structure standardizing. - """ - if batch: - return [image_preprocess(img, batch=False) for img in image] - - if isinstance(image, basestring): - b64_str = re.sub('^data:image/.+;base64,', '', image) - if os.path.isfile(image): - # check type of element - outImage = Image.open(image) - elif B64_PATTERN.match(b64_str) is not None: - return b64_str - else: - raise IndicoError("Snose tring provided must be a valid filepath or base64 encoded string") - - elif isinstance(image, list): # image passed in is a list and not np.array - warnings.warn( - "Input as lists of pixels will be deprecated in the next major update", - DeprecationWarning - ) - outImage = process_list_image(image) - elif isinstance(image, Image.Image): - outImage = image - elif type(image).__name__ == "ndarray": # image is from numpy/scipy - out_image = Image.fromarray(image) - else: - raise IndicoError("Image must be a filepath, base64 encoded string, or a numpy array") - - # image resizing - outImage = outImage.resize(size) - - # convert to base64 - temp_output = StringIO.StringIO() - outImage.save(temp_output, format='PNG') - temp_output.seek(0) - output_s = temp_output.read() - - return base64.b64encode(output_s) - - -def get_list_dimensions(_list): - """ - Takes a nested list and returns the size of each dimension followed - by the element type in the list - """ - if isinstance(_list, list) or isinstance(_list, tuple): - return [len(_list)] + get_list_dimensions(_list[0]) - return [] - - -def get_element_type(_list, dimens): - """ - Given the dimensions of a nested list and the list, returns the type of the - elements in the inner list. - """ - elem = _list - for _ in xrange(len(dimens)): - elem = elem[0] - return type(elem) - - -def process_list_image(_list): - """ - Processes list to be [[(int, int, int), ...]] - """ - # Check if list is empty - if not _list: - return _list - - dimens = get_list_dimensions(_list) - data_type = get_element_type(_list, dimens) - - seq_obj = [] - - outImage = Image.new("RGB", (dimens[0], dimens[1])) - for i in xrange(dimens[0]): - for j in xrange(dimens[1]): - elem = _list[i][j] - if len(dimens) >= 3: - #RGB(A) - if data_type == float: - seq_obj.append((int(elem[0] * 255), int(elem[1] * 255), int(elem[2] * 255))) - else: - seq_obj.append(elem[0:3]) - elif data_type == float: - #Grayscale 0 - 1.0f - seq_obj.append((int(elem * 255), ) * 3) - else: - #Grayscale 0 - 255 - seq_obj.append((elem, ) * 3) - - #Needs to be 0 - 255 in flattened list of (R, G, B) - outImage.putdata(data = seq_obj) - - return outImage - def is_url(data, batch=False): if batch and isinstance(data[0], basestring): diff --git a/indicoio/utils/api.py b/indicoio/utils/api.py new file mode 100644 index 0000000..71a43b5 --- /dev/null +++ b/indicoio/utils/api.py @@ -0,0 +1,39 @@ +""" +Handles making requests to the IndicoApi Server +""" + +import json, requests + +from indicoio.utils.errors import IndicoError, DataStructureException +from indicoio import JSON_HEADERS +from indicoio import config + +def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, **kwargs): + data = {'data': arg} + data.update(**kwargs) + json_data = json.dumps(data) + if not cloud: + cloud=config.cloud + + if cloud: + host = "%s.indico.domains" % cloud + else: + # default to indico public cloud + host = config.PUBLIC_API_HOST + + url = config.url_protocol + "//%s/%s" % (host, api) + url = url + "/batch" if url_params.get("batch", False) else url + url += "?key=%s" % (url_params.get("api_key", None) or config.api_key) + if "apis" in url_params: + url += "&apis=%s" % ",".join(url_params["apis"]) + + response = requests.post(url, data=json_data, headers=JSON_HEADERS) + if response.status_code == 503 and cloud != None: + raise IndicoError("Private cloud '%s' does not include api '%s'" % (cloud, api)) + + json_results = response.json() + results = json_results.get('results', False) + if results is False: + error = json_results.get('error') + raise IndicoError(error) + return results diff --git a/indicoio/utils/errors.py b/indicoio/utils/errors.py index e7d0d5d..731029a 100644 --- a/indicoio/utils/errors.py +++ b/indicoio/utils/errors.py @@ -1,2 +1,20 @@ +""" +Contains Indico Custom Errors +""" + class IndicoError(ValueError): pass + +class DataStructureException(Exception): + """ + If a non-accepted datastructure is passed, throws an exception + """ + def __init__(self, callback, passed_structure, accepted_structures): + self.callback = callback.__name__ + self.structure = str(type(passed_structure)) + self.accepted = [str(structure) for structure in accepted_structures] + + def __str__(self): + return """ + function %s does not accept %s, accepted types are: %s + """ % (self.callback, self.structure, str(self.accepted)) diff --git a/indicoio/utils/image.py b/indicoio/utils/image.py new file mode 100644 index 0000000..5fc9d1b --- /dev/null +++ b/indicoio/utils/image.py @@ -0,0 +1,110 @@ +""" +Image Utils +Handles preprocessing images before they are sent to the server +""" +import os.path, base64, StringIO, re, warnings + +from PIL import Image + +from indicoio.utils.errors import IndicoError, DataStructureException + +B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)") + +def image_preprocess(image, size=(48,48), batch=False): + """ + Takes an image and prepares it for sending to the api including + resizing and image data/structure standardizing. + """ + if batch: + return [image_preprocess(img, batch=False) for img in image] + + if isinstance(image, basestring): + b64_str = re.sub('^data:image/.+;base64,', '', image) + if os.path.isfile(image): + # check type of element + outImage = Image.open(image) + elif B64_PATTERN.match(b64_str) is not None: + return b64_str + else: + raise IndicoError("Snose tring provided must be a valid filepath or base64 encoded string") + + elif isinstance(image, list): # image passed in is a list and not np.array + warnings.warn( + "Input as lists of pixels will be deprecated in the next major update", + DeprecationWarning + ) + outImage = process_list_image(image) + elif isinstance(image, Image.Image): + outImage = image + elif type(image).__name__ == "ndarray": # image is from numpy/scipy + out_image = Image.fromarray(image) + else: + raise IndicoError("Image must be a filepath, base64 encoded string, or a numpy array") + + # image resizing + outImage = outImage.resize(size) + + # convert to base64 + temp_output = StringIO.StringIO() + outImage.save(temp_output, format='PNG') + temp_output.seek(0) + output_s = temp_output.read() + + return base64.b64encode(output_s) + + +def get_list_dimensions(_list): + """ + Takes a nested list and returns the size of each dimension followed + by the element type in the list + """ + if isinstance(_list, list) or isinstance(_list, tuple): + return [len(_list)] + get_list_dimensions(_list[0]) + return [] + + +def get_element_type(_list, dimens): + """ + Given the dimensions of a nested list and the list, returns the type of the + elements in the inner list. + """ + elem = _list + for _ in xrange(len(dimens)): + elem = elem[0] + return type(elem) + + +def process_list_image(_list): + """ + Processes list to be [[(int, int, int), ...]] + """ + # Check if list is empty + if not _list: + return _list + + dimens = get_list_dimensions(_list) + data_type = get_element_type(_list, dimens) + + seq_obj = [] + + outImage = Image.new("RGB", (dimens[0], dimens[1])) + for i in xrange(dimens[0]): + for j in xrange(dimens[1]): + elem = _list[i][j] + if len(dimens) >= 3: + #RGB(A) + if data_type == float: + seq_obj.append((int(elem[0] * 255), int(elem[1] * 255), int(elem[2] * 255))) + else: + seq_obj.append(elem[0:3]) + elif data_type == float: + #Grayscale 0 - 1.0f + seq_obj.append((int(elem * 255), ) * 3) + else: + #Grayscale 0 - 255 + seq_obj.append((elem, ) * 3) + + #Needs to be 0 - 255 in flattened list of (R, G, B) + outImage.putdata(data = seq_obj) + + return outImage diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index 78ca691..31725c3 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -1,6 +1,7 @@ from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES +from indicoio.utils.api import api_handler +from indicoio.utils.image import image_preprocess from indicoio.utils.errors import IndicoError -from indicoio.utils import api_handler, image_preprocess CLIENT_SERVER_MAP = dict((api, api.strip().replace("_", "").lower()) for api in API_NAMES)