From 49a94d29869145e1e64b3a17dc734a56140902bc Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Mon, 1 Jun 2015 13:35:45 -0400 Subject: [PATCH 1/4] ADD: Endpoint for multi api requests --- indicoio/__init__.py | 15 ++--- indicoio/config.py | 15 +++++ indicoio/utils/__init__.py | 1 - indicoio/utils/multi.py | 121 +++++++++++++++++++++++++++++++++++++ tests/test_remote.py | 46 ++++++++++++++ 5 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 indicoio/utils/multi.py diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 58b92c8..6cbaab4 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -16,18 +16,11 @@ from indicoio.text.tagging import text_tags from indicoio.images.fer import fer from indicoio.images.features import facial_features from indicoio.images.features import image_features +from indicoio.utils.multi import predict_image, predict_text -apis = [ - 'political', - 'posneg', - 'sentiment', - 'language', - 'fer', - 'facial_features', - 'image_features', - 'text_tags' -] -apis = dict((api, globals().get(api)) for api in apis) +from indicoio.config import API_NAMES + +apis = dict((api, globals().get(api)) for api in API_NAMES) for api in apis: globals()[api] = partial(apis[api]) diff --git a/indicoio/config.py b/indicoio/config.py index 7f271e8..895c76a 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -45,11 +45,26 @@ class Settings(ConfigParser.ConfigParser): None ) +TEXT_APIS = [ + 'text_tags', + 'political', + 'sentiment', + 'language' +] + +IMAGE_APIS = [ + 'fer', + 'facial_features', + 'image_features' +] + +API_NAMES = IMAGE_APIS + TEXT_APIS + ["predict_text", "predict_image"] SETTINGS = Settings(files=[ os.path.expanduser("~/.indicorc"), os.path.join(os.getcwd(), '.indicorc') ]) + api_key = SETTINGS.api_key() cloud = SETTINGS.cloud() PUBLIC_API_HOST = 'apiv2.indico.io' diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index b7fb5eb..86cfa96 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -28,7 +28,6 @@ def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): url = url + "/batch" if batch else url url += "?key=%s" % api_key - response = requests.post(url, data=json_data, headers=JSON_HEADERS) if response.status_code == 503 and cloud != None: raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api)) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py new file mode 100644 index 0000000..fab8098 --- /dev/null +++ b/indicoio/utils/multi.py @@ -0,0 +1,121 @@ +from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES +from indicoio.utils import api_handler + + +CLIENT_SERVER_MAP = dict((api, api.strip().replace("_", "").lower()) for api in API_NAMES) +SERVER_CLIENT_MAP = dict((v, k) for k, v in CLIENT_SERVER_MAP.iteritems()) + +def multi(data, type, apis, available, batch=False, **kwargs): + """ + Helper to make multi requests of different types. + + :param data: data to be sent in JSON. + :param type: String type of API request + :param apis: List of apis to use. + :param apis: List of apis available for use. + :type data: str or image + :type type: str or unicode + :type apis: list of str + :type available: list of str + :rtype: Dictionary of api responses + """ + # Client side api name checking - strictly only accept func name api + invalid_apis = [api for api in apis if api not in available] + if invalid_apis: + raise ValueError("%s are not valid %s APIs. Please reference the available APIs below:\n%s" + % (", ".join(invalid_apis), type, ", ".join(available)) + ) + # Convert client api names to server names before sending request + apis = map(CLIENT_SERVER_MAP.get, apis) + result = api_handler(data, apis=apis, batch=batch, **kwargs) + + if batch: + return [handle_response(each) for each in result] + + return handle_response(result) + +def handle_response(result): + try: + # Parse out the results to a dicionary of api: result + return dict((SERVER_CLIENT_MAP[api], parsed_response(res)) + for api, res in result.iteritems()) + except KeyError: + for api in result: + if "error" in result[api]: + raise ValueError(result[api]["error"]) + raise Exception("Sorry, %s API returned an unexpected response:\n%s" % (api, result[api])) + + +def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwargs): + """ + Given input text, returns the results of specified text apis. Possible apis + include: [ 'text_tags', 'political', 'sentiment', 'language' ] + + Example usage: + + .. code-block:: python + + >>> import indicoio + >>> text = 'Monday: Delightful with mostly sunny skies. Highs in the low 70s.' + >>> results = indicoio.text(data = text, apis = ["language", "sentiment"]) + >>> language_results = results["langauge"] + >>> sentiment_results = results["sentiment"] + + :param text: The text to be analyzed. + :param apis: List of apis to use. + :type text: str or unicode + :type apis: list of str + :rtype: Dictionary of api responses + """ + + return multi( + api="apis", + data=input_text, + type="text", + available = TEXT_APIS, + cloud=cloud, + batch=batch, + api_key=api_key, + apis=apis, + **kwargs) + + +def predict_image(image, apis, cloud=None, batch=False, api_key=None, **kwargs): + """ + Given input image, returns the results of specified image apis. Possible apis + include: ['fer', 'facial_features', 'image_features'] + + Example usage: + + .. code-block:: python + + >>> import indicoio + >>> import numpy as np + >>> face = np.zeros((48,48)).tolist() + >>> results = indicoio.image(image = face, apis = ["fer", "facial_features"]) + >>> fer = results["fer"] + >>> facial_features = results["facial_features"] + + :param text: The text to be analyzed. + :param apis: List of apis to use. + :type text: str or unicode + :type apis: list of str + :rtype: Dictionary of api responses + """ + + return multi( + api="apis", + data=image, + type="image", + available=IMAGE_APIS, + cloud=cloud, + batch=batch, + api_key=api_key, + apis=apis, + **kwargs) + +def parsed_response(response): + result = response.get('results') or response.get('error', False) + if result: + return result + raise KeyError diff --git a/tests/test_remote.py b/tests/test_remote.py index 32f4b3f..b9f85bd 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -9,6 +9,7 @@ from indicoio import config from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags +from indicoio import predict_image, predict_text, batch_predict_image, batch_predict_text DIR = os.path.dirname(os.path.realpath(__file__)) @@ -107,6 +108,51 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) + def test_multi_api_image(self): + test_data = generate_array((48,48)) + response = predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) + + def test_multi_api_text(self): + test_data = 'clearly an english sentence' + response = predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) + + def test_batch_multi_api_image(self): + test_data = [generate_array((48,48))] + response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.IMAGE_APIS)) + + def test_batch_multi_api_text(self): + test_data = ['clearly an english sentence'] + response = batch_predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + + def test_multi_api_bad_api(self): + self.assertRaises(ValueError, + batch_predict_text, + "this shouldn't work", + apis=["sentiment", "somethingbad"]) + + def test_multi_bad_mixed_api(self): + self.assertRaises(ValueError, + predict_text, + "this shouldn't work", + apis=["fer", "sentiment", "facial_features"]) + def test_batch_multi_bad_mixed_api(self): + self.assertRaises(ValueError, + batch_predict_text, + ["this shouldn't work"], + apis=["fer", "sentiment", "facial_features"]) + def test_batch_set_cloud(self): test_data = ['clearly an english sentence'] self.assertRaises(ConnectionError, From adf8295f82adee9e679c3ad342d6d39cb5dbe258 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Wed, 3 Jun 2015 15:37:01 -0400 Subject: [PATCH 2/4] ADD: default apis for text/image with tests --- indicoio/utils/multi.py | 4 ++-- tests/test_remote.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index fab8098..a947e05 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -46,7 +46,7 @@ def handle_response(result): raise Exception("Sorry, %s API returned an unexpected response:\n%s" % (api, result[api])) -def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwargs): +def predict_text(input_text, apis=TEXT_APIS, cloud=None, batch=False, api_key=None, **kwargs): """ Given input text, returns the results of specified text apis. Possible apis include: [ 'text_tags', 'political', 'sentiment', 'language' ] @@ -80,7 +80,7 @@ def predict_text(input_text, apis, cloud=None, batch=False, api_key=None, **kwar **kwargs) -def predict_image(image, apis, cloud=None, batch=False, api_key=None, **kwargs): +def predict_image(image, apis=IMAGE_APIS, cloud=None, batch=False, api_key=None, **kwargs): """ Given input image, returns the results of specified image apis. Possible apis include: ['fer', 'facial_features', 'image_features'] diff --git a/tests/test_remote.py b/tests/test_remote.py index b9f85bd..3b50c55 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -136,6 +136,13 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + def test_default_multi_api_text(self): + test_data = ['clearly an english sentence'] + response = batch_predict_text(test_data, api_key=self.api_key) + + self.assertTrue(isinstance(response, list)) + self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + def test_multi_api_bad_api(self): self.assertRaises(ValueError, batch_predict_text, From 5623f6de9068af35ab7cd15d6e92224f100204c8 Mon Sep 17 00:00:00 2001 From: Madison May Date: Fri, 5 Jun 2015 04:06:14 -0400 Subject: [PATCH 3/4] Updated format for batch --- indicoio/utils/multi.py | 4 ---- tests/test_remote.py | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index a947e05..c023e62 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -28,10 +28,6 @@ def multi(data, type, apis, available, batch=False, **kwargs): # Convert client api names to server names before sending request apis = map(CLIENT_SERVER_MAP.get, apis) result = api_handler(data, apis=apis, batch=batch, **kwargs) - - if batch: - return [handle_response(each) for each in result] - return handle_response(result) def handle_response(result): diff --git a/tests/test_remote.py b/tests/test_remote.py index 3b50c55..62ae094 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -126,22 +126,22 @@ class BatchAPIRun(unittest.TestCase): test_data = [generate_array((48,48))] response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.IMAGE_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS)) def test_batch_multi_api_text(self): test_data = ['clearly an english sentence'] response = batch_predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) def test_default_multi_api_text(self): test_data = ['clearly an english sentence'] response = batch_predict_text(test_data, api_key=self.api_key) - self.assertTrue(isinstance(response, list)) - self.assertTrue(set(response[0].keys()) == set(config.TEXT_APIS)) + self.assertTrue(isinstance(response, dict)) + self.assertTrue(set(response.keys()) == set(config.TEXT_APIS)) def test_multi_api_bad_api(self): self.assertRaises(ValueError, From 6e0ac4cd5d466452f3eb3c5a5cbec3153de05d92 Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Fri, 5 Jun 2015 13:50:13 -0400 Subject: [PATCH 4/4] ADD: url params for apis for multi api req --- indicoio/images/features.py | 4 ++-- indicoio/images/fer.py | 2 +- indicoio/text/lang.py | 2 +- indicoio/text/sentiment.py | 4 ++-- indicoio/text/tagging.py | 2 +- indicoio/utils/__init__.py | 11 +++++------ indicoio/utils/multi.py | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 3c20010..fc2fd10 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -25,7 +25,7 @@ def facial_features(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: List containing feature responses """ image = image_preprocess(image, batch=batch) - return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="facialfeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs) def image_features(image, cloud=None, batch=False, api_key=None, **kwargs): """ @@ -58,4 +58,4 @@ def image_features(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: List containing features """ image = image_preprocess(image, batch=batch, size=(64,64)) - return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="imagefeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 642f086..9a85266 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -27,4 +27,4 @@ def fer(image, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary containing emotion probability pairs """ image = image_preprocess(image, batch=batch) - return api_handler(image, cloud=cloud, api="fer", batch=batch, api_key=api_key, **kwargs) + return api_handler(image, cloud=cloud, api="fer", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 547c18b..d4c42d2 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -24,4 +24,4 @@ def language(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of language probability pairs """ - return api_handler(text, cloud=cloud, api="language", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="language", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 1464943..d61f584 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -26,7 +26,7 @@ def political(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, cloud=cloud, api="political", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="political", url_params={"batch":batch, "api_key":api_key}, **kwargs) def posneg(text, cloud=None, batch=False, api_key=None, **kwargs): """ @@ -49,4 +49,4 @@ def posneg(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Float """ - return api_handler(text, cloud=cloud, api="sentiment", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="sentiment", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 13a7246..8be1f75 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -23,4 +23,4 @@ def text_tags(text, cloud=None, batch=False, api_key=None, **kwargs): :rtype: Dictionary of class probability pairs """ - return api_handler(text, cloud=cloud, api="texttags", batch=batch, api_key=api_key, **kwargs) + return api_handler(text, cloud=cloud, api="texttags", url_params={"batch":batch, "api_key":api_key}, **kwargs) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index 86cfa96..89f9a3e 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -8,7 +8,7 @@ from indicoio import config B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)") -def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): +def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, **kwargs): data = {'data': arg} data.update(**kwargs) json_data = json.dumps(data) @@ -21,12 +21,11 @@ def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs): # default to indico public cloud host = config.PUBLIC_API_HOST - if not api_key: - api_key = config.api_key - url = config.url_protocol + "//%s/%s" % (host, api) - url = url + "/batch" if batch else url - url += "?key=%s" % api_key + url = url + "/batch" if url_params.get("batch", False) else url + url += "?key=%s" % (url_params.get("api_key", None) or config.api_key) + if "apis" in url_params: + url += "&apis=%s" % ",".join(url_params["apis"]) response = requests.post(url, data=json_data, headers=JSON_HEADERS) if response.status_code == 503 and cloud != None: diff --git a/indicoio/utils/multi.py b/indicoio/utils/multi.py index c023e62..e8f033f 100644 --- a/indicoio/utils/multi.py +++ b/indicoio/utils/multi.py @@ -27,7 +27,7 @@ def multi(data, type, apis, available, batch=False, **kwargs): ) # Convert client api names to server names before sending request apis = map(CLIENT_SERVER_MAP.get, apis) - result = api_handler(data, apis=apis, batch=batch, **kwargs) + result = api_handler(data, url_params = {"apis":apis, "batch":batch}, **kwargs) return handle_response(result) def handle_response(result):