From 6b9251f4f5dfb0b9b2356a8e62a7706fee2b7dce Mon Sep 17 00:00:00 2001 From: Anne Carlson Date: Thu, 26 Feb 2015 16:33:00 -0500 Subject: [PATCH 1/7] Passing in a root_url variable and pulling config from environment variables --- indicoio/__init__.py | 11 +++-------- indicoio/config.py | 5 +++-- indicoio/images/features.py | 15 ++++++++------- indicoio/images/fer.py | 16 +++++++++------- indicoio/text/lang.py | 9 +++++---- indicoio/text/sentiment.py | 13 +++++++------ indicoio/text/tagging.py | 7 ++++--- 7 files changed, 39 insertions(+), 37 deletions(-) diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 0921836..eaa98b1 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -1,5 +1,4 @@ from functools import partial -import indicoio.config as config JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'} @@ -13,14 +12,10 @@ from indicoio.images.fer import fer from indicoio.images.features import facial_features from indicoio.images.features import image_features -apis = ['political', 'posneg', 'sentiment', 'language', 'fer', +apis = ['political', 'posneg', 'sentiment', 'language', 'fer', 'facial_features', 'image_features', 'text_tags'] apis = dict((api, globals().get(api)) for api in apis) -class Namespace(object): pass -local = Namespace() for api in apis: - globals()[api] = partial(apis[api], config.api_root) - globals()['batch_' + api] = partial(apis[api], config.api_root, batch=True) - setattr(local, api, partial(apis[api], config.local_api_root)) - setattr(local, 'batch_' + api, partial(apis[api], config.local_api_root, batch=True)) + globals()[api] = partial(apis[api]) + globals()['batch_' + api] = partial(apis[api], batch=True) diff --git a/indicoio/config.py b/indicoio/config.py index 150d0f9..fa38e85 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,2 +1,3 @@ -local_api_root = "http://localhost:9438/" -api_root = "http://apiv1.indico.io/" \ No newline at end of file +import os + +api_root = os.getenv("INDICO_PRIVATE_CLOUD_URL") or "http://apiv1.indico.io/" diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 8ed7903..737309e 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -4,12 +4,13 @@ import requests import numpy as np from indicoio.utils import image_preprocess, api_handler +import indicoio.config as config -def facial_features(api_root, image, batch=False, auth=None, **kwargs): +def facial_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): """ Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face. Useful as a form of feature engineering for face oriented tasks. - Input should be in a list of list format, resizing will be attempted internally but for best + Input should be in a list of list format, resizing will be attempted internally but for best performance, images should be already sized at 48x48 pixels. Example usage: @@ -27,18 +28,18 @@ def facial_features(api_root, image, batch=False, auth=None, **kwargs): :type image: list of lists :rtype: List containing feature responses """ - return api_handler(image, api_root + "facialfeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, url_root + "facialfeatures", batch=batch, auth=auth, **kwargs) -def image_features(api_root, image, batch=False, auth=None, **kwargs): +def image_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): """ - Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. + Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. Useful as a form of feature engineering for image oriented tasks. * Input can be either grayscale or rgb color and should either be a numpy array or nested list format. * Input data should be either uint8 0-255 range values or floating point between 0 and 1. * Large images (i.e. 1024x768+) are much bigger than needed, resizing will be done internally to 64x64 if needed. * For ideal performance, images should be square aspect ratio but non-square aspect ratios are supported as well. - + Example usage: .. code-block:: python @@ -60,4 +61,4 @@ def image_features(api_root, image, batch=False, auth=None, **kwargs): :rtype: List containing features """ image = image_preprocess(image, batch=batch) - return api_handler(image, api_root + "imagefeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, url_root + "imagefeatures", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 2248fe8..2b9527f 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -2,12 +2,14 @@ import json import requests import numpy as np -from indicoio.utils import api_handler -def fer(api_root, image, batch=False, auth=None, **kwargs): +from indicoio.utils import api_handler +import indicoio.config as config + +def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs): """ Given a grayscale input image of a face, returns a probability distribution over emotional state. - Input should be in a list of list format, resizing will be attempted internally but for best + Input should be in a list of list format, resizing will be attempted internally but for best performance, images should be already sized at 48x48 pixels.. Example usage: @@ -19,13 +21,13 @@ def fer(api_root, image, batch=False, auth=None, **kwargs): >>> face = np.zeros((48,48)).tolist() >>> emotions = fer(face) >>> emotions - {u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839, - u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344, + {u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839, + u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344, u'Fear': 0.08523385724298838, u'Happy': 0.04117802220878012} :param image: The image to be analyzed. :type image: list of lists :rtype: Dictionary containing emotion probability pairs """ - - return api_handler(image, api_root + "fer", batch=batch, auth=auth, **kwargs) + + return api_handler(image, url_root + "fer", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 51bd339..8746c7c 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,8 +1,9 @@ from indicoio.utils import api_handler +import indicoio.config as config -def language(api_root, text, batch=False, auth=None, **kwargs): +def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs): """ - Given input text, returns a probability distribution over 33 possible + Given input text, returns a probability distribution over 33 possible languages of what language the text was written in. Example usage: @@ -22,5 +23,5 @@ def language(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Dictionary of language probability pairs """ - - return api_handler(text, api_root + "language", batch=batch, auth=auth, **kwargs) + + return api_handler(text, url_root + "language", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 387bb46..73e29af 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -1,7 +1,8 @@ from indicoio import JSON_HEADERS from indicoio.utils import api_handler +import indicoio.config as config -def political(api_root, text, batch=False, auth=None, **kwargs): +def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over the political alignment of the speaker. @@ -15,7 +16,7 @@ def political(api_root, text, batch=False, auth=None, **kwargs): Hopefully, driverless cars will chance economics from ownership to fee for service.' >>> affiliation = political(text) >>> affiliation - {u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122, + {u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122, u'Liberal': 0.13730032938784784, u'Conservative': 0.07287981563170784} >>> least_like = affiliation.keys()[np.argmin(affiliation.values())] >>> most_like = affiliation.keys()[np.argmax(affiliation.values())] @@ -27,9 +28,9 @@ def political(api_root, text, batch=False, auth=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, api_root + "political", batch=batch, auth=auth, **kwargs) + return api_handler(text, url_root + "political", batch=batch, auth=auth, **kwargs) -def posneg(api_root, text, batch=False, auth=None, **kwargs): +def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs): """ Given input text, returns a scalar estimate of the sentiment of that text. Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment. @@ -49,5 +50,5 @@ def posneg(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Float """ - - return api_handler(text, api_root + "sentiment", batch=batch, auth=auth, **kwargs) + + return api_handler(text, url_root + "sentiment", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 6fabd7b..f820018 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,6 +1,7 @@ from indicoio.utils import api_handler +import indicoio.config as config -def text_tags(api_root, text, batch=False, auth=None, **kwargs): +def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 100 document categories @@ -21,5 +22,5 @@ def text_tags(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Dictionary of class probability pairs """ - - return api_handler(text, api_root + "texttags", batch=batch, auth=auth, **kwargs) + + return api_handler(text, url_root + "texttags", batch=batch, auth=auth, **kwargs) From dc47fe5b0a673a5bfbe89d9eb39cc75bb6fe7666 Mon Sep 17 00:00:00 2001 From: Anne Carlson Date: Thu, 26 Feb 2015 18:50:20 -0500 Subject: [PATCH 2/7] Initial test coverage for new configuration system --- README | 99 ------------------------- README.md | 37 ++++++++-- README.rst | 123 ++++++++++++++++++++++++++++++++ indicoio/config.py | 4 +- indicoio/utils/__init__.py | 4 +- tests/remote/test_remote.py | 22 +++++- tests/remote/test_unit_tests.py | 15 ++++ 7 files changed, 192 insertions(+), 112 deletions(-) delete mode 100644 README create mode 100644 README.rst create mode 100644 tests/remote/test_unit_tests.py diff --git a/README b/README deleted file mode 100644 index 9cd838f..0000000 --- a/README +++ /dev/null @@ -1,99 +0,0 @@ -indicoio-python -=============== - -A wrapper for a series of APIs made by indico. - -Check out the main site on: - -http://indico.io - -Check out our documentation on: - -http://indicoiopython.s3-website-us-west-2.amazonaws.com/indicoio.html - -Our APIs are totally free to use, and ready to be used in your application. No data or training required. - -Current APIs ------------- - -Right now this wrapper supports the following apps: - -- Positive/Negative Sentiment Analysis -- Political Sentiment Analysis -- Image Feature Extraction -- Facial Emotion Recognition -- Facial Feature Extraction -- Language Detection -- Text Topic Tagging - -Examples --------- -``` ->>> import numpy as np - ->>> from indicoio import political, sentiment, fer, facial_features, language - ->>> political("Guns don't kill people. People kill people.") -{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} - ->>> sentiment('Worst movie ever.') -{u'Sentiment': 0.07062467665597527} - ->>> sentiment('Really enjoyed the movie.') -{u'Sentiment': 0.8105182526856075} - ->>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets." - ->>> tag_dict = text_tags(test_text) - ->>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3] -[u'startups_and_entrepreneurship', u'investment', u'business'] - ->>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1 -{u'startups_and_entrepreneurship': 0.21888586688354486} - ->>> text_tags(test_text, top_n=1) # return only keys with top_n values -{u'startups_and_entrepreneurship': 0.21888586688354486} - ->>> tag_dict -{u'fashion': 0.011450126534350728, u'art': 0.00358698972755963, u'energy': 0.005537894035625527, ...} - ->>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() - ->>> fer(test_face) -{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} - ->>> facial_features(test_face) -[0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777] - ->>> language_dict = language('Quis custodiet ipsos custodes') - ->>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5] -[u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish'] - ->>> language_dict -{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} - -``` - -If you have a local indico server running, simply import from `indicoio.local`. - -``` ->>> from indicoio.local import political, sentiment, fer, facial_features, language -``` - -If you'd like to use our batch api interface, please send an email to contact@indico.io. - -``` ->>> from indicio import batch_sentiment -batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) -``` - - -Installation ------------- -``` -pip install indicoio -``` - -Announcement: Indico has partnered with Experfy, a data science consulting marketplace based in the Harvard Innovation Lab. Through Experfy, we are helping our data science community members find lucrative projects and advance their skills. Please signup for Experfy at https://www.experfy.com/ to get started. \ No newline at end of file diff --git a/README.md b/README.md index 98c11ef..4452f3c 100644 --- a/README.md +++ b/README.md @@ -78,13 +78,6 @@ Examples >>> language_dict {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} -``` - -If you have a local indico server running, simply import from `indicoio.local`. - -``` ->>> from indicoio.local import political, sentiment, fer, facial_features, language -``` Batch API Access ---------------- @@ -95,3 +88,33 @@ If you'd like to use our batch api interface, please send an email to contact@in >>> from indicio import batch_sentiment batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) ``` + +Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc + +Private Cloud API Access +------------------------ + +If you'd like to use our private cloud interface, please send an email to contact@indico.io. + +``` +>>> from indicio import sentiment +sentiment("Text to analyze", hostname="http://exampleprivatecloud.io/", auth=("example@example.com", "********")) +``` + +Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc + +indicorc +------------------------ + +Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. + +Here is an example of a valid indicorc file: + +``` +[auth] +username = test@example.com +password = secret + +[private_cloud] +hostname = example.indico.io +``` diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..4c2c8c9 --- /dev/null +++ b/README.rst @@ -0,0 +1,123 @@ +indicoio-python +=============== + +A wrapper for a series of APIs made by indico. + +Check out the main site on: + +http://indico.io + +Our APIs are totally free to use, and ready to be used in your +application. No data or training required. + +Installation +------------ + +:: + + pip install indicoio + +Documentation +------------- + +Available at `indico.reame.io `__ + +Current APIs +------------ + +Right now this wrapper supports the following apps: + +- Positive/Negative Sentiment Analysis +- Political Sentiment Analysis +- Image Feature Extraction +- Facial Emotion Recognition +- Facial Feature Extraction +- Language Detection +- Text Topic Tagging + +Examples +-------- + +:: + + >>> import numpy as np + + >>> from indicoio import political, sentiment, fer, facial_features, language + + >>> political("Guns don't kill people. People kill people.") + {u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} + + >>> sentiment('Worst movie ever.') + {u'Sentiment': 0.07062467665597527} + + >>> sentiment('Really enjoyed the movie.') + {u'Sentiment': 0.8105182526856075} + + >>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets." + + >>> tag_dict = text_tags(test_text) + + >>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3] + [u'startups_and_entrepreneurship', u'investment', u'business'] + + >>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1 + {u'startups_and_entrepreneurship': 0.21888586688354486} + + >>> text_tags(test_text, top_n=1) # return only keys with top_n values + {u'startups_and_entrepreneurship': 0.21888586688354486} + + >>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() + + >>> fer(test_face) + {u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} + + >>> facial_features(test_face) + [0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777] + + >>> language_dict = language('Quis custodiet ipsos custodes') + + >>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5] + [u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish'] + + >>> language_dict + {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} + + + Batch API Access + ---------------- + + If you'd like to use our batch api interface, please send an email to contact@indico.io. + + from indicio import batch\_sentiment batch\_sentiment(['Text + to analyze', 'More text'], auth=("example@example.com", + "\*\*\*\*\*\*\*\*")) + +:: + + + Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc + + Private Cloud API Access + ------------------------ + + If you'd like to use our private cloud interface, please send an email to contact@indico.io. + + from indicio import sentiment sentiment("Text to analyze", + hostname="http://exampleprivatecloud.io/", + auth=("example@example.com", "\*\*\*\*\*\*\*\*")) + +:: + + + Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc + + indicorc + ------------------------ + + Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. + + Here is an example of a valid indicorc file: + +[auth] username = test@example.com password = secret + +[private\_cloud] hostname = example.indico.io \`\`\` diff --git a/indicoio/config.py b/indicoio/config.py index fa38e85..fefaff1 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,3 +1,5 @@ import os -api_root = os.getenv("INDICO_PRIVATE_CLOUD_URL") or "http://apiv1.indico.io/" +def get_api_root(): + return os.environ.get("INDICO_PRIVATE_CLOUD_URL") or "http://apiv1.indico.io/" +api_root = get_api_root() diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index fd098b2..adace55 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -120,7 +120,7 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs): def image_preprocess(image, batch=False): """ - Takes an image and prepares it for sending to the api including + Takes an image and prepares it for sending to the api including resizing and image data/structure standardizing. """ if batch: @@ -137,4 +137,4 @@ def image_preprocess(image, batch=False): image = image[:,:,:3] image = resize(image,(64,64)) image = image.tolist() - return image \ No newline at end of file + return image diff --git a/tests/remote/test_remote.py b/tests/remote/test_remote.py index bac7fa5..31e8a45 100644 --- a/tests/remote/test_remote.py +++ b/tests/remote/test_remote.py @@ -1,5 +1,6 @@ import unittest import os +from requests import ConnectionError import numpy as np import skimage.io @@ -49,7 +50,7 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(isinstance(response[0], list)) self.assertEqual(len(response[0]), 48) - + def test_batch_image_features_greyscale(self): test_data = [np.random.rand(64, 64).tolist()] response = batch_image_features(test_data, auth=self.auth) @@ -63,13 +64,21 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(isinstance(response[0], list)) self.assertEqual(len(response[0]), 2048) - + def test_batch_language(self): test_data = ['clearly an english sentence'] response = batch_language(test_data, auth=self.auth) self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) + def test_batch_set_url_root(self): + test_data = ['clearly an english sentence'] + self.assertRaises(ConnectionError, + batch_language, + test_data, + auth=self.auth, + url_root='http://not.a.real.url/') + class FullAPIRun(unittest.TestCase): @@ -156,7 +165,7 @@ class FullAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertEqual(len(response), 48) self.check_range(response) - + def test_good_image_features_greyscale(self): test_image = np.random.rand(64, 64).tolist() response = image_features(test_image) @@ -214,6 +223,13 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(language_set, set(language_dict.keys())) assert language_dict['English'] > 0.25 + def test_set_url_root(self): + test_data = 'clearly an english sentence' + self.assertRaises(ConnectionError, + language, + test_data, + url_root='http://not.a.real.url/') + if __name__ == "__main__": unittest.main() diff --git a/tests/remote/test_unit_tests.py b/tests/remote/test_unit_tests.py new file mode 100644 index 0000000..e3f6c1d --- /dev/null +++ b/tests/remote/test_unit_tests.py @@ -0,0 +1,15 @@ +import os + +from indicoio import config + +def test_batch_set_url_root_as_env_var(): + test_data = ['clearly an english sentence'] + old_private_cloud_url = os.environ.get("INDICO_PRIVATE_CLOUD_URL") + os.environ["INDICO_PRIVATE_CLOUD_URL"] = "http://not.a.real.url/" + + assert config.get_api_root() == "http://not.a.real.url/" + + if old_private_cloud_url: + os.environ["INDICO_PRIVATE_CLOUD_URL"] = old_private_cloud_url + else: + del(os.environ["INDICO_PRIVATE_CLOUD_URL"]) From 804bab63c35ba5aa3e0c42cca2a0bf05e053c321 Mon Sep 17 00:00:00 2001 From: Madison May Date: Thu, 26 Feb 2015 18:55:05 -0500 Subject: [PATCH 3/7] Configparser for indico config file --- .indicorc | 3 + indicoio/config.py | 30 +++++- tests/local/__init__.py | 0 tests/local/test_local.py | 156 ------------------------------ tests/remote/__init__.py | 0 tests/{remote => }/test_remote.py | 0 6 files changed, 30 insertions(+), 159 deletions(-) create mode 100644 .indicorc delete mode 100644 tests/local/__init__.py delete mode 100644 tests/local/test_local.py delete mode 100644 tests/remote/__init__.py rename tests/{remote => }/test_remote.py (100%) diff --git a/.indicorc b/.indicorc new file mode 100644 index 0000000..959b888 --- /dev/null +++ b/.indicorc @@ -0,0 +1,3 @@ +[private_cloud] +url_root = 127.0.0.1 + diff --git a/indicoio/config.py b/indicoio/config.py index fefaff1..11da024 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,5 +1,29 @@ import os -def get_api_root(): - return os.environ.get("INDICO_PRIVATE_CLOUD_URL") or "http://apiv1.indico.io/" -api_root = get_api_root() +import ConfigParser + +settings = ConfigParser.ConfigParser() + +settings_paths = [ + os.path.expanduser("~/.indicorc"), + os.path.join(os.getcwd(), '.indicorc') +] + +settings.read(settings_paths) + +def get_section(parser, section): + try: + return dict(parser.items(section)) + except ConfigParser.NoSectionError: + return {} + +auth_settings = get_section(settings, 'auth') +private_cloud_settings = get_section(settings, 'private_cloud') + +api_root = ( + os.getenv("INDICO_PRIVATE_CLOUD_URL") or + private_cloud_settings.get('url_root') or + "http://apiv1.indico.io/" +) + +auth = (auth_settings.get('username'), auth_settings.get('password')) diff --git a/tests/local/__init__.py b/tests/local/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/local/test_local.py b/tests/local/test_local.py deleted file mode 100644 index 9d8f1ba..0000000 --- a/tests/local/test_local.py +++ /dev/null @@ -1,156 +0,0 @@ -import unittest -import os - -import numpy as np - -from indicoio.local import political, sentiment, fer, facial_features, language, image_features, text_tags - -DIR = os.path.dirname(os.path.realpath(__file__)) - - -class FullAPIRun(unittest.TestCase): - - def load_image(self, relpath, as_grey=False): - image_path = os.path.normpath(os.path.join(DIR, relpath)) - image = skimage.io.imread(image_path, as_grey=True).tolist() - return image - - def check_range(self, list, minimum=0.9, maximum=0.1, span=0.5): - vector = np.asarray(list) - self.assertTrue(vector.max() > maximum) - self.assertTrue(vector.min() < minimum) - self.assertTrue(np.ptp(vector) > span) - - def test_text_tags(self): - text = "On Monday, president Barack Obama will be..." - results = text_tags(text) - max_keys = sorted(results.keys(), key=lambda x:results.get(x), reverse=True) - assert 'political_discussion' in max_keys[:5] - results = text_tags(text, top_n=5) - assert len(results) is 5 - results = text_tags(text, threshold=0.1) - for v in results.values(): - assert v >= 0.1 - - def test_political(self): - political_set = set(['Libertarian', 'Liberal', 'Conservative', 'Green']) - test_string = "Guns don't kill people, people kill people." - response = political(test_string) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(political_set, set(response.keys())) - - test_string = "Save the whales" - response = political(test_string) - - self.assertTrue(isinstance(response, dict)) - assert response['Green'] > 0.5 - - def test_posneg(self): - test_string = "Worst song ever." - response = sentiment(test_string) - - self.assertTrue(isinstance(response, float)) - self.assertTrue(response < 0.5) - - test_string = "Best song ever." - response = sentiment(test_string) - self.assertTrue(isinstance(response, float)) - self.assertTrue(response > 0.5) - - def test_good_fer(self): - fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) - test_face = np.random.rand(48,48).tolist() - response = fer(test_face) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(fer_set, set(response.keys())) - - def test_happy_fer(self): - test_face = self.load_image("../data/happy.png", as_grey=True) - response = fer(test_face) - self.assertTrue(isinstance(response, dict)) - self.assertTrue(response['Happy'] > 0.5) - - def test_fear_fer(self): - test_face = self.load_image("../data/fear.png", as_grey=True) - response = fer(test_face) - self.assertTrue(isinstance(response, dict)) - self.assertTrue(response['Fear'] > 0.25) - - def test_bad_fer(self): - fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) - test_face = np.random.rand(56,56).tolist() - response = fer(test_face) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(fer_set, set(response.keys())) - - def test_good_facial_features(self): - test_face = np.random.rand(48,48).tolist() - response = facial_features(test_face) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 48) - self.check_range(response) - - def test_good_image_features_greyscale(self): - test_image = np.random.rand(64, 64).tolist() - response = image_features(test_image) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 2048) - self.check_range(response) - - def test_good_image_features_rgb(self): - test_image = np.random.rand(64, 64, 3).tolist() - response = image_features(test_image) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 2048) - self.check_range(response) - - def test_language(self): - language_set = set([ - 'English', - 'Spanish', - 'Tagalog', - 'Esperanto', - 'French', - 'Chinese', - 'French', - 'Bulgarian', - 'Latin', - 'Slovak', - 'Hebrew', - 'Russian', - 'German', - 'Japanese', - 'Korean', - 'Portuguese', - 'Italian', - 'Polish', - 'Turkish', - 'Dutch', - 'Arabic', - 'Persian (Farsi)', - 'Czech', - 'Swedish', - 'Indonesian', - 'Vietnamese', - 'Romanian', - 'Greek', - 'Danish', - 'Hungarian', - 'Thai', - 'Finnish', - 'Norwegian', - 'Lithuanian' - ]) - language_dict = language('clearly an english sentence') - self.assertEqual(language_set, set(language_dict.keys())) - assert language_dict['English'] > 0.25 - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/remote/__init__.py b/tests/remote/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/remote/test_remote.py b/tests/test_remote.py similarity index 100% rename from tests/remote/test_remote.py rename to tests/test_remote.py From 57f91a138b9fb2d3d259a497e4589071a2e3160f Mon Sep 17 00:00:00 2001 From: Madison May Date: Thu, 26 Feb 2015 19:10:06 -0500 Subject: [PATCH 4/7] Rename parameters for ux reasons + reorganize testing --- tests/{remote/test_unit_tests.py => test_configure.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{remote/test_unit_tests.py => test_configure.py} (100%) diff --git a/tests/remote/test_unit_tests.py b/tests/test_configure.py similarity index 100% rename from tests/remote/test_unit_tests.py rename to tests/test_configure.py From 110abaf7a6ac51c0c37a7b0e0f1ff99ca8f07b37 Mon Sep 17 00:00:00 2001 From: Madison May Date: Thu, 26 Feb 2015 20:45:11 -0500 Subject: [PATCH 5/7] Cleaner api handler interface + extended tests --- .indicorc | 3 - AUTHORS | 1 + CHANGES.txt | 1 + README.md | 29 +++++--- README.rst | 28 ++++--- indicoio/__init__.py | 2 +- indicoio/config.py | 67 +++++++++++------ indicoio/images/features.py | 8 +- indicoio/images/fer.py | 4 +- indicoio/text/lang.py | 4 +- indicoio/text/sentiment.py | 8 +- indicoio/text/tagging.py | 4 +- indicoio/utils/__init__.py | 28 ++++--- setup.py | 4 +- tests/test_configure.py | 141 +++++++++++++++++++++++++++++++++--- tests/test_remote.py | 19 +++-- 16 files changed, 258 insertions(+), 93 deletions(-) delete mode 100644 .indicorc diff --git a/.indicorc b/.indicorc deleted file mode 100644 index 959b888..0000000 --- a/.indicorc +++ /dev/null @@ -1,3 +0,0 @@ -[private_cloud] -url_root = 127.0.0.1 - diff --git a/AUTHORS b/AUTHORS index 2aed9bb..1220f28 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Slater Victoroff Alec Radford Aidan McLaughlin Madison May +Annie Carlson diff --git a/CHANGES.txt b/CHANGES.txt index 8a8c653..1f6e0b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,3 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage v0.4.15, Sat Dec 20 -- Bug fix release +v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls diff --git a/README.md b/README.md index 4452f3c..20f19dc 100644 --- a/README.md +++ b/README.md @@ -86,35 +86,46 @@ If you'd like to use our batch api interface, please send an email to contact@in ``` >>> from indicio import batch_sentiment -batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) +>>> batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) ``` -Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc +Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. -Private Cloud API Access +Private cloud API Access ------------------------ -If you'd like to use our private cloud interface, please send an email to contact@indico.io. +If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. ``` >>> from indicio import sentiment -sentiment("Text to analyze", hostname="http://exampleprivatecloud.io/", auth=("example@example.com", "********")) +>>> sentiment("Text to analyze", cloud="example", auth=("example@example.com", "********")) ``` -Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc +The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. -indicorc +Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + +Configuration ------------------------ -Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. +Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. Here is an example of a valid indicorc file: + ``` [auth] username = test@example.com password = secret [private_cloud] -hostname = example.indico.io +cloud = example ``` + +Environment variables take precedence over any configuration found in the indicorc file. +The following environment variables are valid: + - $INDICO_USERNAME + - $INDICO_PASSWORD + - $INDICO_CLOUD + + Finally, any values explicitly passed in to an api call will override configuration options set in the indicorc file or in an environment variable. diff --git a/README.rst b/README.rst index 4c2c8c9..ff1b9a6 100644 --- a/README.rst +++ b/README.rst @@ -95,29 +95,39 @@ Examples :: - Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc + Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. - Private Cloud API Access + Private cloud API Access ------------------------ - If you'd like to use our private cloud interface, please send an email to contact@indico.io. + If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. from indicio import sentiment sentiment("Text to analyze", - hostname="http://exampleprivatecloud.io/", - auth=("example@example.com", "\*\*\*\*\*\*\*\*")) + cloud="example", auth=("example@example.com", + "\*\*\*\*\*\*\*\*")) :: - Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc + The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. - indicorc + Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + + Configuration ------------------------ - Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. + Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. Here is an example of a valid indicorc file: [auth] username = test@example.com password = secret -[private\_cloud] hostname = example.indico.io \`\`\` +[private\_cloud] cloud = example \`\`\` + +Environment variables take precedence over any configuration found in +the indicorc file. The following environment variables are valid: - +:math:`INDICO_USERNAME - `\ INDICO\_PASSWORD - $INDICO\_CLOUD + +Finally, any values explicitly passed in to an api call will override +configuration options set in the indicorc file or in an environment +variable. diff --git a/indicoio/__init__.py b/indicoio/__init__.py index eaa98b1..62a09e0 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -2,7 +2,7 @@ from functools import partial JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'} -Version, version, __version__, VERSION = ('0.4.15',) * 4 +Version, version, __version__, VERSION = ('0.5.0',) * 4 from indicoio.text.sentiment import political, posneg from indicoio.text.sentiment import posneg as sentiment diff --git a/indicoio/config.py b/indicoio/config.py index 11da024..c639cd1 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,29 +1,54 @@ import os +from StringIO import StringIO import ConfigParser -settings = ConfigParser.ConfigParser() +class Settings(ConfigParser.ConfigParser): -settings_paths = [ + def __init__(self, *args, **kwargs): + """ + files: filepaths or open file objects + """ + self.files = kwargs.pop('files') + + ConfigParser.ConfigParser.__init__(self, *args, **kwargs) + + for fd in self.files: + try: + self.readfp(fd) + except AttributeError: + self.read(fd) + + self.auth_settings = self.get_section('auth') + self.private_cloud_settings = self.get_section('private_cloud') + + def get_section(self, section): + """ + Retrieve a ConfigParser section as a dictionary, default to {} + """ + try: + return dict(self.items(section)) + except ConfigParser.NoSectionError: + return {} + + def cloud(self): + return ( + os.getenv("INDICO_CLOUD") or + self.private_cloud_settings.get('cloud') or + None + ) + + def auth(self): + return ( + os.getenv("INDICO_USERNAME") or self.auth_settings.get('username'), + os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password') + ) + +settings = Settings(files=[ os.path.expanduser("~/.indicorc"), os.path.join(os.getcwd(), '.indicorc') -] +]) -settings.read(settings_paths) - -def get_section(parser, section): - try: - return dict(parser.items(section)) - except ConfigParser.NoSectionError: - return {} - -auth_settings = get_section(settings, 'auth') -private_cloud_settings = get_section(settings, 'private_cloud') - -api_root = ( - os.getenv("INDICO_PRIVATE_CLOUD_URL") or - private_cloud_settings.get('url_root') or - "http://apiv1.indico.io/" -) - -auth = (auth_settings.get('username'), auth_settings.get('password')) +auth = settings.auth() +cloud = settings.cloud() +public_api_host = 'apiv1.indico.io' diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 737309e..e796946 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import image_preprocess, api_handler import indicoio.config as config -def facial_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def facial_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face. Useful as a form of feature engineering for face oriented tasks. @@ -28,9 +28,9 @@ def facial_features(image, url_root=config.api_root, batch=False, auth=None, **k :type image: list of lists :rtype: List containing feature responses """ - return api_handler(image, url_root + "facialfeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs) -def image_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def image_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. Useful as a form of feature engineering for image oriented tasks. @@ -61,4 +61,4 @@ def image_features(image, url_root=config.api_root, batch=False, auth=None, **kw :rtype: List containing features """ image = image_preprocess(image, batch=batch) - return api_handler(image, url_root + "imagefeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 2b9527f..e8378a7 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import api_handler import indicoio.config as config -def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def fer(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given a grayscale input image of a face, returns a probability distribution over emotional state. Input should be in a list of list format, resizing will be attempted internally but for best @@ -30,4 +30,4 @@ def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary containing emotion probability pairs """ - return api_handler(image, url_root + "fer", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="fer", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 8746c7c..86ffa4c 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def language(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 33 possible languages of what language the text was written in. @@ -24,4 +24,4 @@ def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of language probability pairs """ - return api_handler(text, url_root + "language", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="language", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 73e29af..494b40e 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -2,7 +2,7 @@ from indicoio import JSON_HEADERS from indicoio.utils import api_handler import indicoio.config as config -def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def political(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over the political alignment of the speaker. @@ -28,9 +28,9 @@ def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, url_root + "political", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs) -def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def posneg(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a scalar estimate of the sentiment of that text. Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment. @@ -51,4 +51,4 @@ def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Float """ - return api_handler(text, url_root + "sentiment", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="sentiment", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index f820018..8848379 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def text_tags(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 100 document categories @@ -23,4 +23,4 @@ def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of class probability pairs """ - return api_handler(text, url_root + "texttags", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="texttags", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index adace55..40f82ae 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -4,26 +4,22 @@ import numpy as np from skimage.transform import resize from indicoio import JSON_HEADERS +from indicoio import config -def auth_query(): - email = os.environ.get("INDICO_EMAIL") - password = os.environ.get("INDICO_PASSWORD") - # store settings - if not email: - email = raw_input("Email: ") - os.environ["INDICO_EMAIL"] = email - - if not password: - password = getpass.getpass("Password: ") - os.environ["INDICO_PASSWORD"] = password - - return (email, password) - -def api_handler(arg, url, batch=False, auth=None, **kwargs): +def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs): data = {'data': arg} data.update(**kwargs) json_data = json.dumps(data) + + if cloud: + host = "%s.indico.domains" + else: + # default to indico public cloud + host = config.public_api_host + + url = "http://%s/%s" % (host, api) + if batch: url += "/batch" @@ -34,6 +30,7 @@ def api_handler(arg, url, batch=False, auth=None, **kwargs): raise ValueError(error) return results + class TypeCheck(object): """ Decorator that performs a typecheck on the input to a function @@ -118,6 +115,7 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs): return dict(zip(keys, norm_array)) return norm_array + def image_preprocess(image, batch=False): """ Takes an image and prepares it for sending to the api including diff --git a/setup.py b/setup.py index 743dda2..d432a8b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: setup( name="IndicoIo", - version='0.4.15', + version='0.5.0', packages=[ "indicoio", "indicoio.text", @@ -21,7 +21,7 @@ setup( Use pre-built state of the art machine learning algorithms with a single line of code. """, license="MIT License (See LICENSE)", - long_description=open("README").read(), + long_description=open("README.rst").read(), url="https://github.com/IndicoDataSolutions/indicoio-python", author="Alec Radford, Slater Victoroff, Aidan McLaughlin", author_email=""" diff --git a/tests/test_configure.py b/tests/test_configure.py index e3f6c1d..a992151 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -1,15 +1,138 @@ import os +import unittest +import textwrap +from StringIO import StringIO from indicoio import config +from indicoio.config import Settings -def test_batch_set_url_root_as_env_var(): - test_data = ['clearly an english sentence'] - old_private_cloud_url = os.environ.get("INDICO_PRIVATE_CLOUD_URL") - os.environ["INDICO_PRIVATE_CLOUD_URL"] = "http://not.a.real.url/" - assert config.get_api_root() == "http://not.a.real.url/" +class TestConfigureEnv(unittest.TestCase): - if old_private_cloud_url: - os.environ["INDICO_PRIVATE_CLOUD_URL"] = old_private_cloud_url - else: - del(os.environ["INDICO_PRIVATE_CLOUD_URL"]) + def setUp(self): + os.environ = {} + + def test_set_cloud_from_env_var(self): + cloud = "invalid/cloud" + os.environ["INDICO_CLOUD"] = cloud + assert config.settings.cloud() == cloud + + def test_set_auth_from_env_var(self): + username = "test" + password = "password" + os.environ["INDICO_USERNAME"] = username + os.environ["INDICO_PASSWORD"] = password + assert config.settings.auth() == (username, password) + + +class TestConfigurationFile(unittest.TestCase): + + def setUp(self): + self.username = "test" + self.password = "password" + self.cloud = "localhost" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.username, self.password, self.cloud) + + config_file = StringIO(textwrap.dedent(config)) + self.settings = Settings(files=[config_file]) + os.environ = {} + + def test_set_cloud_from_config_file(self): + assert self.settings.cloud() == self.cloud + + def test_set_auth_from_config_file(self): + assert self.settings.auth() == (self.username, self.password) + + +class TestPrecedence(unittest.TestCase): + + def setUp(self): + self.file_username = "file-username" + self.file_password = "file-password" + self.file_cloud = "file-cloud" + + self.env_username = "env-username" + self.env_password = "env-password" + self.env_cloud = "env-cloud" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.file_username, self.file_password, self.file_cloud) + + config_file = StringIO(textwrap.dedent(config)) + os.environ = { + 'INDICO_CLOUD': self.env_cloud, + 'INDICO_USERNAME': self.env_username, + 'INDICO_PASSWORD': self.env_password + } + self.settings = Settings(files=[config_file]) + + def test_set_cloud_from_config_file(self): + assert self.settings.cloud() == self.env_cloud + + def test_set_auth_from_config_file(self): + assert self.settings.auth() == (self.env_username, self.env_password) + + +class TestConfigFilePrecedence(unittest.TestCase): + + def setUp(self): + self.high_priority_username = "high-priority-username" + self.high_priority_password = "high-priority-password" + self.high_priority_cloud = "high-priority-cloud" + + self.low_priority_username = "low-priority-username" + self.low_priority_password = "low-priority-password" + self.low_priority_cloud = "low-priority-cloud" + + high_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.high_priority_username, + self.high_priority_password, + self.high_priority_cloud + ) + + low_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.low_priority_username, + self.low_priority_password, + self.low_priority_cloud + ) + + high_priority_config_file = StringIO(textwrap.dedent(high_priority_config)) + low_priority_config_file = StringIO(textwrap.dedent(low_priority_config)) + + os.environ = {} + self.settings = Settings(files=[ + low_priority_config_file, + high_priority_config_file + ]) + + def test_cloud_config_file_priority(self): + assert self.settings.cloud() == self.high_priority_cloud + + def test_auth_config_file_priority(self): + assert self.settings.auth() == (self.high_priority_username, self.high_priority_password) diff --git a/tests/test_remote.py b/tests/test_remote.py index 31e8a45..44e31bd 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -6,6 +6,7 @@ import numpy as np import skimage.io from nose.plugins.skip import Skip, SkipTest +from indicoio import config from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags @@ -15,11 +16,9 @@ DIR = os.path.dirname(os.path.realpath(__file__)) class BatchAPIRun(unittest.TestCase): def setUp(self): - self.username = os.getenv("INDICO_USERNAME") - self.password = os.getenv("INDICO_PASSWORD") - self.auth = (self.username, self.password) + self.auth = config.auth - if not self.username or not self.password: + if not all(self.auth): raise SkipTest def test_batch_texttags(self): @@ -71,13 +70,13 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) - def test_batch_set_url_root(self): + def test_batch_set_cloud(self): test_data = ['clearly an english sentence'] self.assertRaises(ConnectionError, batch_language, test_data, auth=self.auth, - url_root='http://not.a.real.url/') + cloud='invalid/cloud') class FullAPIRun(unittest.TestCase): @@ -139,13 +138,13 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(fer_set, set(response.keys())) def test_happy_fer(self): - test_face = self.load_image("../data/happy.png", as_grey=True) + test_face = self.load_image("data/happy.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Happy'] > 0.5) def test_fear_fer(self): - test_face = self.load_image("../data/fear.png", as_grey=True) + test_face = self.load_image("data/fear.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Fear'] > 0.25) @@ -223,12 +222,12 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(language_set, set(language_dict.keys())) assert language_dict['English'] > 0.25 - def test_set_url_root(self): + def test_set_cloud(self): test_data = 'clearly an english sentence' self.assertRaises(ConnectionError, language, test_data, - url_root='http://not.a.real.url/') + cloud='invalid/cloud') if __name__ == "__main__": From e9fc025e9548715eac20a4711d4cdca09339423f Mon Sep 17 00:00:00 2001 From: Madison May Date: Fri, 27 Feb 2015 15:14:42 -0500 Subject: [PATCH 6/7] Ensuring calls default to the values found by the Settings object in indicoio.config --- CHANGES.txt | 2 +- indicoio/__init__.py | 12 +++++++++-- indicoio/config.py | 8 ++++---- indicoio/images/features.py | 4 ++-- indicoio/images/fer.py | 2 +- indicoio/text/lang.py | 2 +- indicoio/text/sentiment.py | 4 ++-- indicoio/text/tagging.py | 2 +- indicoio/utils/__init__.py | 8 +++++--- tests/test_configure.py | 41 +++++++++++++++++++++++++++++++++++-- tests/test_remote.py | 2 +- 11 files changed, 67 insertions(+), 20 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1f6e0b0..7984ade 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,4 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage v0.4.15, Sat Dec 20 -- Bug fix release -v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls +v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls, README updates diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 62a09e0..58e53de 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -12,8 +12,16 @@ from indicoio.images.fer import fer from indicoio.images.features import facial_features from indicoio.images.features import image_features -apis = ['political', 'posneg', 'sentiment', 'language', 'fer', - 'facial_features', 'image_features', 'text_tags'] +apis = [ + 'political', + 'posneg', + 'sentiment', + 'language', + 'fer', + 'facial_features', + 'image_features', + 'text_tags' +] apis = dict((api, globals().get(api)) for api in apis) for api in apis: diff --git a/indicoio/config.py b/indicoio/config.py index c639cd1..5633302 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -44,11 +44,11 @@ class Settings(ConfigParser.ConfigParser): os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password') ) -settings = Settings(files=[ +SETTINGS = Settings(files=[ os.path.expanduser("~/.indicorc"), os.path.join(os.getcwd(), '.indicorc') ]) -auth = settings.auth() -cloud = settings.cloud() -public_api_host = 'apiv1.indico.io' +AUTH = SETTINGS.auth() +CLOUD = SETTINGS.cloud() +PUBLIC_API_HOST = 'apiv1.indico.io' diff --git a/indicoio/images/features.py b/indicoio/images/features.py index e796946..ea9608b 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import image_preprocess, api_handler import indicoio.config as config -def facial_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): +def facial_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face. Useful as a form of feature engineering for face oriented tasks. @@ -30,7 +30,7 @@ def facial_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs) """ return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs) -def image_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): +def image_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. Useful as a form of feature engineering for image oriented tasks. diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index e8378a7..3365dec 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import api_handler import indicoio.config as config -def fer(image, cloud=config.cloud, batch=False, auth=None, **kwargs): +def fer(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given a grayscale input image of a face, returns a probability distribution over emotional state. Input should be in a list of list format, resizing will be attempted internally but for best diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 86ffa4c..cf6c1e7 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def language(text, cloud=config.cloud, batch=False, auth=None, **kwargs): +def language(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 33 possible languages of what language the text was written in. diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 494b40e..893fcd7 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -2,7 +2,7 @@ from indicoio import JSON_HEADERS from indicoio.utils import api_handler import indicoio.config as config -def political(text, cloud=config.cloud, batch=False, auth=None, **kwargs): +def political(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over the political alignment of the speaker. @@ -30,7 +30,7 @@ def political(text, cloud=config.cloud, batch=False, auth=None, **kwargs): return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs) -def posneg(text, cloud=config.cloud, batch=False, auth=None, **kwargs): +def posneg(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a scalar estimate of the sentiment of that text. Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment. diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 8848379..52500eb 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def text_tags(text, cloud=config.cloud, batch=False, auth=None, **kwargs): +def text_tags(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 100 document categories diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index 40f82ae..85fb81d 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -13,16 +13,18 @@ def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs): json_data = json.dumps(data) if cloud: - host = "%s.indico.domains" + host = "%s.indico.domains" % cloud else: # default to indico public cloud - host = config.public_api_host + host = config.PUBLIC_API_HOST url = "http://%s/%s" % (host, api) - if batch: url += "/batch" + if not auth: + auth = config.AUTH + response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth).json() results = response.get('results', False) if results is False: diff --git a/tests/test_configure.py b/tests/test_configure.py index a992151..9a11cb3 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -8,24 +8,36 @@ from indicoio.config import Settings class TestConfigureEnv(unittest.TestCase): + """ + Ensure that environment variables are handled by the `Settings` parser + """ def setUp(self): os.environ = {} def test_set_cloud_from_env_var(self): + """ + Ensure cloud hostname is read in from environment variables + """ cloud = "invalid/cloud" os.environ["INDICO_CLOUD"] = cloud - assert config.settings.cloud() == cloud + assert config.SETTINGS.cloud() == cloud def test_set_auth_from_env_var(self): + """ + Ensure cloud authentication credentials are read in from environment variables + """ username = "test" password = "password" os.environ["INDICO_USERNAME"] = username os.environ["INDICO_PASSWORD"] = password - assert config.settings.auth() == (username, password) + assert config.SETTINGS.auth() == (username, password) class TestConfigurationFile(unittest.TestCase): + """ + Ensure that the `Settings` parser reads in configuration files properly + """ def setUp(self): self.username = "test" @@ -45,13 +57,22 @@ class TestConfigurationFile(unittest.TestCase): os.environ = {} def test_set_cloud_from_config_file(self): + """ + Ensure cloud hostname is read in from file + """ assert self.settings.cloud() == self.cloud def test_set_auth_from_config_file(self): + """ + Ensure cloud authentication credentials are read in from file + """ assert self.settings.auth() == (self.username, self.password) class TestPrecedence(unittest.TestCase): + """ + Ensure that environment variables take precedence to config files + """ def setUp(self): self.file_username = "file-username" @@ -79,13 +100,23 @@ class TestPrecedence(unittest.TestCase): self.settings = Settings(files=[config_file]) def test_set_cloud_from_config_file(self): + """ + Ensure cloud hosts set in environment variables are used over those in config files + """ assert self.settings.cloud() == self.env_cloud def test_set_auth_from_config_file(self): + """ + Ensure cloud authentication credentials set in environment variables + are used over those in config files + """ assert self.settings.auth() == (self.env_username, self.env_password) class TestConfigFilePrecedence(unittest.TestCase): + """ + Ensure that files passed in to a `Settings` object are assigned proper priority + """ def setUp(self): self.high_priority_username = "high-priority-username" @@ -132,7 +163,13 @@ class TestConfigFilePrecedence(unittest.TestCase): ]) def test_cloud_config_file_priority(self): + """ + Ensure the cloud subdomain priority is handled properly + """ assert self.settings.cloud() == self.high_priority_cloud def test_auth_config_file_priority(self): + """ + Ensure the cloud auth priority is handled properly + """ assert self.settings.auth() == (self.high_priority_username, self.high_priority_password) diff --git a/tests/test_remote.py b/tests/test_remote.py index 44e31bd..0c9a218 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -16,7 +16,7 @@ DIR = os.path.dirname(os.path.realpath(__file__)) class BatchAPIRun(unittest.TestCase): def setUp(self): - self.auth = config.auth + self.auth = config.AUTH if not all(self.auth): raise SkipTest From 6dd0738a8b6cdfe96009195ac007ffa169e9bf81 Mon Sep 17 00:00:00 2001 From: Madison May Date: Fri, 27 Feb 2015 16:12:16 -0500 Subject: [PATCH 7/7] Raise clear exception when a given private cloud does not support a given api --- indicoio/utils/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index 85fb81d..d9ad14b 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -25,10 +25,14 @@ def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs): if not auth: auth = config.AUTH - response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth).json() - results = response.get('results', False) + response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth) + if response.status_code == 503 and cloud != None: + raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api)) + + json_results = response.json() + results = json_results.get('results', False) if results is False: - error = response.get('error') + error = json_results.get('error') raise ValueError(error) return results