diff --git a/AUTHORS b/AUTHORS index 2aed9bb..1220f28 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Slater Victoroff Alec Radford Aidan McLaughlin Madison May +Annie Carlson diff --git a/CHANGES.txt b/CHANGES.txt index 8a8c653..7984ade 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,3 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage v0.4.15, Sat Dec 20 -- Bug fix release +v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls, README updates diff --git a/README b/README deleted file mode 100644 index 9cd838f..0000000 --- a/README +++ /dev/null @@ -1,99 +0,0 @@ -indicoio-python -=============== - -A wrapper for a series of APIs made by indico. - -Check out the main site on: - -http://indico.io - -Check out our documentation on: - -http://indicoiopython.s3-website-us-west-2.amazonaws.com/indicoio.html - -Our APIs are totally free to use, and ready to be used in your application. No data or training required. - -Current APIs ------------- - -Right now this wrapper supports the following apps: - -- Positive/Negative Sentiment Analysis -- Political Sentiment Analysis -- Image Feature Extraction -- Facial Emotion Recognition -- Facial Feature Extraction -- Language Detection -- Text Topic Tagging - -Examples --------- -``` ->>> import numpy as np - ->>> from indicoio import political, sentiment, fer, facial_features, language - ->>> political("Guns don't kill people. People kill people.") -{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} - ->>> sentiment('Worst movie ever.') -{u'Sentiment': 0.07062467665597527} - ->>> sentiment('Really enjoyed the movie.') -{u'Sentiment': 0.8105182526856075} - ->>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets." - ->>> tag_dict = text_tags(test_text) - ->>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3] -[u'startups_and_entrepreneurship', u'investment', u'business'] - ->>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1 -{u'startups_and_entrepreneurship': 0.21888586688354486} - ->>> text_tags(test_text, top_n=1) # return only keys with top_n values -{u'startups_and_entrepreneurship': 0.21888586688354486} - ->>> tag_dict -{u'fashion': 0.011450126534350728, u'art': 0.00358698972755963, u'energy': 0.005537894035625527, ...} - ->>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() - ->>> fer(test_face) -{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} - ->>> facial_features(test_face) -[0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777] - ->>> language_dict = language('Quis custodiet ipsos custodes') - ->>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5] -[u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish'] - ->>> language_dict -{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} - -``` - -If you have a local indico server running, simply import from `indicoio.local`. - -``` ->>> from indicoio.local import political, sentiment, fer, facial_features, language -``` - -If you'd like to use our batch api interface, please send an email to contact@indico.io. - -``` ->>> from indicio import batch_sentiment -batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) -``` - - -Installation ------------- -``` -pip install indicoio -``` - -Announcement: Indico has partnered with Experfy, a data science consulting marketplace based in the Harvard Innovation Lab. Through Experfy, we are helping our data science community members find lucrative projects and advance their skills. Please signup for Experfy at https://www.experfy.com/ to get started. \ No newline at end of file diff --git a/README.md b/README.md index 98c11ef..20f19dc 100644 --- a/README.md +++ b/README.md @@ -78,13 +78,6 @@ Examples >>> language_dict {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} -``` - -If you have a local indico server running, simply import from `indicoio.local`. - -``` ->>> from indicoio.local import political, sentiment, fer, facial_features, language -``` Batch API Access ---------------- @@ -93,5 +86,46 @@ If you'd like to use our batch api interface, please send an email to contact@in ``` >>> from indicio import batch_sentiment -batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) +>>> batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) ``` + +Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. + +Private cloud API Access +------------------------ + +If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. + +``` +>>> from indicio import sentiment +>>> sentiment("Text to analyze", cloud="example", auth=("example@example.com", "********")) +``` + +The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. + +Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + +Configuration +------------------------ + +Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. + +Here is an example of a valid indicorc file: + + +``` +[auth] +username = test@example.com +password = secret + +[private_cloud] +cloud = example +``` + +Environment variables take precedence over any configuration found in the indicorc file. +The following environment variables are valid: + - $INDICO_USERNAME + - $INDICO_PASSWORD + - $INDICO_CLOUD + + Finally, any values explicitly passed in to an api call will override configuration options set in the indicorc file or in an environment variable. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..ff1b9a6 --- /dev/null +++ b/README.rst @@ -0,0 +1,133 @@ +indicoio-python +=============== + +A wrapper for a series of APIs made by indico. + +Check out the main site on: + +http://indico.io + +Our APIs are totally free to use, and ready to be used in your +application. No data or training required. + +Installation +------------ + +:: + + pip install indicoio + +Documentation +------------- + +Available at `indico.reame.io `__ + +Current APIs +------------ + +Right now this wrapper supports the following apps: + +- Positive/Negative Sentiment Analysis +- Political Sentiment Analysis +- Image Feature Extraction +- Facial Emotion Recognition +- Facial Feature Extraction +- Language Detection +- Text Topic Tagging + +Examples +-------- + +:: + + >>> import numpy as np + + >>> from indicoio import political, sentiment, fer, facial_features, language + + >>> political("Guns don't kill people. People kill people.") + {u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146} + + >>> sentiment('Worst movie ever.') + {u'Sentiment': 0.07062467665597527} + + >>> sentiment('Really enjoyed the movie.') + {u'Sentiment': 0.8105182526856075} + + >>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets." + + >>> tag_dict = text_tags(test_text) + + >>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3] + [u'startups_and_entrepreneurship', u'investment', u'business'] + + >>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1 + {u'startups_and_entrepreneurship': 0.21888586688354486} + + >>> text_tags(test_text, top_n=1) # return only keys with top_n values + {u'startups_and_entrepreneurship': 0.21888586688354486} + + >>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist() + + >>> fer(test_face) + {u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382} + + >>> facial_features(test_face) + [0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777] + + >>> language_dict = language('Quis custodiet ipsos custodes') + + >>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5] + [u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish'] + + >>> language_dict + {u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...} + + + Batch API Access + ---------------- + + If you'd like to use our batch api interface, please send an email to contact@indico.io. + + from indicio import batch\_sentiment batch\_sentiment(['Text + to analyze', 'More text'], auth=("example@example.com", + "\*\*\*\*\*\*\*\*")) + +:: + + + Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. + + Private cloud API Access + ------------------------ + + If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. + + from indicio import sentiment sentiment("Text to analyze", + cloud="example", auth=("example@example.com", + "\*\*\*\*\*\*\*\*")) + +:: + + + The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. + + Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + + Configuration + ------------------------ + + Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. + + Here is an example of a valid indicorc file: + +[auth] username = test@example.com password = secret + +[private\_cloud] cloud = example \`\`\` + +Environment variables take precedence over any configuration found in +the indicorc file. The following environment variables are valid: - +:math:`INDICO_USERNAME - `\ INDICO\_PASSWORD - $INDICO\_CLOUD + +Finally, any values explicitly passed in to an api call will override +configuration options set in the indicorc file or in an environment +variable. diff --git a/indicoio/__init__.py b/indicoio/__init__.py index 0921836..58e53de 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -1,9 +1,8 @@ from functools import partial -import indicoio.config as config JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'} -Version, version, __version__, VERSION = ('0.4.15',) * 4 +Version, version, __version__, VERSION = ('0.5.0',) * 4 from indicoio.text.sentiment import political, posneg from indicoio.text.sentiment import posneg as sentiment @@ -13,14 +12,18 @@ from indicoio.images.fer import fer from indicoio.images.features import facial_features from indicoio.images.features import image_features -apis = ['political', 'posneg', 'sentiment', 'language', 'fer', - 'facial_features', 'image_features', 'text_tags'] +apis = [ + 'political', + 'posneg', + 'sentiment', + 'language', + 'fer', + 'facial_features', + 'image_features', + 'text_tags' +] apis = dict((api, globals().get(api)) for api in apis) -class Namespace(object): pass -local = Namespace() for api in apis: - globals()[api] = partial(apis[api], config.api_root) - globals()['batch_' + api] = partial(apis[api], config.api_root, batch=True) - setattr(local, api, partial(apis[api], config.local_api_root)) - setattr(local, 'batch_' + api, partial(apis[api], config.local_api_root, batch=True)) + globals()[api] = partial(apis[api]) + globals()['batch_' + api] = partial(apis[api], batch=True) diff --git a/indicoio/config.py b/indicoio/config.py index 150d0f9..5633302 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,2 +1,54 @@ -local_api_root = "http://localhost:9438/" -api_root = "http://apiv1.indico.io/" \ No newline at end of file +import os +from StringIO import StringIO + +import ConfigParser + +class Settings(ConfigParser.ConfigParser): + + def __init__(self, *args, **kwargs): + """ + files: filepaths or open file objects + """ + self.files = kwargs.pop('files') + + ConfigParser.ConfigParser.__init__(self, *args, **kwargs) + + for fd in self.files: + try: + self.readfp(fd) + except AttributeError: + self.read(fd) + + self.auth_settings = self.get_section('auth') + self.private_cloud_settings = self.get_section('private_cloud') + + def get_section(self, section): + """ + Retrieve a ConfigParser section as a dictionary, default to {} + """ + try: + return dict(self.items(section)) + except ConfigParser.NoSectionError: + return {} + + def cloud(self): + return ( + os.getenv("INDICO_CLOUD") or + self.private_cloud_settings.get('cloud') or + None + ) + + def auth(self): + return ( + os.getenv("INDICO_USERNAME") or self.auth_settings.get('username'), + os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password') + ) + +SETTINGS = Settings(files=[ + os.path.expanduser("~/.indicorc"), + os.path.join(os.getcwd(), '.indicorc') +]) + +AUTH = SETTINGS.auth() +CLOUD = SETTINGS.cloud() +PUBLIC_API_HOST = 'apiv1.indico.io' diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 8ed7903..ea9608b 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -4,12 +4,13 @@ import requests import numpy as np from indicoio.utils import image_preprocess, api_handler +import indicoio.config as config -def facial_features(api_root, image, batch=False, auth=None, **kwargs): +def facial_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face. Useful as a form of feature engineering for face oriented tasks. - Input should be in a list of list format, resizing will be attempted internally but for best + Input should be in a list of list format, resizing will be attempted internally but for best performance, images should be already sized at 48x48 pixels. Example usage: @@ -27,18 +28,18 @@ def facial_features(api_root, image, batch=False, auth=None, **kwargs): :type image: list of lists :rtype: List containing feature responses """ - return api_handler(image, api_root + "facialfeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs) -def image_features(api_root, image, batch=False, auth=None, **kwargs): +def image_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ - Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. + Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. Useful as a form of feature engineering for image oriented tasks. * Input can be either grayscale or rgb color and should either be a numpy array or nested list format. * Input data should be either uint8 0-255 range values or floating point between 0 and 1. * Large images (i.e. 1024x768+) are much bigger than needed, resizing will be done internally to 64x64 if needed. * For ideal performance, images should be square aspect ratio but non-square aspect ratios are supported as well. - + Example usage: .. code-block:: python @@ -60,4 +61,4 @@ def image_features(api_root, image, batch=False, auth=None, **kwargs): :rtype: List containing features """ image = image_preprocess(image, batch=batch) - return api_handler(image, api_root + "imagefeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 2248fe8..3365dec 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -2,12 +2,14 @@ import json import requests import numpy as np -from indicoio.utils import api_handler -def fer(api_root, image, batch=False, auth=None, **kwargs): +from indicoio.utils import api_handler +import indicoio.config as config + +def fer(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given a grayscale input image of a face, returns a probability distribution over emotional state. - Input should be in a list of list format, resizing will be attempted internally but for best + Input should be in a list of list format, resizing will be attempted internally but for best performance, images should be already sized at 48x48 pixels.. Example usage: @@ -19,13 +21,13 @@ def fer(api_root, image, batch=False, auth=None, **kwargs): >>> face = np.zeros((48,48)).tolist() >>> emotions = fer(face) >>> emotions - {u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839, - u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344, + {u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839, + u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344, u'Fear': 0.08523385724298838, u'Happy': 0.04117802220878012} :param image: The image to be analyzed. :type image: list of lists :rtype: Dictionary containing emotion probability pairs """ - - return api_handler(image, api_root + "fer", batch=batch, auth=auth, **kwargs) + + return api_handler(image, cloud=cloud, api="fer", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 51bd339..cf6c1e7 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,8 +1,9 @@ from indicoio.utils import api_handler +import indicoio.config as config -def language(api_root, text, batch=False, auth=None, **kwargs): +def language(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ - Given input text, returns a probability distribution over 33 possible + Given input text, returns a probability distribution over 33 possible languages of what language the text was written in. Example usage: @@ -22,5 +23,5 @@ def language(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Dictionary of language probability pairs """ - - return api_handler(text, api_root + "language", batch=batch, auth=auth, **kwargs) + + return api_handler(text, cloud=cloud, api="language", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 387bb46..893fcd7 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -1,7 +1,8 @@ from indicoio import JSON_HEADERS from indicoio.utils import api_handler +import indicoio.config as config -def political(api_root, text, batch=False, auth=None, **kwargs): +def political(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over the political alignment of the speaker. @@ -15,7 +16,7 @@ def political(api_root, text, batch=False, auth=None, **kwargs): Hopefully, driverless cars will chance economics from ownership to fee for service.' >>> affiliation = political(text) >>> affiliation - {u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122, + {u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122, u'Liberal': 0.13730032938784784, u'Conservative': 0.07287981563170784} >>> least_like = affiliation.keys()[np.argmin(affiliation.values())] >>> most_like = affiliation.keys()[np.argmax(affiliation.values())] @@ -27,9 +28,9 @@ def political(api_root, text, batch=False, auth=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, api_root + "political", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs) -def posneg(api_root, text, batch=False, auth=None, **kwargs): +def posneg(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a scalar estimate of the sentiment of that text. Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment. @@ -49,5 +50,5 @@ def posneg(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Float """ - - return api_handler(text, api_root + "sentiment", batch=batch, auth=auth, **kwargs) + + return api_handler(text, cloud=cloud, api="sentiment", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index 6fabd7b..52500eb 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,6 +1,7 @@ from indicoio.utils import api_handler +import indicoio.config as config -def text_tags(api_root, text, batch=False, auth=None, **kwargs): +def text_tags(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 100 document categories @@ -21,5 +22,5 @@ def text_tags(api_root, text, batch=False, auth=None, **kwargs): :type text: str or unicode :rtype: Dictionary of class probability pairs """ - - return api_handler(text, api_root + "texttags", batch=batch, auth=auth, **kwargs) + + return api_handler(text, cloud=cloud, api="texttags", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index fd098b2..d9ad14b 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -4,36 +4,39 @@ import numpy as np from skimage.transform import resize from indicoio import JSON_HEADERS +from indicoio import config -def auth_query(): - email = os.environ.get("INDICO_EMAIL") - password = os.environ.get("INDICO_PASSWORD") - # store settings - if not email: - email = raw_input("Email: ") - os.environ["INDICO_EMAIL"] = email - - if not password: - password = getpass.getpass("Password: ") - os.environ["INDICO_PASSWORD"] = password - - return (email, password) - -def api_handler(arg, url, batch=False, auth=None, **kwargs): +def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs): data = {'data': arg} data.update(**kwargs) json_data = json.dumps(data) + + if cloud: + host = "%s.indico.domains" % cloud + else: + # default to indico public cloud + host = config.PUBLIC_API_HOST + + url = "http://%s/%s" % (host, api) if batch: url += "/batch" - response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth).json() - results = response.get('results', False) + if not auth: + auth = config.AUTH + + response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth) + if response.status_code == 503 and cloud != None: + raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api)) + + json_results = response.json() + results = json_results.get('results', False) if results is False: - error = response.get('error') + error = json_results.get('error') raise ValueError(error) return results + class TypeCheck(object): """ Decorator that performs a typecheck on the input to a function @@ -118,9 +121,10 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs): return dict(zip(keys, norm_array)) return norm_array + def image_preprocess(image, batch=False): """ - Takes an image and prepares it for sending to the api including + Takes an image and prepares it for sending to the api including resizing and image data/structure standardizing. """ if batch: @@ -137,4 +141,4 @@ def image_preprocess(image, batch=False): image = image[:,:,:3] image = resize(image,(64,64)) image = image.tolist() - return image \ No newline at end of file + return image diff --git a/setup.py b/setup.py index 743dda2..d432a8b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: setup( name="IndicoIo", - version='0.4.15', + version='0.5.0', packages=[ "indicoio", "indicoio.text", @@ -21,7 +21,7 @@ setup( Use pre-built state of the art machine learning algorithms with a single line of code. """, license="MIT License (See LICENSE)", - long_description=open("README").read(), + long_description=open("README.rst").read(), url="https://github.com/IndicoDataSolutions/indicoio-python", author="Alec Radford, Slater Victoroff, Aidan McLaughlin", author_email=""" diff --git a/tests/local/__init__.py b/tests/local/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/local/test_local.py b/tests/local/test_local.py deleted file mode 100644 index 9d8f1ba..0000000 --- a/tests/local/test_local.py +++ /dev/null @@ -1,156 +0,0 @@ -import unittest -import os - -import numpy as np - -from indicoio.local import political, sentiment, fer, facial_features, language, image_features, text_tags - -DIR = os.path.dirname(os.path.realpath(__file__)) - - -class FullAPIRun(unittest.TestCase): - - def load_image(self, relpath, as_grey=False): - image_path = os.path.normpath(os.path.join(DIR, relpath)) - image = skimage.io.imread(image_path, as_grey=True).tolist() - return image - - def check_range(self, list, minimum=0.9, maximum=0.1, span=0.5): - vector = np.asarray(list) - self.assertTrue(vector.max() > maximum) - self.assertTrue(vector.min() < minimum) - self.assertTrue(np.ptp(vector) > span) - - def test_text_tags(self): - text = "On Monday, president Barack Obama will be..." - results = text_tags(text) - max_keys = sorted(results.keys(), key=lambda x:results.get(x), reverse=True) - assert 'political_discussion' in max_keys[:5] - results = text_tags(text, top_n=5) - assert len(results) is 5 - results = text_tags(text, threshold=0.1) - for v in results.values(): - assert v >= 0.1 - - def test_political(self): - political_set = set(['Libertarian', 'Liberal', 'Conservative', 'Green']) - test_string = "Guns don't kill people, people kill people." - response = political(test_string) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(political_set, set(response.keys())) - - test_string = "Save the whales" - response = political(test_string) - - self.assertTrue(isinstance(response, dict)) - assert response['Green'] > 0.5 - - def test_posneg(self): - test_string = "Worst song ever." - response = sentiment(test_string) - - self.assertTrue(isinstance(response, float)) - self.assertTrue(response < 0.5) - - test_string = "Best song ever." - response = sentiment(test_string) - self.assertTrue(isinstance(response, float)) - self.assertTrue(response > 0.5) - - def test_good_fer(self): - fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) - test_face = np.random.rand(48,48).tolist() - response = fer(test_face) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(fer_set, set(response.keys())) - - def test_happy_fer(self): - test_face = self.load_image("../data/happy.png", as_grey=True) - response = fer(test_face) - self.assertTrue(isinstance(response, dict)) - self.assertTrue(response['Happy'] > 0.5) - - def test_fear_fer(self): - test_face = self.load_image("../data/fear.png", as_grey=True) - response = fer(test_face) - self.assertTrue(isinstance(response, dict)) - self.assertTrue(response['Fear'] > 0.25) - - def test_bad_fer(self): - fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy']) - test_face = np.random.rand(56,56).tolist() - response = fer(test_face) - - self.assertTrue(isinstance(response, dict)) - self.assertEqual(fer_set, set(response.keys())) - - def test_good_facial_features(self): - test_face = np.random.rand(48,48).tolist() - response = facial_features(test_face) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 48) - self.check_range(response) - - def test_good_image_features_greyscale(self): - test_image = np.random.rand(64, 64).tolist() - response = image_features(test_image) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 2048) - self.check_range(response) - - def test_good_image_features_rgb(self): - test_image = np.random.rand(64, 64, 3).tolist() - response = image_features(test_image) - - self.assertTrue(isinstance(response, list)) - self.assertEqual(len(response), 2048) - self.check_range(response) - - def test_language(self): - language_set = set([ - 'English', - 'Spanish', - 'Tagalog', - 'Esperanto', - 'French', - 'Chinese', - 'French', - 'Bulgarian', - 'Latin', - 'Slovak', - 'Hebrew', - 'Russian', - 'German', - 'Japanese', - 'Korean', - 'Portuguese', - 'Italian', - 'Polish', - 'Turkish', - 'Dutch', - 'Arabic', - 'Persian (Farsi)', - 'Czech', - 'Swedish', - 'Indonesian', - 'Vietnamese', - 'Romanian', - 'Greek', - 'Danish', - 'Hungarian', - 'Thai', - 'Finnish', - 'Norwegian', - 'Lithuanian' - ]) - language_dict = language('clearly an english sentence') - self.assertEqual(language_set, set(language_dict.keys())) - assert language_dict['English'] > 0.25 - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/remote/__init__.py b/tests/remote/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_configure.py b/tests/test_configure.py new file mode 100644 index 0000000..9a11cb3 --- /dev/null +++ b/tests/test_configure.py @@ -0,0 +1,175 @@ +import os +import unittest +import textwrap +from StringIO import StringIO + +from indicoio import config +from indicoio.config import Settings + + +class TestConfigureEnv(unittest.TestCase): + """ + Ensure that environment variables are handled by the `Settings` parser + """ + + def setUp(self): + os.environ = {} + + def test_set_cloud_from_env_var(self): + """ + Ensure cloud hostname is read in from environment variables + """ + cloud = "invalid/cloud" + os.environ["INDICO_CLOUD"] = cloud + assert config.SETTINGS.cloud() == cloud + + def test_set_auth_from_env_var(self): + """ + Ensure cloud authentication credentials are read in from environment variables + """ + username = "test" + password = "password" + os.environ["INDICO_USERNAME"] = username + os.environ["INDICO_PASSWORD"] = password + assert config.SETTINGS.auth() == (username, password) + + +class TestConfigurationFile(unittest.TestCase): + """ + Ensure that the `Settings` parser reads in configuration files properly + """ + + def setUp(self): + self.username = "test" + self.password = "password" + self.cloud = "localhost" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.username, self.password, self.cloud) + + config_file = StringIO(textwrap.dedent(config)) + self.settings = Settings(files=[config_file]) + os.environ = {} + + def test_set_cloud_from_config_file(self): + """ + Ensure cloud hostname is read in from file + """ + assert self.settings.cloud() == self.cloud + + def test_set_auth_from_config_file(self): + """ + Ensure cloud authentication credentials are read in from file + """ + assert self.settings.auth() == (self.username, self.password) + + +class TestPrecedence(unittest.TestCase): + """ + Ensure that environment variables take precedence to config files + """ + + def setUp(self): + self.file_username = "file-username" + self.file_password = "file-password" + self.file_cloud = "file-cloud" + + self.env_username = "env-username" + self.env_password = "env-password" + self.env_cloud = "env-cloud" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.file_username, self.file_password, self.file_cloud) + + config_file = StringIO(textwrap.dedent(config)) + os.environ = { + 'INDICO_CLOUD': self.env_cloud, + 'INDICO_USERNAME': self.env_username, + 'INDICO_PASSWORD': self.env_password + } + self.settings = Settings(files=[config_file]) + + def test_set_cloud_from_config_file(self): + """ + Ensure cloud hosts set in environment variables are used over those in config files + """ + assert self.settings.cloud() == self.env_cloud + + def test_set_auth_from_config_file(self): + """ + Ensure cloud authentication credentials set in environment variables + are used over those in config files + """ + assert self.settings.auth() == (self.env_username, self.env_password) + + +class TestConfigFilePrecedence(unittest.TestCase): + """ + Ensure that files passed in to a `Settings` object are assigned proper priority + """ + + def setUp(self): + self.high_priority_username = "high-priority-username" + self.high_priority_password = "high-priority-password" + self.high_priority_cloud = "high-priority-cloud" + + self.low_priority_username = "low-priority-username" + self.low_priority_password = "low-priority-password" + self.low_priority_cloud = "low-priority-cloud" + + high_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.high_priority_username, + self.high_priority_password, + self.high_priority_cloud + ) + + low_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.low_priority_username, + self.low_priority_password, + self.low_priority_cloud + ) + + high_priority_config_file = StringIO(textwrap.dedent(high_priority_config)) + low_priority_config_file = StringIO(textwrap.dedent(low_priority_config)) + + os.environ = {} + self.settings = Settings(files=[ + low_priority_config_file, + high_priority_config_file + ]) + + def test_cloud_config_file_priority(self): + """ + Ensure the cloud subdomain priority is handled properly + """ + assert self.settings.cloud() == self.high_priority_cloud + + def test_auth_config_file_priority(self): + """ + Ensure the cloud auth priority is handled properly + """ + assert self.settings.auth() == (self.high_priority_username, self.high_priority_password) diff --git a/tests/remote/test_remote.py b/tests/test_remote.py similarity index 90% rename from tests/remote/test_remote.py rename to tests/test_remote.py index bac7fa5..0c9a218 100644 --- a/tests/remote/test_remote.py +++ b/tests/test_remote.py @@ -1,10 +1,12 @@ import unittest import os +from requests import ConnectionError import numpy as np import skimage.io from nose.plugins.skip import Skip, SkipTest +from indicoio import config from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags @@ -14,11 +16,9 @@ DIR = os.path.dirname(os.path.realpath(__file__)) class BatchAPIRun(unittest.TestCase): def setUp(self): - self.username = os.getenv("INDICO_USERNAME") - self.password = os.getenv("INDICO_PASSWORD") - self.auth = (self.username, self.password) + self.auth = config.AUTH - if not self.username or not self.password: + if not all(self.auth): raise SkipTest def test_batch_texttags(self): @@ -49,7 +49,7 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(isinstance(response[0], list)) self.assertEqual(len(response[0]), 48) - + def test_batch_image_features_greyscale(self): test_data = [np.random.rand(64, 64).tolist()] response = batch_image_features(test_data, auth=self.auth) @@ -63,13 +63,21 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(isinstance(response[0], list)) self.assertEqual(len(response[0]), 2048) - + def test_batch_language(self): test_data = ['clearly an english sentence'] response = batch_language(test_data, auth=self.auth) self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) + def test_batch_set_cloud(self): + test_data = ['clearly an english sentence'] + self.assertRaises(ConnectionError, + batch_language, + test_data, + auth=self.auth, + cloud='invalid/cloud') + class FullAPIRun(unittest.TestCase): @@ -130,13 +138,13 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(fer_set, set(response.keys())) def test_happy_fer(self): - test_face = self.load_image("../data/happy.png", as_grey=True) + test_face = self.load_image("data/happy.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Happy'] > 0.5) def test_fear_fer(self): - test_face = self.load_image("../data/fear.png", as_grey=True) + test_face = self.load_image("data/fear.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Fear'] > 0.25) @@ -156,7 +164,7 @@ class FullAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertEqual(len(response), 48) self.check_range(response) - + def test_good_image_features_greyscale(self): test_image = np.random.rand(64, 64).tolist() response = image_features(test_image) @@ -214,6 +222,13 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(language_set, set(language_dict.keys())) assert language_dict['English'] > 0.25 + def test_set_cloud(self): + test_data = 'clearly an english sentence' + self.assertRaises(ConnectionError, + language, + test_data, + cloud='invalid/cloud') + if __name__ == "__main__": unittest.main()