diff --git a/.indicorc b/.indicorc deleted file mode 100644 index 959b888..0000000 --- a/.indicorc +++ /dev/null @@ -1,3 +0,0 @@ -[private_cloud] -url_root = 127.0.0.1 - diff --git a/AUTHORS b/AUTHORS index 2aed9bb..1220f28 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Slater Victoroff Alec Radford Aidan McLaughlin Madison May +Annie Carlson diff --git a/CHANGES.txt b/CHANGES.txt index 8a8c653..1f6e0b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,3 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage v0.4.15, Sat Dec 20 -- Bug fix release +v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls diff --git a/README.md b/README.md index 4452f3c..20f19dc 100644 --- a/README.md +++ b/README.md @@ -86,35 +86,46 @@ If you'd like to use our batch api interface, please send an email to contact@in ``` >>> from indicio import batch_sentiment -batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) +>>> batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********")) ``` -Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc +Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. -Private Cloud API Access +Private cloud API Access ------------------------ -If you'd like to use our private cloud interface, please send an email to contact@indico.io. +If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. ``` >>> from indicio import sentiment -sentiment("Text to analyze", hostname="http://exampleprivatecloud.io/", auth=("example@example.com", "********")) +>>> sentiment("Text to analyze", cloud="example", auth=("example@example.com", "********")) ``` -Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc +The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. -indicorc +Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + +Configuration ------------------------ -Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. +Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. Here is an example of a valid indicorc file: + ``` [auth] username = test@example.com password = secret [private_cloud] -hostname = example.indico.io +cloud = example ``` + +Environment variables take precedence over any configuration found in the indicorc file. +The following environment variables are valid: + - $INDICO_USERNAME + - $INDICO_PASSWORD + - $INDICO_CLOUD + + Finally, any values explicitly passed in to an api call will override configuration options set in the indicorc file or in an environment variable. diff --git a/README.rst b/README.rst index 4c2c8c9..ff1b9a6 100644 --- a/README.rst +++ b/README.rst @@ -95,29 +95,39 @@ Examples :: - Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc + Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file. - Private Cloud API Access + Private cloud API Access ------------------------ - If you'd like to use our private cloud interface, please send an email to contact@indico.io. + If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option. from indicio import sentiment sentiment("Text to analyze", - hostname="http://exampleprivatecloud.io/", - auth=("example@example.com", "\*\*\*\*\*\*\*\*")) + cloud="example", auth=("example@example.com", + "\*\*\*\*\*\*\*\*")) :: - Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc + The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. - indicorc + Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file. + + Configuration ------------------------ - Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional. + Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional. Here is an example of a valid indicorc file: [auth] username = test@example.com password = secret -[private\_cloud] hostname = example.indico.io \`\`\` +[private\_cloud] cloud = example \`\`\` + +Environment variables take precedence over any configuration found in +the indicorc file. The following environment variables are valid: - +:math:`INDICO_USERNAME - `\ INDICO\_PASSWORD - $INDICO\_CLOUD + +Finally, any values explicitly passed in to an api call will override +configuration options set in the indicorc file or in an environment +variable. diff --git a/indicoio/__init__.py b/indicoio/__init__.py index eaa98b1..62a09e0 100644 --- a/indicoio/__init__.py +++ b/indicoio/__init__.py @@ -2,7 +2,7 @@ from functools import partial JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'} -Version, version, __version__, VERSION = ('0.4.15',) * 4 +Version, version, __version__, VERSION = ('0.5.0',) * 4 from indicoio.text.sentiment import political, posneg from indicoio.text.sentiment import posneg as sentiment diff --git a/indicoio/config.py b/indicoio/config.py index 11da024..c639cd1 100644 --- a/indicoio/config.py +++ b/indicoio/config.py @@ -1,29 +1,54 @@ import os +from StringIO import StringIO import ConfigParser -settings = ConfigParser.ConfigParser() +class Settings(ConfigParser.ConfigParser): -settings_paths = [ + def __init__(self, *args, **kwargs): + """ + files: filepaths or open file objects + """ + self.files = kwargs.pop('files') + + ConfigParser.ConfigParser.__init__(self, *args, **kwargs) + + for fd in self.files: + try: + self.readfp(fd) + except AttributeError: + self.read(fd) + + self.auth_settings = self.get_section('auth') + self.private_cloud_settings = self.get_section('private_cloud') + + def get_section(self, section): + """ + Retrieve a ConfigParser section as a dictionary, default to {} + """ + try: + return dict(self.items(section)) + except ConfigParser.NoSectionError: + return {} + + def cloud(self): + return ( + os.getenv("INDICO_CLOUD") or + self.private_cloud_settings.get('cloud') or + None + ) + + def auth(self): + return ( + os.getenv("INDICO_USERNAME") or self.auth_settings.get('username'), + os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password') + ) + +settings = Settings(files=[ os.path.expanduser("~/.indicorc"), os.path.join(os.getcwd(), '.indicorc') -] +]) -settings.read(settings_paths) - -def get_section(parser, section): - try: - return dict(parser.items(section)) - except ConfigParser.NoSectionError: - return {} - -auth_settings = get_section(settings, 'auth') -private_cloud_settings = get_section(settings, 'private_cloud') - -api_root = ( - os.getenv("INDICO_PRIVATE_CLOUD_URL") or - private_cloud_settings.get('url_root') or - "http://apiv1.indico.io/" -) - -auth = (auth_settings.get('username'), auth_settings.get('password')) +auth = settings.auth() +cloud = settings.cloud() +public_api_host = 'apiv1.indico.io' diff --git a/indicoio/images/features.py b/indicoio/images/features.py index 737309e..e796946 100644 --- a/indicoio/images/features.py +++ b/indicoio/images/features.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import image_preprocess, api_handler import indicoio.config as config -def facial_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def facial_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face. Useful as a form of feature engineering for face oriented tasks. @@ -28,9 +28,9 @@ def facial_features(image, url_root=config.api_root, batch=False, auth=None, **k :type image: list of lists :rtype: List containing feature responses """ - return api_handler(image, url_root + "facialfeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs) -def image_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def image_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given an input image, returns a 2048 dimensional sparse feature vector explaining that image. Useful as a form of feature engineering for image oriented tasks. @@ -61,4 +61,4 @@ def image_features(image, url_root=config.api_root, batch=False, auth=None, **kw :rtype: List containing features """ image = image_preprocess(image, batch=batch) - return api_handler(image, url_root + "imagefeatures", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/images/fer.py b/indicoio/images/fer.py index 2b9527f..e8378a7 100644 --- a/indicoio/images/fer.py +++ b/indicoio/images/fer.py @@ -6,7 +6,7 @@ import numpy as np from indicoio.utils import api_handler import indicoio.config as config -def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs): +def fer(image, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given a grayscale input image of a face, returns a probability distribution over emotional state. Input should be in a list of list format, resizing will be attempted internally but for best @@ -30,4 +30,4 @@ def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary containing emotion probability pairs """ - return api_handler(image, url_root + "fer", batch=batch, auth=auth, **kwargs) + return api_handler(image, cloud=cloud, api="fer", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/lang.py b/indicoio/text/lang.py index 8746c7c..86ffa4c 100644 --- a/indicoio/text/lang.py +++ b/indicoio/text/lang.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def language(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 33 possible languages of what language the text was written in. @@ -24,4 +24,4 @@ def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of language probability pairs """ - return api_handler(text, url_root + "language", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="language", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/sentiment.py b/indicoio/text/sentiment.py index 73e29af..494b40e 100644 --- a/indicoio/text/sentiment.py +++ b/indicoio/text/sentiment.py @@ -2,7 +2,7 @@ from indicoio import JSON_HEADERS from indicoio.utils import api_handler import indicoio.config as config -def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def political(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over the political alignment of the speaker. @@ -28,9 +28,9 @@ def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of party probability pairs """ - return api_handler(text, url_root + "political", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs) -def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def posneg(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a scalar estimate of the sentiment of that text. Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment. @@ -51,4 +51,4 @@ def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Float """ - return api_handler(text, url_root + "sentiment", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="sentiment", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/text/tagging.py b/indicoio/text/tagging.py index f820018..8848379 100644 --- a/indicoio/text/tagging.py +++ b/indicoio/text/tagging.py @@ -1,7 +1,7 @@ from indicoio.utils import api_handler import indicoio.config as config -def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs): +def text_tags(text, cloud=config.cloud, batch=False, auth=None, **kwargs): """ Given input text, returns a probability distribution over 100 document categories @@ -23,4 +23,4 @@ def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs): :rtype: Dictionary of class probability pairs """ - return api_handler(text, url_root + "texttags", batch=batch, auth=auth, **kwargs) + return api_handler(text, cloud=cloud, api="texttags", batch=batch, auth=auth, **kwargs) diff --git a/indicoio/utils/__init__.py b/indicoio/utils/__init__.py index adace55..40f82ae 100644 --- a/indicoio/utils/__init__.py +++ b/indicoio/utils/__init__.py @@ -4,26 +4,22 @@ import numpy as np from skimage.transform import resize from indicoio import JSON_HEADERS +from indicoio import config -def auth_query(): - email = os.environ.get("INDICO_EMAIL") - password = os.environ.get("INDICO_PASSWORD") - # store settings - if not email: - email = raw_input("Email: ") - os.environ["INDICO_EMAIL"] = email - - if not password: - password = getpass.getpass("Password: ") - os.environ["INDICO_PASSWORD"] = password - - return (email, password) - -def api_handler(arg, url, batch=False, auth=None, **kwargs): +def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs): data = {'data': arg} data.update(**kwargs) json_data = json.dumps(data) + + if cloud: + host = "%s.indico.domains" + else: + # default to indico public cloud + host = config.public_api_host + + url = "http://%s/%s" % (host, api) + if batch: url += "/batch" @@ -34,6 +30,7 @@ def api_handler(arg, url, batch=False, auth=None, **kwargs): raise ValueError(error) return results + class TypeCheck(object): """ Decorator that performs a typecheck on the input to a function @@ -118,6 +115,7 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs): return dict(zip(keys, norm_array)) return norm_array + def image_preprocess(image, batch=False): """ Takes an image and prepares it for sending to the api including diff --git a/setup.py b/setup.py index 743dda2..d432a8b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: setup( name="IndicoIo", - version='0.4.15', + version='0.5.0', packages=[ "indicoio", "indicoio.text", @@ -21,7 +21,7 @@ setup( Use pre-built state of the art machine learning algorithms with a single line of code. """, license="MIT License (See LICENSE)", - long_description=open("README").read(), + long_description=open("README.rst").read(), url="https://github.com/IndicoDataSolutions/indicoio-python", author="Alec Radford, Slater Victoroff, Aidan McLaughlin", author_email=""" diff --git a/tests/test_configure.py b/tests/test_configure.py index e3f6c1d..a992151 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -1,15 +1,138 @@ import os +import unittest +import textwrap +from StringIO import StringIO from indicoio import config +from indicoio.config import Settings -def test_batch_set_url_root_as_env_var(): - test_data = ['clearly an english sentence'] - old_private_cloud_url = os.environ.get("INDICO_PRIVATE_CLOUD_URL") - os.environ["INDICO_PRIVATE_CLOUD_URL"] = "http://not.a.real.url/" - assert config.get_api_root() == "http://not.a.real.url/" +class TestConfigureEnv(unittest.TestCase): - if old_private_cloud_url: - os.environ["INDICO_PRIVATE_CLOUD_URL"] = old_private_cloud_url - else: - del(os.environ["INDICO_PRIVATE_CLOUD_URL"]) + def setUp(self): + os.environ = {} + + def test_set_cloud_from_env_var(self): + cloud = "invalid/cloud" + os.environ["INDICO_CLOUD"] = cloud + assert config.settings.cloud() == cloud + + def test_set_auth_from_env_var(self): + username = "test" + password = "password" + os.environ["INDICO_USERNAME"] = username + os.environ["INDICO_PASSWORD"] = password + assert config.settings.auth() == (username, password) + + +class TestConfigurationFile(unittest.TestCase): + + def setUp(self): + self.username = "test" + self.password = "password" + self.cloud = "localhost" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.username, self.password, self.cloud) + + config_file = StringIO(textwrap.dedent(config)) + self.settings = Settings(files=[config_file]) + os.environ = {} + + def test_set_cloud_from_config_file(self): + assert self.settings.cloud() == self.cloud + + def test_set_auth_from_config_file(self): + assert self.settings.auth() == (self.username, self.password) + + +class TestPrecedence(unittest.TestCase): + + def setUp(self): + self.file_username = "file-username" + self.file_password = "file-password" + self.file_cloud = "file-cloud" + + self.env_username = "env-username" + self.env_password = "env-password" + self.env_cloud = "env-cloud" + config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % (self.file_username, self.file_password, self.file_cloud) + + config_file = StringIO(textwrap.dedent(config)) + os.environ = { + 'INDICO_CLOUD': self.env_cloud, + 'INDICO_USERNAME': self.env_username, + 'INDICO_PASSWORD': self.env_password + } + self.settings = Settings(files=[config_file]) + + def test_set_cloud_from_config_file(self): + assert self.settings.cloud() == self.env_cloud + + def test_set_auth_from_config_file(self): + assert self.settings.auth() == (self.env_username, self.env_password) + + +class TestConfigFilePrecedence(unittest.TestCase): + + def setUp(self): + self.high_priority_username = "high-priority-username" + self.high_priority_password = "high-priority-password" + self.high_priority_cloud = "high-priority-cloud" + + self.low_priority_username = "low-priority-username" + self.low_priority_password = "low-priority-password" + self.low_priority_cloud = "low-priority-cloud" + + high_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.high_priority_username, + self.high_priority_password, + self.high_priority_cloud + ) + + low_priority_config = """ + [auth] + username = %s + password = %s + + [private_cloud] + cloud = %s + """ % ( + self.low_priority_username, + self.low_priority_password, + self.low_priority_cloud + ) + + high_priority_config_file = StringIO(textwrap.dedent(high_priority_config)) + low_priority_config_file = StringIO(textwrap.dedent(low_priority_config)) + + os.environ = {} + self.settings = Settings(files=[ + low_priority_config_file, + high_priority_config_file + ]) + + def test_cloud_config_file_priority(self): + assert self.settings.cloud() == self.high_priority_cloud + + def test_auth_config_file_priority(self): + assert self.settings.auth() == (self.high_priority_username, self.high_priority_password) diff --git a/tests/test_remote.py b/tests/test_remote.py index 31e8a45..44e31bd 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -6,6 +6,7 @@ import numpy as np import skimage.io from nose.plugins.skip import Skip, SkipTest +from indicoio import config from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features from indicoio import batch_language, batch_image_features, batch_text_tags @@ -15,11 +16,9 @@ DIR = os.path.dirname(os.path.realpath(__file__)) class BatchAPIRun(unittest.TestCase): def setUp(self): - self.username = os.getenv("INDICO_USERNAME") - self.password = os.getenv("INDICO_PASSWORD") - self.auth = (self.username, self.password) + self.auth = config.auth - if not self.username or not self.password: + if not all(self.auth): raise SkipTest def test_batch_texttags(self): @@ -71,13 +70,13 @@ class BatchAPIRun(unittest.TestCase): self.assertTrue(isinstance(response, list)) self.assertTrue(response[0]['English'] > 0.25) - def test_batch_set_url_root(self): + def test_batch_set_cloud(self): test_data = ['clearly an english sentence'] self.assertRaises(ConnectionError, batch_language, test_data, auth=self.auth, - url_root='http://not.a.real.url/') + cloud='invalid/cloud') class FullAPIRun(unittest.TestCase): @@ -139,13 +138,13 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(fer_set, set(response.keys())) def test_happy_fer(self): - test_face = self.load_image("../data/happy.png", as_grey=True) + test_face = self.load_image("data/happy.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Happy'] > 0.5) def test_fear_fer(self): - test_face = self.load_image("../data/fear.png", as_grey=True) + test_face = self.load_image("data/fear.png", as_grey=True) response = fer(test_face) self.assertTrue(isinstance(response, dict)) self.assertTrue(response['Fear'] > 0.25) @@ -223,12 +222,12 @@ class FullAPIRun(unittest.TestCase): self.assertEqual(language_set, set(language_dict.keys())) assert language_dict['English'] > 0.25 - def test_set_url_root(self): + def test_set_cloud(self): test_data = 'clearly an english sentence' self.assertRaises(ConnectionError, language, test_data, - url_root='http://not.a.real.url/') + cloud='invalid/cloud') if __name__ == "__main__":