Cleaner api handler interface + extended tests

.indicorc

@@ -1,3 +0,0 @@
-[private_cloud]
-url_root = 127.0.0.1
-

AUTHORS

@@ -2,3 +2,4 @@ Slater Victoroff <slater@indico.io>
 Alec Radford <alec@indico.io>
 Aidan McLaughlin <aidan@indico.io>
 Madison May <madison@indico.io>
+Annie Carlson <annie@indico.io>

CHANGES.txt

@@ -18,3 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface
 v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API
 v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage
 v0.4.15, Sat Dec 20 -- Bug fix release
+v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls

README.md

@@ -86,35 +86,46 @@ If you'd like to use our batch api interface, please send an email to contact@in
 
 ```
 >>> from indicio import batch_sentiment
-batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********"))
+>>> batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********"))
 ```
 
-Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc
+Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file.
 
-Private Cloud API Access
+Private cloud API Access
 ------------------------
 
-If you'd like to use our private cloud interface, please send an email to contact@indico.io.
+If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option.
 
 ```
 >>> from indicio import sentiment
-sentiment("Text to analyze", hostname="http://exampleprivatecloud.io/", auth=("example@example.com", "********"))
+>>> sentiment("Text to analyze", cloud="example", auth=("example@example.com", "********"))
 ```
 
-Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc
+The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. 
 
-indicorc
+Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file.
+
+Configuration
 ------------------------
 
-Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional.
+Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional.
 
 Here is an example of a valid indicorc file:
 
+
 ```
 [auth]
 username = test@example.com
 password = secret
 
 [private_cloud]
-hostname = example.indico.io
+cloud = example
 ```
+
+Environment variables take precedence over any configuration found in the indicorc file.
+The following environment variables are valid:
+ - $INDICO_USERNAME
+ - $INDICO_PASSWORD
+ - $INDICO_CLOUD
+
+ Finally, any values explicitly passed in to an api call will override configuration options set in the indicorc file or in an environment variable.

README.rst

@@ -95,29 +95,39 @@ Examples
 ::
 
 
-    Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in indicorc
+    Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file.
 
-    Private Cloud API Access
+    Private cloud API Access
     ------------------------
 
-    If you'd like to use our private cloud interface, please send an email to contact@indico.io.
+    If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option.
 
             from indicio import sentiment sentiment("Text to analyze",
-            hostname="http://exampleprivatecloud.io/",
+            cloud="example", auth=("example@example.com",
-            auth=("example@example.com", "\*\*\*\*\*\*\*\*"))
+            "\*\*\*\*\*\*\*\*"))
 
 ::
 
 
-    Private cloud hostnames can also be set as the environment variable "INDICO_PRIVATE_CLOUD_URL" or as 'hostname' in indicorc
+    The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains. 
 
-    indicorc
+    Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file.
+
+    Configuration
     ------------------------
 
-    Indicoio-python will look first for $HOME/.indicorc then ./.indicorc for the optional configuration file. The indicorc can be used to set an authentication username and password as well as the private cloud hostname, so they don't need to be specified for every call. All sectiions are optional.
+    Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional.
 
     Here is an example of a valid indicorc file:
 
 [auth] username = test@example.com password = secret
 
-[private\_cloud] hostname = example.indico.io \`\`\`
+[private\_cloud] cloud = example \`\`\`
+
+Environment variables take precedence over any configuration found in
+the indicorc file. The following environment variables are valid: -
+:math:`INDICO_USERNAME  - `\ INDICO\_PASSWORD - $INDICO\_CLOUD
+
+Finally, any values explicitly passed in to an api call will override
+configuration options set in the indicorc file or in an environment
+variable.

indicoio/__init__.py

@@ -2,7 +2,7 @@ from functools import partial
 
 JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'}
 
-Version, version, __version__, VERSION = ('0.4.15',) * 4
+Version, version, __version__, VERSION = ('0.5.0',) * 4
 
 from indicoio.text.sentiment import political, posneg
 from indicoio.text.sentiment import posneg as sentiment

indicoio/config.py

@@ -1,29 +1,54 @@
 import os
+from StringIO import StringIO
 
 import ConfigParser
 
-settings = ConfigParser.ConfigParser()
+class Settings(ConfigParser.ConfigParser):
 
-settings_paths = [
+    def __init__(self, *args, **kwargs):
+        """
+        files: filepaths or open file objects
+        """
+        self.files = kwargs.pop('files')
+
+        ConfigParser.ConfigParser.__init__(self, *args, **kwargs)
+
+        for fd in self.files:
+            try:
+                self.readfp(fd)
+            except AttributeError: 
+                self.read(fd)
+
+        self.auth_settings = self.get_section('auth')
+        self.private_cloud_settings = self.get_section('private_cloud')
+
+    def get_section(self, section):
+        """
+        Retrieve a ConfigParser section as a dictionary, default to {}
+        """
+        try:
+            return dict(self.items(section))
+        except ConfigParser.NoSectionError:
+            return {}
+
+    def cloud(self):
+        return (
+            os.getenv("INDICO_CLOUD") or 
+            self.private_cloud_settings.get('cloud') or 
+            None
+        )
+
+    def auth(self):
+        return (
+            os.getenv("INDICO_USERNAME") or self.auth_settings.get('username'),
+            os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password')
+        )
+
+settings = Settings(files=[
     os.path.expanduser("~/.indicorc"),
     os.path.join(os.getcwd(), '.indicorc')
-]
+])
 
-settings.read(settings_paths)
+auth = settings.auth()
-
+cloud = settings.cloud()
-def get_section(parser, section):
+public_api_host = 'apiv1.indico.io'
-    try:
-        return dict(parser.items(section))
-    except ConfigParser.NoSectionError:
-        return {}
-
-auth_settings = get_section(settings, 'auth')
-private_cloud_settings = get_section(settings, 'private_cloud')
-
-api_root = (
-    os.getenv("INDICO_PRIVATE_CLOUD_URL") or 
-    private_cloud_settings.get('url_root') or 
-    "http://apiv1.indico.io/"
-)
-
-auth = (auth_settings.get('username'), auth_settings.get('password'))

indicoio/images/features.py

@@ -6,7 +6,7 @@ import numpy as np
 from indicoio.utils import image_preprocess, api_handler
 import indicoio.config as config
 
-def facial_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def facial_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face.
     Useful as a form of feature engineering for face oriented tasks.
@@ -28,9 +28,9 @@ def facial_features(image, url_root=config.api_root, batch=False, auth=None, **k
     :type image: list of lists
     :rtype: List containing feature responses
     """
-    return api_handler(image, url_root + "facialfeatures", batch=batch, auth=auth, **kwargs)
+    return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs)
 
-def image_features(image, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def image_features(image, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given an input image, returns a 2048 dimensional sparse feature vector explaining that image.
     Useful as a form of feature engineering for image oriented tasks.
@@ -61,4 +61,4 @@ def image_features(image, url_root=config.api_root, batch=False, auth=None, **kw
     :rtype: List containing features
     """
     image = image_preprocess(image, batch=batch)
-    return api_handler(image, url_root + "imagefeatures", batch=batch, auth=auth, **kwargs)
+    return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, auth=auth, **kwargs)

indicoio/images/fer.py

@@ -6,7 +6,7 @@ import numpy as np
 from indicoio.utils import api_handler
 import indicoio.config as config
 
-def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def fer(image, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given a grayscale input image of a face, returns a probability distribution over emotional state.
     Input should be in a list of list format, resizing will be attempted internally but for best
@@ -30,4 +30,4 @@ def fer(image, url_root=config.api_root, batch=False, auth=None, **kwargs):
     :rtype: Dictionary containing emotion probability pairs
     """
 
-    return api_handler(image, url_root + "fer", batch=batch, auth=auth, **kwargs)
+    return api_handler(image, cloud=cloud, api="fer", batch=batch, auth=auth, **kwargs)

indicoio/text/lang.py

@@ -1,7 +1,7 @@
 from indicoio.utils import api_handler
 import indicoio.config as config
 
-def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def language(text, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given input text, returns a probability distribution over 33 possible
     languages of what language the text was written in.
@@ -24,4 +24,4 @@ def language(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
     :rtype: Dictionary of language probability pairs
     """
 
-    return api_handler(text, url_root + "language", batch=batch, auth=auth, **kwargs)
+    return api_handler(text, cloud=cloud, api="language", batch=batch, auth=auth, **kwargs)

indicoio/text/sentiment.py

@@ -2,7 +2,7 @@ from indicoio import JSON_HEADERS
 from indicoio.utils import api_handler
 import indicoio.config as config
 
-def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def political(text, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given input text, returns a probability distribution over the political alignment of the speaker.
 
@@ -28,9 +28,9 @@ def political(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
     :rtype: Dictionary of party probability pairs
     """
 
-    return api_handler(text, url_root + "political", batch=batch, auth=auth, **kwargs)
+    return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs)
 
-def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def posneg(text, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given input text, returns a scalar estimate of the sentiment of that text.
     Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment.
@@ -51,4 +51,4 @@ def posneg(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
     :rtype: Float
     """
 
-    return api_handler(text, url_root + "sentiment", batch=batch, auth=auth, **kwargs)
+    return api_handler(text, cloud=cloud, api="sentiment", batch=batch, auth=auth, **kwargs)

indicoio/text/tagging.py

@@ -1,7 +1,7 @@
 from indicoio.utils import api_handler
 import indicoio.config as config
 
-def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
+def text_tags(text, cloud=config.cloud, batch=False, auth=None, **kwargs):
     """
     Given input text, returns a probability distribution over 100 document categories
 
@@ -23,4 +23,4 @@ def text_tags(text, url_root=config.api_root, batch=False, auth=None, **kwargs):
     :rtype: Dictionary of class probability pairs
     """
 
-    return api_handler(text, url_root + "texttags", batch=batch, auth=auth, **kwargs)
+    return api_handler(text, cloud=cloud, api="texttags", batch=batch, auth=auth, **kwargs)

indicoio/utils/__init__.py

@@ -4,26 +4,22 @@ import numpy as np
 from skimage.transform import resize
 
 from indicoio import JSON_HEADERS
+from indicoio import config
 
-def auth_query():
-    email = os.environ.get("INDICO_EMAIL")
-    password = os.environ.get("INDICO_PASSWORD")
 
-    # store settings
+def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs):
-    if not email:
-        email = raw_input("Email: ")
-        os.environ["INDICO_EMAIL"] = email
-
-    if not password:
-        password = getpass.getpass("Password: ")
-        os.environ["INDICO_PASSWORD"] = password
-
-    return (email, password)
-
-def api_handler(arg, url, batch=False, auth=None, **kwargs):
     data = {'data': arg}
     data.update(**kwargs)
     json_data = json.dumps(data)
+
+    if cloud:
+        host = "%s.indico.domains"
+    else: 
+        # default to indico public cloud
+        host = config.public_api_host
+
+    url = "http://%s/%s" % (host, api)
+
     if batch:
         url += "/batch"
 
@@ -34,6 +30,7 @@ def api_handler(arg, url, batch=False, auth=None, **kwargs):
         raise ValueError(error)
     return results
 
+
 class TypeCheck(object):
     """
     Decorator that performs a typecheck on the input to a function
@@ -118,6 +115,7 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs):
         return dict(zip(keys, norm_array))
     return norm_array
 
+
 def image_preprocess(image, batch=False):
     """
     Takes an image and prepares it for sending to the api including

setup.py

@@ -8,7 +8,7 @@ except ImportError:
 
 setup(
     name="IndicoIo",
-    version='0.4.15',
+    version='0.5.0',
     packages=[
         "indicoio",
         "indicoio.text",
@@ -21,7 +21,7 @@ setup(
         Use pre-built state of the art machine learning algorithms with a single line of code.
     """,
     license="MIT License (See LICENSE)",
-    long_description=open("README").read(),
+    long_description=open("README.rst").read(),
     url="https://github.com/IndicoDataSolutions/indicoio-python",
     author="Alec Radford, Slater Victoroff, Aidan McLaughlin",
     author_email="""

tests/test_configure.py

@@ -1,15 +1,138 @@
 import os
+import unittest
+import textwrap
+from StringIO import StringIO
 
 from indicoio import config
+from indicoio.config import Settings
 
-def test_batch_set_url_root_as_env_var():
-    test_data = ['clearly an english sentence']
-    old_private_cloud_url = os.environ.get("INDICO_PRIVATE_CLOUD_URL")
-    os.environ["INDICO_PRIVATE_CLOUD_URL"] = "http://not.a.real.url/"
 
-    assert config.get_api_root() == "http://not.a.real.url/"
+class TestConfigureEnv(unittest.TestCase):
 
-    if old_private_cloud_url:
+    def setUp(self):
-        os.environ["INDICO_PRIVATE_CLOUD_URL"] = old_private_cloud_url
+        os.environ = {}
-    else:
+
-        del(os.environ["INDICO_PRIVATE_CLOUD_URL"])
+    def test_set_cloud_from_env_var(self):
+        cloud = "invalid/cloud"
+        os.environ["INDICO_CLOUD"] = cloud
+        assert config.settings.cloud() == cloud
+
+    def test_set_auth_from_env_var(self):
+        username = "test"
+        password = "password"
+        os.environ["INDICO_USERNAME"] = username
+        os.environ["INDICO_PASSWORD"] = password
+        assert config.settings.auth() == (username, password)
+
+
+class TestConfigurationFile(unittest.TestCase):
+
+    def setUp(self):
+        self.username = "test"
+        self.password = "password"
+        self.cloud = "localhost"
+        config = """
+        [auth]
+        username = %s
+        password = %s
+
+        [private_cloud]
+        cloud = %s
+        """ % (self.username, self.password, self.cloud)
+
+        config_file = StringIO(textwrap.dedent(config))
+        self.settings = Settings(files=[config_file])
+        os.environ = {}
+
+    def test_set_cloud_from_config_file(self):
+        assert self.settings.cloud() == self.cloud
+
+    def test_set_auth_from_config_file(self):   
+        assert self.settings.auth() == (self.username, self.password)        
+
+
+class TestPrecedence(unittest.TestCase):
+
+    def setUp(self):
+        self.file_username = "file-username"
+        self.file_password = "file-password"
+        self.file_cloud = "file-cloud"
+
+        self.env_username = "env-username"
+        self.env_password = "env-password"      
+        self.env_cloud = "env-cloud"
+        config = """
+        [auth]
+        username = %s
+        password = %s
+
+        [private_cloud]
+        cloud = %s
+        """ % (self.file_username, self.file_password, self.file_cloud)
+
+        config_file = StringIO(textwrap.dedent(config))
+        os.environ = {
+            'INDICO_CLOUD': self.env_cloud,
+            'INDICO_USERNAME': self.env_username,
+            'INDICO_PASSWORD': self.env_password
+        }
+        self.settings = Settings(files=[config_file])
+
+    def test_set_cloud_from_config_file(self):
+        assert self.settings.cloud() == self.env_cloud
+
+    def test_set_auth_from_config_file(self):   
+        assert self.settings.auth() == (self.env_username, self.env_password)        
+
+
+class TestConfigFilePrecedence(unittest.TestCase):
+
+    def setUp(self):
+        self.high_priority_username = "high-priority-username"
+        self.high_priority_password = "high-priority-password"
+        self.high_priority_cloud = "high-priority-cloud"
+
+        self.low_priority_username = "low-priority-username"
+        self.low_priority_password = "low-priority-password"
+        self.low_priority_cloud = "low-priority-cloud"
+
+        high_priority_config = """
+        [auth]
+        username = %s
+        password = %s
+
+        [private_cloud]
+        cloud = %s
+        """ % (
+            self.high_priority_username, 
+            self.high_priority_password, 
+            self.high_priority_cloud
+        )
+
+        low_priority_config = """
+        [auth]
+        username = %s
+        password = %s
+
+        [private_cloud]
+        cloud = %s
+        """ % (
+            self.low_priority_username, 
+            self.low_priority_password, 
+            self.low_priority_cloud
+        )
+
+        high_priority_config_file = StringIO(textwrap.dedent(high_priority_config))
+        low_priority_config_file = StringIO(textwrap.dedent(low_priority_config))
+
+        os.environ = {}
+        self.settings = Settings(files=[
+            low_priority_config_file,
+            high_priority_config_file
+        ])
+
+    def test_cloud_config_file_priority(self):
+        assert self.settings.cloud() == self.high_priority_cloud
+
+    def test_auth_config_file_priority(self):   
+        assert self.settings.auth() == (self.high_priority_username, self.high_priority_password)        

tests/test_remote.py

@@ -6,6 +6,7 @@ import numpy as np
 import skimage.io
 from nose.plugins.skip import Skip, SkipTest
 
+from indicoio import config
 from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags
 from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features
 from indicoio import batch_language, batch_image_features, batch_text_tags
@@ -15,11 +16,9 @@ DIR = os.path.dirname(os.path.realpath(__file__))
 class BatchAPIRun(unittest.TestCase):
 
     def setUp(self):
-        self.username = os.getenv("INDICO_USERNAME")
+        self.auth = config.auth
-        self.password = os.getenv("INDICO_PASSWORD")
-        self.auth = (self.username, self.password)
 
-        if not self.username or not self.password:
+        if not all(self.auth):
             raise SkipTest
 
     def test_batch_texttags(self):
@@ -71,13 +70,13 @@ class BatchAPIRun(unittest.TestCase):
         self.assertTrue(isinstance(response, list))
         self.assertTrue(response[0]['English'] > 0.25)
 
-    def test_batch_set_url_root(self):
+    def test_batch_set_cloud(self):
         test_data = ['clearly an english sentence']
         self.assertRaises(ConnectionError,
                           batch_language,
                           test_data,
                           auth=self.auth,
-                          url_root='http://not.a.real.url/')
+                          cloud='invalid/cloud')
 
 
 class FullAPIRun(unittest.TestCase):
@@ -139,13 +138,13 @@ class FullAPIRun(unittest.TestCase):
         self.assertEqual(fer_set, set(response.keys()))
 
     def test_happy_fer(self):
-        test_face = self.load_image("../data/happy.png", as_grey=True)
+        test_face = self.load_image("data/happy.png", as_grey=True)
         response = fer(test_face)
         self.assertTrue(isinstance(response, dict))
         self.assertTrue(response['Happy'] > 0.5)
 
     def test_fear_fer(self):
-        test_face = self.load_image("../data/fear.png", as_grey=True)
+        test_face = self.load_image("data/fear.png", as_grey=True)
         response = fer(test_face)
         self.assertTrue(isinstance(response, dict))
         self.assertTrue(response['Fear'] > 0.25)
@@ -223,12 +222,12 @@ class FullAPIRun(unittest.TestCase):
         self.assertEqual(language_set, set(language_dict.keys()))
         assert language_dict['English'] > 0.25
 
-    def test_set_url_root(self):
+    def test_set_cloud(self):
         test_data = 'clearly an english sentence'
         self.assertRaises(ConnectionError,
                           language,
                           test_data,
-                          url_root='http://not.a.real.url/')
+                          cloud='invalid/cloud')
 
 
 if __name__ == "__main__":