wassname/IndicoIo-python
1
0
Fork 0
mirror of https://github.com/wassname/IndicoIo-python.git synced 2026-06-27 16:10:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28 from IndicoDataSolutions/annie/private_cloud

Browse Source 
Annie/private cloud
This commit is contained in:
Slater-Victoroff
2015-02-27 17:03:59 -05:00
parent a1621e8531 6dd0738a8b
commit e6d40b7bdd
19 changed files with 502 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AUTHORS
+1
View File
@@ -2,3 +2,4 @@ Slater Victoroff <slater@indico.io>
Alec Radford <alec@indico.io>
Aidan McLaughlin <aidan@indico.io>
Madison May <madison@indico.io>
Annie Carlson <annie@indico.io>
CHANGES.txt
+1
View File
@@ -18,3 +18,4 @@ v0.4.12, Fri Dec 19 -- Added batch support interface
v0.4.13, Fri Dec 19 -- Added optional arguments to text tags API
v0.4.14, Sat Dec 20 -- Fix for batch image features preprocessing, increased test coverage
v0.4.15, Sat Dec 20 -- Bug fix release
v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc file to reduce redundant authorization calls, README updates
README
-99
View File
@@ -1,99 +0,0 @@
indicoio-python
===============
A wrapper for a series of APIs made by indico.
Check out the main site on:
http://indico.io
Check out our documentation on:
http://indicoiopython.s3-website-us-west-2.amazonaws.com/indicoio.html
Our APIs are totally free to use, and ready to be used in your application. No data or training required.
Current APIs
------------
Right now this wrapper supports the following apps:
- Positive/Negative Sentiment Analysis
- Political Sentiment Analysis
- Image Feature Extraction
- Facial Emotion Recognition
- Facial Feature Extraction
- Language Detection
- Text Topic Tagging
Examples
--------
```
>>> import numpy as np
>>> from indicoio import political, sentiment, fer, facial_features, language
>>> political("Guns don't kill people. People kill people.")
{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146}
>>> sentiment('Worst movie ever.')
{u'Sentiment': 0.07062467665597527}
>>> sentiment('Really enjoyed the movie.')
{u'Sentiment': 0.8105182526856075}
>>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets."
>>> tag_dict = text_tags(test_text)
>>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3]
[u'startups_and_entrepreneurship', u'investment', u'business']
>>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1
{u'startups_and_entrepreneurship': 0.21888586688354486}
>>> text_tags(test_text, top_n=1) # return only keys with top_n values
{u'startups_and_entrepreneurship': 0.21888586688354486}
>>> tag_dict
{u'fashion': 0.011450126534350728, u'art': 0.00358698972755963, u'energy': 0.005537894035625527, ...}
>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist()
>>> fer(test_face)
{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382}
>>> facial_features(test_face)
[0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777]
>>> language_dict = language('Quis custodiet ipsos custodes')
>>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5]
[u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish']
>>> language_dict
{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...}
```
If you have a local indico server running, simply import from `indicoio.local`.
```
>>> from indicoio.local import political, sentiment, fer, facial_features, language
```
If you'd like to use our batch api interface, please send an email to contact@indico.io.
```
>>> from indicio import batch_sentiment
batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********"))
```
Installation
------------
```
pip install indicoio
```
Announcement: Indico has partnered with Experfy, a data science consulting marketplace based in the Harvard Innovation Lab. Through Experfy, we are helping our data science community members find lucrative projects and advance their skills. Please signup for Experfy at https://www.experfy.com/ to get started.
README.md
+42 -8
View File
@@ -78,13 +78,6 @@ Examples
>>> language_dict
{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...}
```
If you have a local indico server running, simply import from `indicoio.local`.
```
>>> from indicoio.local import political, sentiment, fer, facial_features, language
```
Batch API Access
----------------
@@ -93,5 +86,46 @@ If you'd like to use our batch api interface, please send an email to contact@in
```
>>> from indicio import batch_sentiment
batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********"))
>>> batch_sentiment(['Text to analyze', 'More text'], auth=("example@example.com", "********"))
```
Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file.
Private cloud API Access
------------------------
If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option.
```
>>> from indicio import sentiment
>>> sentiment("Text to analyze", cloud="example", auth=("example@example.com", "********"))
```
The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains.
Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file.
Configuration
------------------------
Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional.
Here is an example of a valid indicorc file:
```
[auth]
username = test@example.com
password = secret
[private_cloud]
cloud = example
```
Environment variables take precedence over any configuration found in the indicorc file.
The following environment variables are valid:
- $INDICO_USERNAME
- $INDICO_PASSWORD
- $INDICO_CLOUD
Finally, any values explicitly passed in to an api call will override configuration options set in the indicorc file or in an environment variable.
README.rst
+133
View File
@@ -0,0 +1,133 @@
indicoio-python
===============
A wrapper for a series of APIs made by indico.
Check out the main site on:
http://indico.io
Our APIs are totally free to use, and ready to be used in your
application. No data or training required.
Installation
------------
::
pip install indicoio
Documentation
-------------
Available at `indico.reame.io <http://indico.readme.io/v1.0/docs>`__
Current APIs
------------
Right now this wrapper supports the following apps:
- Positive/Negative Sentiment Analysis
- Political Sentiment Analysis
- Image Feature Extraction
- Facial Emotion Recognition
- Facial Feature Extraction
- Language Detection
- Text Topic Tagging
Examples
--------
::
>>> import numpy as np
>>> from indicoio import political, sentiment, fer, facial_features, language
>>> political("Guns don't kill people. People kill people.")
{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146}
>>> sentiment('Worst movie ever.')
{u'Sentiment': 0.07062467665597527}
>>> sentiment('Really enjoyed the movie.')
{u'Sentiment': 0.8105182526856075}
>>> test_text = "Facebook blog posts about Android tech make better journalism than most news outlets."
>>> tag_dict = text_tags(test_text)
>>> sorted(tag_dict.keys(), key=lambda x: tag_dict[x], reverse=True)[:3]
[u'startups_and_entrepreneurship', u'investment', u'business']
>>> text_tags(test_text, threshold=0.1) # return only keys with value > 0.1
{u'startups_and_entrepreneurship': 0.21888586688354486}
>>> text_tags(test_text, top_n=1) # return only keys with top_n values
{u'startups_and_entrepreneurship': 0.21888586688354486}
>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist()
>>> fer(test_face)
{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382}
>>> facial_features(test_face)
[0.0, -0.02568680526917187, 0.21645604230056517, -0.1519435786033145, -0.5648621854611555, 3.0607368045577226, 0.11434321880792693, -0.02163810928547493, -0.44224330594186484, 0.3024315632285246, -2.6068048934495276, 2.497798330306638, 3.040558335205844, 0.741045340525325, 0.37198135618478817, -0.33132377802172325, -0.9804190889833034, 0.5046575784709395, -0.5609132323152847, 1.679107064439151, 0.6825037853544341, -1.5977176226648016, 1.8959464303080562, -0.7812860715595836, -2.998394007543733, -0.22637273967347724, -0.9642457010679496, 1.4557274834236749, 2.412244419186633, 2.3151771738421965, 0.7881483386786367, 1.6622850935863422, 0.1304768990234367, 1.9344501393866649, 3.1271558035162914, -0.10250886439220543, 1.4921395116492966, 2.761645355670677, 1.6903473594991179, 1.009209807271491, 0.07273926986120445, -1.4941708135718021, -2.082786362439631, 1.0160924044870847, 2.5326580674673895, -0.8328208491083264, 2.0390177029762935, 3.0342637531932777]
>>> language_dict = language('Quis custodiet ipsos custodes')
>>> sorted(language_dict.keys(), key=lambda x: language_dict[x], reverse=True)[:5]
[u'Latin', u'Dutch', u'Greek', u'Portuguese', u'Spanish']
>>> language_dict
{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...}
Batch API Access
----------------
If you'd like to use our batch api interface, please send an email to contact@indico.io.
from indicio import batch\_sentiment batch\_sentiment(['Text
to analyze', 'More text'], auth=("example@example.com",
"\*\*\*\*\*\*\*\*"))
::
Authentication credentials can also be set as the environment variables "INDICO_USERNAME" and "INDICO_PASSWORD" or as 'username' and 'password' in the indicorc file.
Private cloud API Access
------------------------
If you're looking to use indico's API for high throughput applications, please contact contact@indico.io about our private cloud option.
from indicio import sentiment sentiment("Text to analyze",
cloud="example", auth=("example@example.com",
"\*\*\*\*\*\*\*\*"))
::
The `cloud` parameter redirects API calls to your private cloud hosted at [cloud].indico.domains.
Private cloud subdomains can also be set as the environment variable "INDICO_CLOUD" or as 'cloud' in the indicorc file.
Configuration
------------------------
Indicoio-python will search ./.indicorc and $HOME/.indicorc for the optional configuration file. Values in the local configuration file (./.indicorc) take precedence over those found in a global configuration file ($HOME/.indicorc). The indicorc file can be used to set an authentication username and password or a private cloud subdomain, so these arguments don't need to be specified for every api call. All sections are optional.
Here is an example of a valid indicorc file:
[auth] username = test@example.com password = secret
[private\_cloud] cloud = example \`\`\`
Environment variables take precedence over any configuration found in
the indicorc file. The following environment variables are valid: -
:math:`INDICO_USERNAME - `\ INDICO\_PASSWORD - $INDICO\_CLOUD
Finally, any values explicitly passed in to an api call will override
configuration options set in the indicorc file or in an environment
variable.
indicoio/__init__.py
+13 -10
View File
@@ -1,9 +1,8 @@
from functools import partial
import indicoio.config as config
JSON_HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'}
Version, version, __version__, VERSION = ('0.4.15',) * 4
Version, version, __version__, VERSION = ('0.5.0',) * 4
from indicoio.text.sentiment import political, posneg
from indicoio.text.sentiment import posneg as sentiment
@@ -13,14 +12,18 @@ from indicoio.images.fer import fer
from indicoio.images.features import facial_features
from indicoio.images.features import image_features
apis = ['political', 'posneg', 'sentiment', 'language', 'fer',
'facial_features', 'image_features', 'text_tags']
apis = [
'political',
'posneg',
'sentiment',
'language',
'fer',
'facial_features',
'image_features',
'text_tags'
]
apis = dict((api, globals().get(api)) for api in apis)
class Namespace(object): pass
local = Namespace()
for api in apis:
globals()[api] = partial(apis[api], config.api_root)
globals()['batch_' + api] = partial(apis[api], config.api_root, batch=True)
setattr(local, api, partial(apis[api], config.local_api_root))
setattr(local, 'batch_' + api, partial(apis[api], config.local_api_root, batch=True))
globals()[api] = partial(apis[api])
globals()['batch_' + api] = partial(apis[api], batch=True)
indicoio/config.py
+54 -2
View File
@@ -1,2 +1,54 @@
local_api_root = "http://localhost:9438/"
api_root = "http://apiv1.indico.io/"
import os
from StringIO import StringIO
import ConfigParser
class Settings(ConfigParser.ConfigParser):
def __init__(self, *args, **kwargs):
"""
files: filepaths or open file objects
"""
self.files = kwargs.pop('files')
ConfigParser.ConfigParser.__init__(self, *args, **kwargs)
for fd in self.files:
try:
self.readfp(fd)
except AttributeError:
self.read(fd)
self.auth_settings = self.get_section('auth')
self.private_cloud_settings = self.get_section('private_cloud')
def get_section(self, section):
"""
Retrieve a ConfigParser section as a dictionary, default to {}
"""
try:
return dict(self.items(section))
except ConfigParser.NoSectionError:
return {}
def cloud(self):
return (
os.getenv("INDICO_CLOUD") or
self.private_cloud_settings.get('cloud') or
None
)
def auth(self):
return (
os.getenv("INDICO_USERNAME") or self.auth_settings.get('username'),
os.getenv("INDICO_PASSWORD") or self.auth_settings.get('password')
)
SETTINGS = Settings(files=[
os.path.expanduser("~/.indicorc"),
os.path.join(os.getcwd(), '.indicorc')
])
AUTH = SETTINGS.auth()
CLOUD = SETTINGS.cloud()
PUBLIC_API_HOST = 'apiv1.indico.io'
indicoio/images/features.py
+8 -7
View File
@@ -4,12 +4,13 @@ import requests
import numpy as np
from indicoio.utils import image_preprocess, api_handler
import indicoio.config as config
def facial_features(api_root, image, batch=False, auth=None, **kwargs):
def facial_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given an grayscale input image of a face, returns a 48 dimensional feature vector explaining that face.
Useful as a form of feature engineering for face oriented tasks.
Input should be in a list of list format, resizing will be attempted internally but for best
Input should be in a list of list format, resizing will be attempted internally but for best
performance, images should be already sized at 48x48 pixels.
Example usage:
@@ -27,18 +28,18 @@ def facial_features(api_root, image, batch=False, auth=None, **kwargs):
:type image: list of lists
:rtype: List containing feature responses
"""
return api_handler(image, api_root + "facialfeatures", batch=batch, auth=auth, **kwargs)
return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, auth=auth, **kwargs)
def image_features(api_root, image, batch=False, auth=None, **kwargs):
def image_features(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given an input image, returns a 2048 dimensional sparse feature vector explaining that image.
Given an input image, returns a 2048 dimensional sparse feature vector explaining that image.
Useful as a form of feature engineering for image oriented tasks.
* Input can be either grayscale or rgb color and should either be a numpy array or nested list format.
* Input data should be either uint8 0-255 range values or floating point between 0 and 1.
* Large images (i.e. 1024x768+) are much bigger than needed, resizing will be done internally to 64x64 if needed.
* For ideal performance, images should be square aspect ratio but non-square aspect ratios are supported as well.
Example usage:
.. code-block:: python
@@ -60,4 +61,4 @@ def image_features(api_root, image, batch=False, auth=None, **kwargs):
:rtype: List containing features
"""
image = image_preprocess(image, batch=batch)
return api_handler(image, api_root + "imagefeatures", batch=batch, auth=auth, **kwargs)
return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, auth=auth, **kwargs)
indicoio/images/fer.py
+9 -7
View File
@@ -2,12 +2,14 @@ import json
import requests
import numpy as np
from indicoio.utils import api_handler
def fer(api_root, image, batch=False, auth=None, **kwargs):
from indicoio.utils import api_handler
import indicoio.config as config
def fer(image, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given a grayscale input image of a face, returns a probability distribution over emotional state.
Input should be in a list of list format, resizing will be attempted internally but for best
Input should be in a list of list format, resizing will be attempted internally but for best
performance, images should be already sized at 48x48 pixels..
Example usage:
@@ -19,13 +21,13 @@ def fer(api_root, image, batch=False, auth=None, **kwargs):
>>> face = np.zeros((48,48)).tolist()
>>> emotions = fer(face)
>>> emotions
{u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839,
u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344,
{u'Angry': 0.6340586827229989, u'Sad': 0.1764309536057839,
u'Neutral': 0.05582989039191157, u'Surprise': 0.0072685938275375344,
u'Fear': 0.08523385724298838, u'Happy': 0.04117802220878012}
:param image: The image to be analyzed.
:type image: list of lists
:rtype: Dictionary containing emotion probability pairs
"""
return api_handler(image, api_root + "fer", batch=batch, auth=auth, **kwargs)
return api_handler(image, cloud=cloud, api="fer", batch=batch, auth=auth, **kwargs)
indicoio/text/lang.py
+5 -4
View File
@@ -1,8 +1,9 @@
from indicoio.utils import api_handler
import indicoio.config as config
def language(api_root, text, batch=False, auth=None, **kwargs):
def language(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given input text, returns a probability distribution over 33 possible
Given input text, returns a probability distribution over 33 possible
languages of what language the text was written in.
Example usage:
@@ -22,5 +23,5 @@ def language(api_root, text, batch=False, auth=None, **kwargs):
:type text: str or unicode
:rtype: Dictionary of language probability pairs
"""
return api_handler(text, api_root + "language", batch=batch, auth=auth, **kwargs)
return api_handler(text, cloud=cloud, api="language", batch=batch, auth=auth, **kwargs)
indicoio/text/sentiment.py
+7 -6
View File
@@ -1,7 +1,8 @@
from indicoio import JSON_HEADERS
from indicoio.utils import api_handler
import indicoio.config as config
def political(api_root, text, batch=False, auth=None, **kwargs):
def political(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given input text, returns a probability distribution over the political alignment of the speaker.
@@ -15,7 +16,7 @@ def political(api_root, text, batch=False, auth=None, **kwargs):
Hopefully, driverless cars will chance economics from ownership to fee for service.'
>>> affiliation = political(text)
>>> affiliation
{u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122,
{u'Libertarian': 0.4923755446986322, u'Green': 0.2974443102818122,
u'Liberal': 0.13730032938784784, u'Conservative': 0.07287981563170784}
>>> least_like = affiliation.keys()[np.argmin(affiliation.values())]
>>> most_like = affiliation.keys()[np.argmax(affiliation.values())]
@@ -27,9 +28,9 @@ def political(api_root, text, batch=False, auth=None, **kwargs):
:rtype: Dictionary of party probability pairs
"""
return api_handler(text, api_root + "political", batch=batch, auth=auth, **kwargs)
return api_handler(text, cloud=cloud, api="political", batch=batch, auth=auth, **kwargs)
def posneg(api_root, text, batch=False, auth=None, **kwargs):
def posneg(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given input text, returns a scalar estimate of the sentiment of that text.
Values are roughly in the range 0 to 1 with 0.5 indicating neutral sentiment.
@@ -49,5 +50,5 @@ def posneg(api_root, text, batch=False, auth=None, **kwargs):
:type text: str or unicode
:rtype: Float
"""
return api_handler(text, api_root + "sentiment", batch=batch, auth=auth, **kwargs)
return api_handler(text, cloud=cloud, api="sentiment", batch=batch, auth=auth, **kwargs)
indicoio/text/tagging.py
+4 -3
View File
@@ -1,6 +1,7 @@
from indicoio.utils import api_handler
import indicoio.config as config
def text_tags(api_root, text, batch=False, auth=None, **kwargs):
def text_tags(text, cloud=config.CLOUD, batch=False, auth=None, **kwargs):
"""
Given input text, returns a probability distribution over 100 document categories
@@ -21,5 +22,5 @@ def text_tags(api_root, text, batch=False, auth=None, **kwargs):
:type text: str or unicode
:rtype: Dictionary of class probability pairs
"""
return api_handler(text, api_root + "texttags", batch=batch, auth=auth, **kwargs)
return api_handler(text, cloud=cloud, api="texttags", batch=batch, auth=auth, **kwargs)
indicoio/utils/__init__.py
+24 -20
View File
@@ -4,36 +4,39 @@ import numpy as np
from skimage.transform import resize
from indicoio import JSON_HEADERS
from indicoio import config
def auth_query():
email = os.environ.get("INDICO_EMAIL")
password = os.environ.get("INDICO_PASSWORD")
# store settings
if not email:
email = raw_input("Email: ")
os.environ["INDICO_EMAIL"] = email
if not password:
password = getpass.getpass("Password: ")
os.environ["INDICO_PASSWORD"] = password
return (email, password)
def api_handler(arg, url, batch=False, auth=None, **kwargs):
def api_handler(arg, cloud, api, batch=False, auth=None, **kwargs):
data = {'data': arg}
data.update(**kwargs)
json_data = json.dumps(data)
if cloud:
host = "%s.indico.domains" % cloud
else:
# default to indico public cloud
host = config.PUBLIC_API_HOST
url = "http://%s/%s" % (host, api)
if batch:
url += "/batch"
response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth).json()
results = response.get('results', False)
if not auth:
auth = config.AUTH
response = requests.post(url, data=json_data, headers=JSON_HEADERS, auth=auth)
if response.status_code == 503 and cloud != None:
raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api))
json_results = response.json()
results = json_results.get('results', False)
if results is False:
error = response.get('error')
error = json_results.get('error')
raise ValueError(error)
return results
class TypeCheck(object):
"""
Decorator that performs a typecheck on the input to a function
@@ -118,9 +121,10 @@ def normalize(array, distribution=1, norm_range=(0, 1), **kwargs):
return dict(zip(keys, norm_array))
return norm_array
def image_preprocess(image, batch=False):
"""
Takes an image and prepares it for sending to the api including
Takes an image and prepares it for sending to the api including
resizing and image data/structure standardizing.
"""
if batch:
@@ -137,4 +141,4 @@ def image_preprocess(image, batch=False):
image = image[:,:,:3]
image = resize(image,(64,64))
image = image.tolist()
return image
return image
setup.py
+2 -2
View File
@@ -8,7 +8,7 @@ except ImportError:
setup(
name="IndicoIo",
version='0.4.15',
version='0.5.0',
packages=[
"indicoio",
"indicoio.text",
@@ -21,7 +21,7 @@ setup(
Use pre-built state of the art machine learning algorithms with a single line of code.
""",
license="MIT License (See LICENSE)",
long_description=open("README").read(),
long_description=open("README.rst").read(),
url="https://github.com/IndicoDataSolutions/indicoio-python",
author="Alec Radford, Slater Victoroff, Aidan McLaughlin",
author_email="""
tests/local/__init__.py
View File
tests/local/test_local.py
-156
View File
@@ -1,156 +0,0 @@
import unittest
import os
import numpy as np
from indicoio.local import political, sentiment, fer, facial_features, language, image_features, text_tags
DIR = os.path.dirname(os.path.realpath(__file__))
class FullAPIRun(unittest.TestCase):
def load_image(self, relpath, as_grey=False):
image_path = os.path.normpath(os.path.join(DIR, relpath))
image = skimage.io.imread(image_path, as_grey=True).tolist()
return image
def check_range(self, list, minimum=0.9, maximum=0.1, span=0.5):
vector = np.asarray(list)
self.assertTrue(vector.max() > maximum)
self.assertTrue(vector.min() < minimum)
self.assertTrue(np.ptp(vector) > span)
def test_text_tags(self):
text = "On Monday, president Barack Obama will be..."
results = text_tags(text)
max_keys = sorted(results.keys(), key=lambda x:results.get(x), reverse=True)
assert 'political_discussion' in max_keys[:5]
results = text_tags(text, top_n=5)
assert len(results) is 5
results = text_tags(text, threshold=0.1)
for v in results.values():
assert v >= 0.1
def test_political(self):
political_set = set(['Libertarian', 'Liberal', 'Conservative', 'Green'])
test_string = "Guns don't kill people, people kill people."
response = political(test_string)
self.assertTrue(isinstance(response, dict))
self.assertEqual(political_set, set(response.keys()))
test_string = "Save the whales"
response = political(test_string)
self.assertTrue(isinstance(response, dict))
assert response['Green'] > 0.5
def test_posneg(self):
test_string = "Worst song ever."
response = sentiment(test_string)
self.assertTrue(isinstance(response, float))
self.assertTrue(response < 0.5)
test_string = "Best song ever."
response = sentiment(test_string)
self.assertTrue(isinstance(response, float))
self.assertTrue(response > 0.5)
def test_good_fer(self):
fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy'])
test_face = np.random.rand(48,48).tolist()
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertEqual(fer_set, set(response.keys()))
def test_happy_fer(self):
test_face = self.load_image("../data/happy.png", as_grey=True)
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertTrue(response['Happy'] > 0.5)
def test_fear_fer(self):
test_face = self.load_image("../data/fear.png", as_grey=True)
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertTrue(response['Fear'] > 0.25)
def test_bad_fer(self):
fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy'])
test_face = np.random.rand(56,56).tolist()
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertEqual(fer_set, set(response.keys()))
def test_good_facial_features(self):
test_face = np.random.rand(48,48).tolist()
response = facial_features(test_face)
self.assertTrue(isinstance(response, list))
self.assertEqual(len(response), 48)
self.check_range(response)
def test_good_image_features_greyscale(self):
test_image = np.random.rand(64, 64).tolist()
response = image_features(test_image)
self.assertTrue(isinstance(response, list))
self.assertEqual(len(response), 2048)
self.check_range(response)
def test_good_image_features_rgb(self):
test_image = np.random.rand(64, 64, 3).tolist()
response = image_features(test_image)
self.assertTrue(isinstance(response, list))
self.assertEqual(len(response), 2048)
self.check_range(response)
def test_language(self):
language_set = set([
'English',
'Spanish',
'Tagalog',
'Esperanto',
'French',
'Chinese',
'French',
'Bulgarian',
'Latin',
'Slovak',
'Hebrew',
'Russian',
'German',
'Japanese',
'Korean',
'Portuguese',
'Italian',
'Polish',
'Turkish',
'Dutch',
'Arabic',
'Persian (Farsi)',
'Czech',
'Swedish',
'Indonesian',
'Vietnamese',
'Romanian',
'Greek',
'Danish',
'Hungarian',
'Thai',
'Finnish',
'Norwegian',
'Lithuanian'
])
language_dict = language('clearly an english sentence')
self.assertEqual(language_set, set(language_dict.keys()))
assert language_dict['English'] > 0.25
if __name__ == "__main__":
unittest.main()
tests/remote/__init__.py
View File
tests/test_configure.py
+175
View File
@@ -0,0 +1,175 @@
import os
import unittest
import textwrap
from StringIO import StringIO
from indicoio import config
from indicoio.config import Settings
class TestConfigureEnv(unittest.TestCase):
"""
Ensure that environment variables are handled by the `Settings` parser
"""
def setUp(self):
os.environ = {}
def test_set_cloud_from_env_var(self):
"""
Ensure cloud hostname is read in from environment variables
"""
cloud = "invalid/cloud"
os.environ["INDICO_CLOUD"] = cloud
assert config.SETTINGS.cloud() == cloud
def test_set_auth_from_env_var(self):
"""
Ensure cloud authentication credentials are read in from environment variables
"""
username = "test"
password = "password"
os.environ["INDICO_USERNAME"] = username
os.environ["INDICO_PASSWORD"] = password
assert config.SETTINGS.auth() == (username, password)
class TestConfigurationFile(unittest.TestCase):
"""
Ensure that the `Settings` parser reads in configuration files properly
"""
def setUp(self):
self.username = "test"
self.password = "password"
self.cloud = "localhost"
config = """
[auth]
username = %s
password = %s
[private_cloud]
cloud = %s
""" % (self.username, self.password, self.cloud)
config_file = StringIO(textwrap.dedent(config))
self.settings = Settings(files=[config_file])
os.environ = {}
def test_set_cloud_from_config_file(self):
"""
Ensure cloud hostname is read in from file
"""
assert self.settings.cloud() == self.cloud
def test_set_auth_from_config_file(self):
"""
Ensure cloud authentication credentials are read in from file
"""
assert self.settings.auth() == (self.username, self.password)
class TestPrecedence(unittest.TestCase):
"""
Ensure that environment variables take precedence to config files
"""
def setUp(self):
self.file_username = "file-username"
self.file_password = "file-password"
self.file_cloud = "file-cloud"
self.env_username = "env-username"
self.env_password = "env-password"
self.env_cloud = "env-cloud"
config = """
[auth]
username = %s
password = %s
[private_cloud]
cloud = %s
""" % (self.file_username, self.file_password, self.file_cloud)
config_file = StringIO(textwrap.dedent(config))
os.environ = {
'INDICO_CLOUD': self.env_cloud,
'INDICO_USERNAME': self.env_username,
'INDICO_PASSWORD': self.env_password
}
self.settings = Settings(files=[config_file])
def test_set_cloud_from_config_file(self):
"""
Ensure cloud hosts set in environment variables are used over those in config files
"""
assert self.settings.cloud() == self.env_cloud
def test_set_auth_from_config_file(self):
"""
Ensure cloud authentication credentials set in environment variables
are used over those in config files
"""
assert self.settings.auth() == (self.env_username, self.env_password)
class TestConfigFilePrecedence(unittest.TestCase):
"""
Ensure that files passed in to a `Settings` object are assigned proper priority
"""
def setUp(self):
self.high_priority_username = "high-priority-username"
self.high_priority_password = "high-priority-password"
self.high_priority_cloud = "high-priority-cloud"
self.low_priority_username = "low-priority-username"
self.low_priority_password = "low-priority-password"
self.low_priority_cloud = "low-priority-cloud"
high_priority_config = """
[auth]
username = %s
password = %s
[private_cloud]
cloud = %s
""" % (
self.high_priority_username,
self.high_priority_password,
self.high_priority_cloud
)
low_priority_config = """
[auth]
username = %s
password = %s
[private_cloud]
cloud = %s
""" % (
self.low_priority_username,
self.low_priority_password,
self.low_priority_cloud
)
high_priority_config_file = StringIO(textwrap.dedent(high_priority_config))
low_priority_config_file = StringIO(textwrap.dedent(low_priority_config))
os.environ = {}
self.settings = Settings(files=[
low_priority_config_file,
high_priority_config_file
])
def test_cloud_config_file_priority(self):
"""
Ensure the cloud subdomain priority is handled properly
"""
assert self.settings.cloud() == self.high_priority_cloud
def test_auth_config_file_priority(self):
"""
Ensure the cloud auth priority is handled properly
"""
assert self.settings.auth() == (self.high_priority_username, self.high_priority_password)
tests/remote/test_remote.py → tests/test_remote.py
+24 -9
View File
@@ -1,10 +1,12 @@
import unittest
import os
from requests import ConnectionError
import numpy as np
import skimage.io
from nose.plugins.skip import Skip, SkipTest
from indicoio import config
from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags
from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features
from indicoio import batch_language, batch_image_features, batch_text_tags
@@ -14,11 +16,9 @@ DIR = os.path.dirname(os.path.realpath(__file__))
class BatchAPIRun(unittest.TestCase):
def setUp(self):
self.username = os.getenv("INDICO_USERNAME")
self.password = os.getenv("INDICO_PASSWORD")
self.auth = (self.username, self.password)
self.auth = config.AUTH
if not self.username or not self.password:
if not all(self.auth):
raise SkipTest
def test_batch_texttags(self):
@@ -49,7 +49,7 @@ class BatchAPIRun(unittest.TestCase):
self.assertTrue(isinstance(response, list))
self.assertTrue(isinstance(response[0], list))
self.assertEqual(len(response[0]), 48)
def test_batch_image_features_greyscale(self):
test_data = [np.random.rand(64, 64).tolist()]
response = batch_image_features(test_data, auth=self.auth)
@@ -63,13 +63,21 @@ class BatchAPIRun(unittest.TestCase):
self.assertTrue(isinstance(response, list))
self.assertTrue(isinstance(response[0], list))
self.assertEqual(len(response[0]), 2048)
def test_batch_language(self):
test_data = ['clearly an english sentence']
response = batch_language(test_data, auth=self.auth)
self.assertTrue(isinstance(response, list))
self.assertTrue(response[0]['English'] > 0.25)
def test_batch_set_cloud(self):
test_data = ['clearly an english sentence']
self.assertRaises(ConnectionError,
batch_language,
test_data,
auth=self.auth,
cloud='invalid/cloud')
class FullAPIRun(unittest.TestCase):
@@ -130,13 +138,13 @@ class FullAPIRun(unittest.TestCase):
self.assertEqual(fer_set, set(response.keys()))
def test_happy_fer(self):
test_face = self.load_image("../data/happy.png", as_grey=True)
test_face = self.load_image("data/happy.png", as_grey=True)
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertTrue(response['Happy'] > 0.5)
def test_fear_fer(self):
test_face = self.load_image("../data/fear.png", as_grey=True)
test_face = self.load_image("data/fear.png", as_grey=True)
response = fer(test_face)
self.assertTrue(isinstance(response, dict))
self.assertTrue(response['Fear'] > 0.25)
@@ -156,7 +164,7 @@ class FullAPIRun(unittest.TestCase):
self.assertTrue(isinstance(response, list))
self.assertEqual(len(response), 48)
self.check_range(response)
def test_good_image_features_greyscale(self):
test_image = np.random.rand(64, 64).tolist()
response = image_features(test_image)
@@ -214,6 +222,13 @@ class FullAPIRun(unittest.TestCase):
self.assertEqual(language_set, set(language_dict.keys()))
assert language_dict['English'] > 0.25
def test_set_cloud(self):
test_data = 'clearly an english sentence'
self.assertRaises(ConnectionError,
language,
test_data,
cloud='invalid/cloud')
if __name__ == "__main__":
unittest.main()