mirror of
https://github.com/wassname/IndicoIo-python.git
synced 2026-06-27 16:10:34 +08:00
Merge pull request #60 from IndicoDataSolutions/development
Development
This commit is contained in:
Madison May
+2
-1
@@ -22,4 +22,5 @@ v0.5.0, Friday Feb 27 -- Updated to support private cloud, allows for indicorc f
|
||||
v0.5.1, Friday Feb 27 -- More README updates, fixed rst formatting issue, added classifiers
|
||||
v0.5.2, Tue March 7 -- Required API keys, configuration settings
|
||||
v0.5.3, Wed Apr 15 -- Added scipy to requirements, edited Readme to not break pypi page
|
||||
v0.6.0, Thu May 29 -- Remove numpy / scipy dependency in favor of Pillow
|
||||
v0.6.0, Thu May 29 -- Remove numpy / scipy dependency in favor of Pillow
|
||||
v0.7.0, Tue Jun 9 -- Added support for calling multiple APIs in a single function and accepting filenames as image API inputs
|
||||
|
||||
@@ -15,7 +15,7 @@ pip install indicoio
|
||||
|
||||
From source:
|
||||
```bash
|
||||
git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git
|
||||
git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git
|
||||
python setup.py install
|
||||
```
|
||||
|
||||
@@ -62,7 +62,7 @@ Examples
|
||||
>>> text_tags(test_text, top_n=1) # return only keys with top_n values
|
||||
{u'startups_and_entrepreneurship': 0.21888586688354486}
|
||||
|
||||
>>> import numpy as np
|
||||
>>> import numpy as np
|
||||
|
||||
>>> test_face = np.linspace(0,50,48*48).reshape(48,48)
|
||||
|
||||
@@ -85,3 +85,32 @@ Each `indicoio` function has a corresponding batch function for analyzing many e
|
||||
>>> batch_sentiment(['Best day ever', 'Worst day ever'])
|
||||
[0.9899001220871786, 0.005709885173415242]
|
||||
```
|
||||
|
||||
|
||||
Calling multiple APIs with a single function
|
||||
---------
|
||||
There are two multiple API functions `predict_text` and `predict_image`. These functions are similar to the existing api functions, but take in an additional `apis` argument as a list of strings of API names (defaults to all existing apis). `predict_text` accepts a list of existing text APIs and vice versa for `predict_image`. These functions also support batch as the other functions do.
|
||||
|
||||
Accepted text API names: `text_tags, political, sentiment, language`
|
||||
|
||||
Accepted image API names: `fer, facial_features, image_features`
|
||||
|
||||
```python
|
||||
>>> from indicoio import predict_text, predict_image, batch_predict_text, batch_predict_image
|
||||
|
||||
>>> predict_text('Best day ever', apis=["sentiment", "language"])
|
||||
{'sentiment': 0.9899001220871786, 'language': {u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, ...}}
|
||||
|
||||
>>> batch_predict_text(['Best day ever', 'Worst day ever'], apis=["sentiment", "language"])
|
||||
{'sentiment': [0.9899001220871786, 0.005709885173415242], 'language': [{u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, u'Romanian': 0.00010661175919993216, ...}, {u'Swedish': 0.4924352805804646, u'Vietnamese': 0.028574824174911372, u'Romanian': 0.004185623723173551, u'Dutch': 0.000717033819689362, u'Korean': 0.0030093489153785826, ...}]}
|
||||
|
||||
>>> import numpy as np
|
||||
|
||||
>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist()
|
||||
|
||||
>>> predict_image(test_face, apis=["fer", "facial_features"])
|
||||
{'facial_features': [0.0, -0.026176479280200796, 0.20707644777495776, ...], 'fer': {u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}}
|
||||
|
||||
>>> batch_predict_image([test_face, test_face], apis=["fer", "facial_features"])
|
||||
{'facial_features': [[0.0, -0.026176479280200796, 0.20707644777495776, ...], [0.0, -0.026176479280200796, 0.20707644777495776, ...]], 'fer': [{u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}, { u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}]}
|
||||
```
|
||||
|
||||
+41
-10
@@ -18,7 +18,7 @@ From source:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git
|
||||
git clone https://github.com/IndicoDataSolutions/IndicoIo-python.git
|
||||
python setup.py install
|
||||
|
||||
API Keys + Setup
|
||||
@@ -54,8 +54,7 @@ Examples
|
||||
>>> indicoio.config.api_key = "YOUR_API_KEY"
|
||||
|
||||
>>> political("Guns don't kill people. People kill people.")
|
||||
{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657,
|
||||
u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146}
|
||||
{u'Libertarian': 0.47740164630834825, u'Green': 0.08454409540443657, u'Liberal': 0.16617097211030055, u'Conservative': 0.2718832861769146}
|
||||
|
||||
>>> sentiment('Worst movie ever.')
|
||||
0.07062467665597527
|
||||
@@ -71,23 +70,21 @@ Examples
|
||||
>>> text_tags(test_text, top_n=1) # return only keys with top_n values
|
||||
{u'startups_and_entrepreneurship': 0.21888586688354486}
|
||||
|
||||
>>> import numpy as np
|
||||
>>> import numpy as np
|
||||
|
||||
>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist()
|
||||
|
||||
>>> fer(test_face)
|
||||
{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361,
|
||||
u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382}
|
||||
{u'Angry': 0.08843749137458341, u'Sad': 0.39091163159204684, u'Neutral': 0.1947947999669361, u'Surprise': 0.03443785859010413, u'Fear': 0.17574534848440568, u'Happy': 0.11567286999192382}
|
||||
|
||||
>>> facial_features(test_face)
|
||||
[0.0, -0.02568680526917187, 0.21645604230056517, ..., 3.0342637531932777]
|
||||
|
||||
>>> language('Quis custodiet ipsos custodes')
|
||||
{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631,
|
||||
u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...}
|
||||
{u'Swedish': 0.00033330636691921914, u'Lithuanian': 0.007328693814717631, u'Vietnamese': 0.0002686116137658802, u'Romanian': 8.133913804076592e-06, ...}
|
||||
|
||||
Batch API Access
|
||||
----------------
|
||||
Batch API
|
||||
---------
|
||||
|
||||
Each ``indicoio`` function has a corresponding batch function for
|
||||
analyzing many examples with a single request. Simply pass in a list of
|
||||
@@ -100,3 +97,37 @@ inputs and receive a list of results in return.
|
||||
>>> batch_sentiment(['Best day ever', 'Worst day ever'])
|
||||
[0.9899001220871786, 0.005709885173415242]
|
||||
|
||||
Calling multiple APIs with a single function
|
||||
--------------------------------------------
|
||||
|
||||
There are two multiple API functions ``predict_text`` and
|
||||
``predict_image``. These functions are similar to the existing api
|
||||
functions, but take in an additional ``apis`` argument as a list of
|
||||
strings of API names (defaults to all existing apis). ``predict_text``
|
||||
accepts a list of existing text APIs and vice versa for
|
||||
``predict_image``. These functions also support batch as the other
|
||||
functions do.
|
||||
|
||||
Accepted text API names: ``text_tags, political, sentiment, language``
|
||||
|
||||
Accepted image API names: ``fer, facial_features, image_features``
|
||||
|
||||
.. code:: python
|
||||
|
||||
>>> from indicoio import predict_text, predict_image, batch_predict_text, batch_predict_image
|
||||
|
||||
>>> predict_text('Best day ever', apis=["sentiment", "language"])
|
||||
{'sentiment': 0.9899001220871786, 'language': {u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, ...}}
|
||||
|
||||
>>> batch_predict_text(['Best day ever', 'Worst day ever'], apis=["sentiment", "language"])
|
||||
{'sentiment': [0.9899001220871786, 0.005709885173415242], 'language': [{u'Swedish': 0.0022464881013042294, u'Vietnamese': 9.887170914498351e-05, u'Romanian': 0.00010661175919993216, ...}, {u'Swedish': 0.4924352805804646, u'Vietnamese': 0.028574824174911372, u'Romanian': 0.004185623723173551, u'Dutch': 0.000717033819689362, u'Korean': 0.0030093489153785826, ...}]}
|
||||
|
||||
>>> import numpy as np
|
||||
|
||||
>>> test_face = np.linspace(0,50,48*48).reshape(48,48).tolist()
|
||||
|
||||
>>> predict_image(test_face, apis=["fer", "facial_features"])
|
||||
{'facial_features': [0.0, -0.026176479280200796, 0.20707644777495776, ...], 'fer': {u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}}
|
||||
|
||||
>>> batch_predict_image([test_face, test_face], apis=["fer", "facial_features"])
|
||||
{'facial_features': [[0.0, -0.026176479280200796, 0.20707644777495776, ...], [0.0, -0.026176479280200796, 0.20707644777495776, ...]], 'fer': [{u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}, { u'Angry': 0.08877494466353497, u'Sad': 0.3933999409104264, u'Neutral': 0.1910612654566151, u'Surprise': 0.0346146405941845, u'Fear': 0.17682159820518667, u'Happy': 0.11532761017005204}]}
|
||||
|
||||
+7
-14
@@ -1,14 +1,14 @@
|
||||
from functools import partial
|
||||
|
||||
Version, version, __version__, VERSION = ('0.7.0',) * 4
|
||||
|
||||
JSON_HEADERS = {
|
||||
'Content-type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
'client-lib': 'python',
|
||||
'version-number': '0.6.0'
|
||||
'version-number': VERSION
|
||||
}
|
||||
|
||||
Version, version, __version__, VERSION = ('0.6.0',) * 4
|
||||
|
||||
from indicoio.text.sentiment import political, posneg
|
||||
from indicoio.text.sentiment import posneg as sentiment
|
||||
from indicoio.text.lang import language
|
||||
@@ -16,18 +16,11 @@ from indicoio.text.tagging import text_tags
|
||||
from indicoio.images.fer import fer
|
||||
from indicoio.images.features import facial_features
|
||||
from indicoio.images.features import image_features
|
||||
from indicoio.utils.multi import predict_image, predict_text
|
||||
|
||||
apis = [
|
||||
'political',
|
||||
'posneg',
|
||||
'sentiment',
|
||||
'language',
|
||||
'fer',
|
||||
'facial_features',
|
||||
'image_features',
|
||||
'text_tags'
|
||||
]
|
||||
apis = dict((api, globals().get(api)) for api in apis)
|
||||
from indicoio.config import API_NAMES
|
||||
|
||||
apis = dict((api, globals().get(api)) for api in API_NAMES)
|
||||
|
||||
for api in apis:
|
||||
globals()[api] = partial(apis[api])
|
||||
|
||||
@@ -45,11 +45,26 @@ class Settings(ConfigParser.ConfigParser):
|
||||
None
|
||||
)
|
||||
|
||||
TEXT_APIS = [
|
||||
'text_tags',
|
||||
'political',
|
||||
'sentiment',
|
||||
'language'
|
||||
]
|
||||
|
||||
IMAGE_APIS = [
|
||||
'fer',
|
||||
'facial_features',
|
||||
'image_features'
|
||||
]
|
||||
|
||||
API_NAMES = IMAGE_APIS + TEXT_APIS + ["predict_text", "predict_image"]
|
||||
|
||||
SETTINGS = Settings(files=[
|
||||
os.path.expanduser("~/.indicorc"),
|
||||
os.path.join(os.getcwd(), '.indicorc')
|
||||
])
|
||||
|
||||
api_key = SETTINGS.api_key()
|
||||
cloud = SETTINGS.cloud()
|
||||
PUBLIC_API_HOST = 'apiv2.indico.io'
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import requests
|
||||
|
||||
from indicoio.utils import image_preprocess, api_handler
|
||||
from indicoio.utils.image import image_preprocess
|
||||
from indicoio.utils.api import api_handler
|
||||
|
||||
def facial_features(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
@@ -25,7 +26,7 @@ def facial_features(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: List containing feature responses
|
||||
"""
|
||||
image = image_preprocess(image, batch=batch)
|
||||
return api_handler(image, cloud=cloud, api="facialfeatures", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(image, cloud=cloud, api="facialfeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
def image_features(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
@@ -58,4 +59,4 @@ def image_features(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: List containing features
|
||||
"""
|
||||
image = image_preprocess(image, batch=batch, size=(64,64))
|
||||
return api_handler(image, cloud=cloud, api="imagefeatures", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(image, cloud=cloud, api="imagefeatures", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import requests
|
||||
|
||||
from indicoio.utils import api_handler, image_preprocess
|
||||
from indicoio.utils.api import api_handler
|
||||
from indicoio.utils.image import image_preprocess
|
||||
import indicoio.config as config
|
||||
|
||||
def fer(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
@@ -27,4 +28,4 @@ def fer(image, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: Dictionary containing emotion probability pairs
|
||||
"""
|
||||
image = image_preprocess(image, batch=batch)
|
||||
return api_handler(image, cloud=cloud, api="fer", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(image, cloud=cloud, api="fer", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from indicoio.utils import api_handler
|
||||
from indicoio.utils.api import api_handler
|
||||
import indicoio.config as config
|
||||
|
||||
def language(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
@@ -24,4 +24,4 @@ def language(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: Dictionary of language probability pairs
|
||||
"""
|
||||
|
||||
return api_handler(text, cloud=cloud, api="language", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(text, cloud=cloud, api="language", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from indicoio.utils import api_handler
|
||||
from indicoio.utils.api import api_handler
|
||||
|
||||
def political(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
@@ -26,7 +26,7 @@ def political(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: Dictionary of party probability pairs
|
||||
"""
|
||||
|
||||
return api_handler(text, cloud=cloud, api="political", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(text, cloud=cloud, api="political", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
def posneg(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
@@ -49,4 +49,4 @@ def posneg(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: Float
|
||||
"""
|
||||
|
||||
return api_handler(text, cloud=cloud, api="sentiment", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(text, cloud=cloud, api="sentiment", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from indicoio.utils import api_handler
|
||||
from indicoio.utils.api import api_handler
|
||||
import indicoio.config as config
|
||||
|
||||
def text_tags(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
@@ -23,4 +23,4 @@ def text_tags(text, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
:rtype: Dictionary of class probability pairs
|
||||
"""
|
||||
|
||||
return api_handler(text, cloud=cloud, api="texttags", batch=batch, api_key=api_key, **kwargs)
|
||||
return api_handler(text, cloud=cloud, api="texttags", url_params={"batch":batch, "api_key":api_key}, **kwargs)
|
||||
|
||||
+5
-153
@@ -1,45 +1,9 @@
|
||||
import inspect, json, getpass, os.path, base64, StringIO, re, warnings
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from indicoio import JSON_HEADERS
|
||||
from indicoio import config
|
||||
|
||||
B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)")
|
||||
|
||||
|
||||
def api_handler(arg, cloud, api, batch=False, api_key=None, **kwargs):
|
||||
data = {'data': arg}
|
||||
data.update(**kwargs)
|
||||
json_data = json.dumps(data)
|
||||
if not cloud:
|
||||
cloud=config.cloud
|
||||
|
||||
if cloud:
|
||||
host = "%s.indico.domains" % cloud
|
||||
else:
|
||||
# default to indico public cloud
|
||||
host = config.PUBLIC_API_HOST
|
||||
|
||||
if not api_key:
|
||||
api_key = config.api_key
|
||||
|
||||
url = config.url_protocol + "//%s/%s" % (host, api)
|
||||
url = url + "/batch" if batch else url
|
||||
url += "?key=%s" % api_key
|
||||
|
||||
|
||||
response = requests.post(url, data=json_data, headers=JSON_HEADERS)
|
||||
if response.status_code == 503 and cloud != None:
|
||||
raise Exception("Private cloud '%s' does not include api '%s'" % (cloud, api))
|
||||
|
||||
json_results = response.json()
|
||||
results = json_results.get('results', False)
|
||||
if results is False:
|
||||
error = json_results.get('error')
|
||||
raise ValueError(error)
|
||||
return results
|
||||
"""
|
||||
Basic utility classes and functions
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from indicoio.utils.errors import DataStructureException
|
||||
|
||||
class TypeCheck(object):
|
||||
"""
|
||||
@@ -68,118 +32,6 @@ class TypeCheck(object):
|
||||
return check_args
|
||||
|
||||
|
||||
class DataStructureException(Exception):
|
||||
"""
|
||||
If a non-accepted datastructure is passed, throws an exception
|
||||
"""
|
||||
def __init__(self, callback, passed_structure, accepted_structures):
|
||||
self.callback = callback.__name__
|
||||
self.structure = str(type(passed_structure))
|
||||
self.accepted = [str(structure) for structure in accepted_structures]
|
||||
|
||||
def __str__(self):
|
||||
return """
|
||||
function %s does not accept %s, accepted types are: %s
|
||||
""" % (self.callback, self.structure, str(self.accepted))
|
||||
|
||||
|
||||
def image_preprocess(image, size=(48,48), batch=False):
|
||||
"""
|
||||
Takes an image and prepares it for sending to the api including
|
||||
resizing and image data/structure standardizing.
|
||||
"""
|
||||
if batch:
|
||||
return [image_preprocess(img, batch=False) for img in image]
|
||||
|
||||
if isinstance(image, basestring):
|
||||
b64_str = re.sub('^data:image/.+;base64,', '', image)
|
||||
if os.path.isfile(image):
|
||||
# check type of element
|
||||
outImage = Image.open(image)
|
||||
elif B64_PATTERN.match(b64_str) is not None:
|
||||
return b64_str
|
||||
else:
|
||||
raise ValueError("Snose tring provided must be a valid filepath or base64 encoded string")
|
||||
|
||||
elif isinstance(image, list): # image passed in is a list and not np.array
|
||||
warnings.warn(
|
||||
"Input as lists of pixels will be deprecated in the next major update",
|
||||
DeprecationWarning
|
||||
)
|
||||
outImage = process_list_image(image)
|
||||
elif type(image).__name__ == "ndarray": # image is from numpy/scipy
|
||||
out_image = Image.fromarray(image)
|
||||
else:
|
||||
raise ValueError("Image must be a filepath, base64 encoded string, or a numpy array")
|
||||
|
||||
# image resizing
|
||||
outImage = outImage.resize(size)
|
||||
|
||||
# convert to base64
|
||||
temp_output = StringIO.StringIO()
|
||||
outImage.save(temp_output, format='PNG')
|
||||
temp_output.seek(0)
|
||||
output_s = temp_output.read()
|
||||
|
||||
return base64.b64encode(output_s)
|
||||
|
||||
|
||||
def get_list_dimensions(_list):
|
||||
"""
|
||||
Takes a nested list and returns the size of each dimension followed
|
||||
by the element type in the list
|
||||
"""
|
||||
if isinstance(_list, list) or isinstance(_list, tuple):
|
||||
return [len(_list)] + get_list_dimensions(_list[0])
|
||||
return []
|
||||
|
||||
|
||||
def get_element_type(_list, dimens):
|
||||
"""
|
||||
Given the dimensions of a nested list and the list, returns the type of the
|
||||
elements in the inner list.
|
||||
"""
|
||||
elem = _list
|
||||
for _ in xrange(len(dimens)):
|
||||
elem = elem[0]
|
||||
return type(elem)
|
||||
|
||||
|
||||
def process_list_image(_list):
|
||||
"""
|
||||
Processes list to be [[(int, int, int), ...]]
|
||||
"""
|
||||
# Check if list is empty
|
||||
if not _list:
|
||||
return _list
|
||||
|
||||
dimens = get_list_dimensions(_list)
|
||||
data_type = get_element_type(_list, dimens)
|
||||
|
||||
seq_obj = []
|
||||
|
||||
outImage = Image.new("RGB", (dimens[0], dimens[1]))
|
||||
for i in xrange(dimens[0]):
|
||||
for j in xrange(dimens[1]):
|
||||
elem = _list[i][j]
|
||||
if len(dimens) >= 3:
|
||||
#RGB(A)
|
||||
if data_type == float:
|
||||
seq_obj.append((int(elem[0] * 255), int(elem[1] * 255), int(elem[2] * 255)))
|
||||
else:
|
||||
seq_obj.append(elem[0:3])
|
||||
elif data_type == float:
|
||||
#Grayscale 0 - 1.0f
|
||||
seq_obj.append((int(elem * 255), ) * 3)
|
||||
else:
|
||||
#Grayscale 0 - 255
|
||||
seq_obj.append((elem, ) * 3)
|
||||
|
||||
#Needs to be 0 - 255 in flattened list of (R, G, B)
|
||||
outImage.putdata(data = seq_obj)
|
||||
|
||||
return outImage
|
||||
|
||||
|
||||
def is_url(data, batch=False):
|
||||
if batch and isinstance(data[0], basestring):
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""
|
||||
Handles making requests to the IndicoApi Server
|
||||
"""
|
||||
|
||||
import json, requests
|
||||
|
||||
from indicoio.utils.errors import IndicoError, DataStructureException
|
||||
from indicoio import JSON_HEADERS
|
||||
from indicoio import config
|
||||
|
||||
def api_handler(arg, cloud, api, url_params = {"batch":False, "api_key":None}, **kwargs):
|
||||
data = {'data': arg}
|
||||
data.update(**kwargs)
|
||||
json_data = json.dumps(data)
|
||||
if not cloud:
|
||||
cloud=config.cloud
|
||||
|
||||
if cloud:
|
||||
host = "%s.indico.domains" % cloud
|
||||
else:
|
||||
# default to indico public cloud
|
||||
host = config.PUBLIC_API_HOST
|
||||
|
||||
url = config.url_protocol + "//%s/%s" % (host, api)
|
||||
url = url + "/batch" if url_params.get("batch", False) else url
|
||||
url += "?key=%s" % (url_params.get("api_key", None) or config.api_key)
|
||||
if "apis" in url_params:
|
||||
url += "&apis=%s" % ",".join(url_params["apis"])
|
||||
|
||||
response = requests.post(url, data=json_data, headers=JSON_HEADERS)
|
||||
if response.status_code == 503 and cloud != None:
|
||||
raise IndicoError("Private cloud '%s' does not include api '%s'" % (cloud, api))
|
||||
|
||||
json_results = response.json()
|
||||
results = json_results.get('results', False)
|
||||
if results is False:
|
||||
error = json_results.get('error')
|
||||
raise IndicoError(error)
|
||||
return results
|
||||
@@ -0,0 +1,20 @@
|
||||
"""
|
||||
Contains Indico Custom Errors
|
||||
"""
|
||||
|
||||
class IndicoError(ValueError):
|
||||
pass
|
||||
|
||||
class DataStructureException(Exception):
|
||||
"""
|
||||
If a non-accepted datastructure is passed, throws an exception
|
||||
"""
|
||||
def __init__(self, callback, passed_structure, accepted_structures):
|
||||
self.callback = callback.__name__
|
||||
self.structure = str(type(passed_structure))
|
||||
self.accepted = [str(structure) for structure in accepted_structures]
|
||||
|
||||
def __str__(self):
|
||||
return """
|
||||
function %s does not accept %s, accepted types are: %s
|
||||
""" % (self.callback, self.structure, str(self.accepted))
|
||||
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
Image Utils
|
||||
Handles preprocessing images before they are sent to the server
|
||||
"""
|
||||
import os.path, base64, StringIO, re, warnings
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from indicoio.utils.errors import IndicoError, DataStructureException
|
||||
|
||||
B64_PATTERN = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)")
|
||||
|
||||
def image_preprocess(image, size=(48,48), batch=False):
|
||||
"""
|
||||
Takes an image and prepares it for sending to the api including
|
||||
resizing and image data/structure standardizing.
|
||||
"""
|
||||
if batch:
|
||||
return [image_preprocess(img, batch=False) for img in image]
|
||||
|
||||
if isinstance(image, basestring):
|
||||
b64_str = re.sub('^data:image/.+;base64,', '', image)
|
||||
if os.path.isfile(image):
|
||||
# check type of element
|
||||
outImage = Image.open(image)
|
||||
elif B64_PATTERN.match(b64_str) is not None:
|
||||
return b64_str
|
||||
else:
|
||||
raise IndicoError("Snose tring provided must be a valid filepath or base64 encoded string")
|
||||
|
||||
elif isinstance(image, list): # image passed in is a list and not np.array
|
||||
warnings.warn(
|
||||
"Input as lists of pixels will be deprecated in the next major update",
|
||||
DeprecationWarning
|
||||
)
|
||||
outImage = process_list_image(image)
|
||||
elif isinstance(image, Image.Image):
|
||||
outImage = image
|
||||
elif type(image).__name__ == "ndarray": # image is from numpy/scipy
|
||||
out_image = Image.fromarray(image)
|
||||
else:
|
||||
raise IndicoError("Image must be a filepath, base64 encoded string, or a numpy array")
|
||||
|
||||
# image resizing
|
||||
outImage = outImage.resize(size)
|
||||
|
||||
# convert to base64
|
||||
temp_output = StringIO.StringIO()
|
||||
outImage.save(temp_output, format='PNG')
|
||||
temp_output.seek(0)
|
||||
output_s = temp_output.read()
|
||||
|
||||
return base64.b64encode(output_s)
|
||||
|
||||
|
||||
def get_list_dimensions(_list):
|
||||
"""
|
||||
Takes a nested list and returns the size of each dimension followed
|
||||
by the element type in the list
|
||||
"""
|
||||
if isinstance(_list, list) or isinstance(_list, tuple):
|
||||
return [len(_list)] + get_list_dimensions(_list[0])
|
||||
return []
|
||||
|
||||
|
||||
def get_element_type(_list, dimens):
|
||||
"""
|
||||
Given the dimensions of a nested list and the list, returns the type of the
|
||||
elements in the inner list.
|
||||
"""
|
||||
elem = _list
|
||||
for _ in xrange(len(dimens)):
|
||||
elem = elem[0]
|
||||
return type(elem)
|
||||
|
||||
|
||||
def process_list_image(_list):
|
||||
"""
|
||||
Processes list to be [[(int, int, int), ...]]
|
||||
"""
|
||||
# Check if list is empty
|
||||
if not _list:
|
||||
return _list
|
||||
|
||||
dimens = get_list_dimensions(_list)
|
||||
data_type = get_element_type(_list, dimens)
|
||||
|
||||
seq_obj = []
|
||||
|
||||
outImage = Image.new("RGB", (dimens[0], dimens[1]))
|
||||
for i in xrange(dimens[0]):
|
||||
for j in xrange(dimens[1]):
|
||||
elem = _list[i][j]
|
||||
if len(dimens) >= 3:
|
||||
#RGB(A)
|
||||
if data_type == float:
|
||||
seq_obj.append((int(elem[0] * 255), int(elem[1] * 255), int(elem[2] * 255)))
|
||||
else:
|
||||
seq_obj.append(elem[0:3])
|
||||
elif data_type == float:
|
||||
#Grayscale 0 - 1.0f
|
||||
seq_obj.append((int(elem * 255), ) * 3)
|
||||
else:
|
||||
#Grayscale 0 - 255
|
||||
seq_obj.append((elem, ) * 3)
|
||||
|
||||
#Needs to be 0 - 255 in flattened list of (R, G, B)
|
||||
outImage.putdata(data = seq_obj)
|
||||
|
||||
return outImage
|
||||
@@ -0,0 +1,116 @@
|
||||
from indicoio.config import TEXT_APIS, IMAGE_APIS, API_NAMES
|
||||
from indicoio.utils.api import api_handler
|
||||
from indicoio.utils.image import image_preprocess
|
||||
from indicoio.utils.errors import IndicoError
|
||||
|
||||
|
||||
CLIENT_SERVER_MAP = dict((api, api.strip().replace("_", "").lower()) for api in API_NAMES)
|
||||
SERVER_CLIENT_MAP = dict((v, k) for k, v in CLIENT_SERVER_MAP.iteritems())
|
||||
|
||||
def multi(data, type, apis, available, batch=False, **kwargs):
|
||||
"""
|
||||
Helper to make multi requests of different types.
|
||||
|
||||
:param data: data to be sent in JSON.
|
||||
:param type: String type of API request
|
||||
:param apis: List of apis to use.
|
||||
:param apis: List of apis available for use.
|
||||
:type data: str or image
|
||||
:type type: str or unicode
|
||||
:type apis: list of str
|
||||
:type available: list of str
|
||||
:rtype: Dictionary of api responses
|
||||
"""
|
||||
# Client side api name checking - strictly only accept func name api
|
||||
invalid_apis = [api for api in apis if api not in available]
|
||||
if invalid_apis:
|
||||
raise IndicoError("%s are not valid %s APIs. Please reference the available APIs below:\n%s"
|
||||
% (", ".join(invalid_apis), type, ", ".join(available))
|
||||
)
|
||||
# Convert client api names to server names before sending request
|
||||
apis = map(CLIENT_SERVER_MAP.get, apis)
|
||||
result = api_handler(data, url_params = {"apis":apis, "batch":batch}, **kwargs)
|
||||
return handle_response(result)
|
||||
|
||||
def handle_response(result):
|
||||
# Parse out the results to a dicionary of api: result
|
||||
return dict((SERVER_CLIENT_MAP[api], parsed_response(api, res))
|
||||
for api, res in result.iteritems())
|
||||
|
||||
|
||||
def predict_text(input_text, apis=TEXT_APIS, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
Given input text, returns the results of specified text apis. Possible apis
|
||||
include: [ 'text_tags', 'political', 'sentiment', 'language' ]
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> import indicoio
|
||||
>>> text = 'Monday: Delightful with mostly sunny skies. Highs in the low 70s.'
|
||||
>>> results = indicoio.text(data = text, apis = ["language", "sentiment"])
|
||||
>>> language_results = results["langauge"]
|
||||
>>> sentiment_results = results["sentiment"]
|
||||
|
||||
:param text: The text to be analyzed.
|
||||
:param apis: List of apis to use.
|
||||
:type text: str or unicode
|
||||
:type apis: list of str
|
||||
:rtype: Dictionary of api responses
|
||||
"""
|
||||
|
||||
return multi(
|
||||
api="apis",
|
||||
data=input_text,
|
||||
type="text",
|
||||
available = TEXT_APIS,
|
||||
cloud=cloud,
|
||||
batch=batch,
|
||||
api_key=api_key,
|
||||
apis=apis,
|
||||
**kwargs)
|
||||
|
||||
|
||||
def predict_image(image, apis=IMAGE_APIS, cloud=None, batch=False, api_key=None, **kwargs):
|
||||
"""
|
||||
Given input image, returns the results of specified image apis. Possible apis
|
||||
include: ['fer', 'facial_features', 'image_features']
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
>>> import indicoio
|
||||
>>> import numpy as np
|
||||
>>> face = np.zeros((48,48)).tolist()
|
||||
>>> results = indicoio.image(image = face, apis = ["fer", "facial_features"])
|
||||
>>> fer = results["fer"]
|
||||
>>> facial_features = results["facial_features"]
|
||||
|
||||
:param text: The text to be analyzed.
|
||||
:param apis: List of apis to use.
|
||||
:type text: str or unicode
|
||||
:type apis: list of str
|
||||
:rtype: Dictionary of api responses
|
||||
"""
|
||||
|
||||
return multi(
|
||||
api="apis",
|
||||
data=image_preprocess(image, batch=batch),
|
||||
type="image",
|
||||
available=IMAGE_APIS,
|
||||
cloud=cloud,
|
||||
batch=batch,
|
||||
api_key=api_key,
|
||||
apis=apis,
|
||||
**kwargs)
|
||||
|
||||
def parsed_response(api, response):
|
||||
result = response.get('results', False)
|
||||
if result:
|
||||
return result
|
||||
raise IndicoError(
|
||||
"Sorry, the %s API returned an unexpected response.\n\t%s"
|
||||
% (api, response.get('error', ""))
|
||||
)
|
||||
@@ -1,6 +1,8 @@
|
||||
"""
|
||||
Setup for indico apis
|
||||
"""
|
||||
from indicoio import VERSION
|
||||
|
||||
try:
|
||||
from setuptools import setup
|
||||
except ImportError:
|
||||
@@ -8,7 +10,7 @@ except ImportError:
|
||||
|
||||
setup(
|
||||
name="IndicoIo",
|
||||
version='0.6.0',
|
||||
version=VERSION,
|
||||
packages=[
|
||||
"indicoio",
|
||||
"indicoio.text",
|
||||
|
||||
+95
-4
@@ -9,6 +9,8 @@ from indicoio import config
|
||||
from indicoio import political, sentiment, fer, facial_features, language, image_features, text_tags
|
||||
from indicoio import batch_political, batch_sentiment, batch_fer, batch_facial_features
|
||||
from indicoio import batch_language, batch_image_features, batch_text_tags
|
||||
from indicoio import predict_image, predict_text, batch_predict_image, batch_predict_text
|
||||
from indicoio.utils.errors import IndicoError
|
||||
|
||||
DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
@@ -48,7 +50,7 @@ class BatchAPIRun(unittest.TestCase):
|
||||
|
||||
def test_batch_fer_bad_b64(self):
|
||||
test_data = ["$bad#FI jeaf9(#0"]
|
||||
self.assertRaises(ValueError, batch_fer, test_data, api_key=self.api_key)
|
||||
self.assertRaises(IndicoError, batch_fer, test_data, api_key=self.api_key)
|
||||
|
||||
def test_batch_fer_good_b64(self):
|
||||
test_data = ["iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAg5JREFUeNrEV4uNgzAMpegGyAgZgQ3KBscIjMAGx03QEdqbgG5AOwG3AWwAnSCXqLZkuUkwhfYsvaLm5xc7sZ1dIhdtUVjsLZRFTvp+LSaLq8UZ/s+KMSbZCcY5RV9E4QQKHG7QtgeCGv4PFt8WpzkCcztu3TiL0eJgkQmsVFn0MK+LzYkRKEGpG1GDyZdKRdaolhAoJewXnJsO1jtKCFDlChZAFxyJj2PnBRU20KZg7oMlOAENijpi8hwmGkKkZW2GzONtVLA/DxHAhTO2I7MCVBSQ6nGDlEBJDhyVYiUBHXBxzQm0wE4FzPYsGs856dA9SAAP2oENzFYqR6iAFQpHIAUzO/nxnOgthF/lM3w/3U8KYXTwxG/1IgIulF+wPQUXDMl75UoJZIHstRWpaGb8IGYqwBoKlG/lgpzoUEBoj50p8QtVrmHgaaXyC/H3BFC+e9kGFlCB0CtBF7FifQ8D9zjQQHj0pdOM3F1pUBoFKdxtqkMClScHJCSDlSxhHSNRT5K+FaZnHglrz+AGoxZLKNLYH6s3CkkuyJlp58wviZ4PuSCWDXl5hmjZtxcSCGbDUD3gK7EMOZBLCETrgVBF5K0lI5bIZ0wfrYh8NWHIAiNTPHpuTOKpCes1VTFaiNaFdGwPfdmaqlj6LmjJbgoSSfUW74K3voz+/W0oIeB7HWu2s+dfx3N+eLX8CTAAwUmKjK/dHS4AAAAASUVORK5CYII="]
|
||||
@@ -62,9 +64,15 @@ class BatchAPIRun(unittest.TestCase):
|
||||
self.assertTrue(isinstance(response, list))
|
||||
self.assertTrue(isinstance(response[0], dict))
|
||||
|
||||
def test_batch_fer_pil_image(self):
|
||||
test_data = [Image.open(os.path.normpath(os.path.join(DIR, "data/fear.png")))]
|
||||
response = batch_fer(test_data, api_key=self.api_key)
|
||||
self.assertTrue(isinstance(response, list))
|
||||
self.assertTrue(isinstance(response[0], dict))
|
||||
|
||||
def test_batch_fer_nonexistant_filepath(self):
|
||||
test_data = ["data/unhappy.png"]
|
||||
self.assertRaises(ValueError, batch_fer, test_data, api_key=self.api_key)
|
||||
self.assertRaises(IndicoError, batch_fer, test_data, api_key=self.api_key)
|
||||
|
||||
|
||||
def test_batch_facial_features(self):
|
||||
@@ -107,6 +115,45 @@ class BatchAPIRun(unittest.TestCase):
|
||||
self.assertTrue(isinstance(response, list))
|
||||
self.assertTrue(response[0]['English'] > 0.25)
|
||||
|
||||
def test_batch_multi_api_image(self):
|
||||
test_data = [generate_array((48,48)), generate_int_array((48,48))]
|
||||
response = batch_predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS))
|
||||
self.assertTrue(isinstance(response["fer"], list))
|
||||
|
||||
def test_batch_multi_api_text(self):
|
||||
test_data = ['clearly an english sentence']
|
||||
response = batch_predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(set(response.keys()) == set(config.TEXT_APIS))
|
||||
|
||||
def test_default_multi_api_text(self):
|
||||
test_data = ['clearly an english sentence']
|
||||
response = batch_predict_text(test_data, api_key=self.api_key)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(set(response.keys()) == set(config.TEXT_APIS))
|
||||
|
||||
def test_multi_api_bad_api(self):
|
||||
self.assertRaises(IndicoError,
|
||||
batch_predict_text,
|
||||
"this shouldn't work",
|
||||
apis=["sentiment", "somethingbad"])
|
||||
|
||||
def test_multi_bad_mixed_api(self):
|
||||
self.assertRaises(IndicoError,
|
||||
predict_text,
|
||||
"this shouldn't work",
|
||||
apis=["fer", "sentiment", "facial_features"])
|
||||
def test_batch_multi_bad_mixed_api(self):
|
||||
self.assertRaises(IndicoError,
|
||||
batch_predict_text,
|
||||
["this shouldn't work"],
|
||||
apis=["fer", "sentiment", "facial_features"])
|
||||
|
||||
def test_batch_set_cloud(self):
|
||||
test_data = ['clearly an english sentence']
|
||||
self.assertRaises(ConnectionError,
|
||||
@@ -118,6 +165,9 @@ class BatchAPIRun(unittest.TestCase):
|
||||
|
||||
class FullAPIRun(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.api_key = config.api_key
|
||||
|
||||
def load_image(self, relpath, as_grey=False):
|
||||
im = Image.open(os.path.normpath(os.path.join(DIR, relpath))).convert('L');
|
||||
pixels = list(im.getdata())
|
||||
@@ -178,12 +228,26 @@ class FullAPIRun(unittest.TestCase):
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertEqual(fer_set, set(response.keys()))
|
||||
|
||||
def test_good_int_array_fer(self):
|
||||
fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy'])
|
||||
test_face = generate_int_array((48,48))
|
||||
response = fer(test_face)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertEqual(fer_set, set(response.keys()))
|
||||
|
||||
def test_happy_fer(self):
|
||||
test_face = self.load_image("data/happy.png", as_grey=True)
|
||||
response = fer(test_face)
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(response['Happy'] > 0.5)
|
||||
|
||||
def test_happy_fer_pil(self):
|
||||
test_face = Image.open(os.path.normpath(os.path.join(DIR, "data/happy.png"))).convert('L');
|
||||
response = fer(test_face)
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(response['Happy'] > 0.5)
|
||||
|
||||
def test_fear_fer(self):
|
||||
test_face = self.load_image("data/fear.png", as_grey=True)
|
||||
response = fer(test_face)
|
||||
@@ -206,6 +270,15 @@ class FullAPIRun(unittest.TestCase):
|
||||
self.assertEqual(len(response), 48)
|
||||
self.check_range(response)
|
||||
|
||||
def test_good_int_array_facial_features(self):
|
||||
fer_set = set(['Angry', 'Sad', 'Neutral', 'Surprise', 'Fear', 'Happy'])
|
||||
test_face = generate_int_array((48,48))
|
||||
response = facial_features(test_face)
|
||||
|
||||
self.assertTrue(isinstance(response, list))
|
||||
self.assertEqual(len(response), 48)
|
||||
self.check_range(response)
|
||||
|
||||
# TODO: uncomment this test once the remote server is updated to
|
||||
# deal with image_urls
|
||||
# def test_image_url(self):
|
||||
@@ -232,6 +305,21 @@ class FullAPIRun(unittest.TestCase):
|
||||
self.assertEqual(len(response), 2048)
|
||||
self.check_range(response)
|
||||
|
||||
def test_multi_api_image(self):
|
||||
test_data = generate_array((48,48))
|
||||
response = predict_image(test_data, apis=config.IMAGE_APIS, api_key=self.api_key)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(set(response.keys()) == set(config.IMAGE_APIS))
|
||||
|
||||
def test_multi_api_text(self):
|
||||
test_data = 'clearly an english sentence'
|
||||
response = predict_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key)
|
||||
|
||||
self.assertTrue(isinstance(response, dict))
|
||||
self.assertTrue(set(response.keys()) == set(config.TEXT_APIS))
|
||||
|
||||
|
||||
def test_language(self):
|
||||
language_set = set([
|
||||
'English',
|
||||
@@ -297,7 +385,7 @@ class FullAPIRun(unittest.TestCase):
|
||||
|
||||
def test_set_api_key(self):
|
||||
test_data = 'clearly an english sentence'
|
||||
self.assertRaises(ValueError,
|
||||
self.assertRaises(IndicoError,
|
||||
language,
|
||||
test_data,
|
||||
api_key ='invalid_api_key')
|
||||
@@ -306,7 +394,7 @@ class FullAPIRun(unittest.TestCase):
|
||||
config.api_key = 'invalid_api_key'
|
||||
|
||||
self.assertEqual(config.api_key, 'invalid_api_key')
|
||||
self.assertRaises(ValueError,
|
||||
self.assertRaises(IndicoError,
|
||||
language,
|
||||
test_data)
|
||||
|
||||
@@ -323,6 +411,9 @@ def flatten(container):
|
||||
def generate_array(size):
|
||||
return [[random.random() for _ in xrange(size[0])] for _ in xrange(size[1])]
|
||||
|
||||
def generate_int_array(size):
|
||||
return [[random.randint(0, 50) for _ in xrange(size[0])] for _ in xrange(size[1])]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user