wassname/pyrobolearn
1
0
Fork 0
mirror of https://github.com/wassname/pyrobolearn.git synced 2026-09-09 11:31:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

update interfaces and add interface examples (speech, openpose, asus xtion)

Browse Source
This commit is contained in:
Brian Delhaisse
2019-06-22 00:02:33 +02:00
parent fb35ddd6dc
commit 7b876780fc
12 changed files with 570 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/interfaces/README.md
+8 -3
View File
@@ -4,8 +4,13 @@ In this folder, you will find examples on what interfaces you can use and on how
You will also be able to connect an interface with an element of the world (in this case, a robot) using bridges,
and see that different bridges can lead to different behaviors while getting the data from the same interface.
Here are few examples that depict the various interfaces:
Here are few examples that depict the various interfaces available to the user:
1. `mouse_keyboard.py`: use the mouse keyboard interface
2. `webcam.py`: use the webcam interface
3. `playstation.py`: use the Playstation joystick controller interface
4. `xbox.py`: use the Xbox joystick controller interface
3. `playstation.py`: use the Playstation game controller interface
4. `xbox.py`: use the Xbox game controller interface
5. `asus_xtion.py`: use the Asus Xtion interface
6. `openpose.py`: use the Openpose interface
7. `speech.py`: use the speech (recognizer/translator) interface
**Note**: some interfaces require to install different libraries, and possibly to configure them. Check the raised errors.
examples/interfaces/asus_xtion.py
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python
"""Run the Asus Xtion interface.
Make sure that the `openni` library is installed with all the correct environment variables set, and that the Asus
Xtion is connected before running this code. Note that it can take some time to initialize the interface.
"""
import argparse
import matplotlib.pyplot as plt
from pyrobolearn.tools.interfaces.camera.asus_xtion import AsusXtionInterface
# create parser
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool,
default=False)
parser.add_argument('-r', '--use_rgb', help='If we should get RGB images. Note that RGB and IR images can not be '
'captured at the same time.', type=bool,
default=True)
parser.add_argument('-d', '--use_depth', help='If we should get depth images.', type=bool,
default=True)
parser.add_argument('-i', '--use_ir', help='If we should get IR images. Note that RGB and IR images can not be '
'captured at the same time.', type=bool,
default=False)
args = parser.parse_args()
# get which pictures to capture
use_rgb, use_depth, use_ir = args.use_rgb, args.use_depth, args.use_ir
if use_rgb and use_ir:
use_ir = False
# create Asus Xtion interface
interface = AsusXtionInterface(use_rgb=use_rgb, use_depth=use_depth, use_ir=use_ir)
# plotting using matplotlib in interactive mode
fig, axes = plt.subplots(1, 2)
plots = [None]*2
titles = []
if use_rgb:
titles.append('RGB')
if use_ir:
titles.append('IR')
if use_depth:
titles.append('Depth')
plt.ion() # interactive mode on
while True:
# if don't use thread call `step` or `run`
data = interface.run()
# get the frame and plot it with matplotlib
if plots[0] is None:
for i in range(len(plots)):
plots[i] = axes[i].imshow(data[i])
axes[i].set_title(titles[i])
else:
for plot, img in zip(plots, data):
plot.set_data(img)
# pause a bit
plt.pause(0.01)
# check if the figure is closed, and if so, get out of the loop
if not plt.fignum_exists(fig.number):
break
plt.ioff() # interactive mode off
plt.show()
examples/interfaces/openpose.py
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python
"""Run the Openpose interface.
Make sure that the webcam is connected, and that the openpose framework has been installed before running this code.
For this code to work, you have to specify the path to the openpose framework, or set the `OPENPOSE_PATH` environment
variable. Note that it can take some time to initialize the interface.
"""
import cv2
import argparse
from pyrobolearn.tools.interfaces.camera.openpose import OpenPoseInterface
# create parser
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--path', help='Absolute path to the openpose framework. If not specified, it will check '
'for the environment variable `OPENPOSE_PATH`.', type=str, default='')
parser.add_argument('-t', '--use_thread', help='If we should run the openpose interface in a thread.', type=bool,
default=False)
parser.add_argument('-f', '--detect_face', help='If we should detect the face with openpose.', type=bool,
default=True)
parser.add_argument('-a', '--detect_hands', help='If we should detect the hands with openpose.', type=bool,
default=False)
args = parser.parse_args()
path = None if args.path == '' else args.path
# create openpose interface
if args.use_thread:
# create and run interface in a thread
interface = OpenPoseInterface(openpose_path=path, detect_face=args.detect_face, detect_hands=args.detect_hands,
use_thread=True, sleep_dt=1. / 10, verbose=True)
raw_input('Press key to stop the openpose interface')
else:
# create interface
interface = OpenPoseInterface(openpose_path=path, detect_face=args.detect_face, detect_hands=args.detect_hands)
# run interface
while True:
frame, keypoints = interface.run()
cv2.imshow("OpenPose 1.4.0 - Tutorial Python API", frame)
# quit display if 'esc' button is pressed
key = cv2.waitKey(15) & 0xFF
if key == 27:
cv2.destroyWindow('frame')
examples/interfaces/speech.py
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python
"""Run the speech interface.
This will perform speech recognition, translation, and synthesization. Make sure that your computer has a microphone
connected.
In the future, interfaces using the Google assistant, Alexa, or a similar tool will be implemented.
"""
import argparse
from pyrobolearn.tools.interfaces.audio.speech import SpeechRecognizerInterface, SpeechTranslatorInterface
# get the available languages.
languages = SpeechRecognizerInterface.available_languages
print("Available languages are: {}".format(languages))
# create parser
parser = argparse.ArgumentParser()
# parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool,
# default=False)
parser.add_argument('-l', '--lang', help='The language that needs to be recognized.', type=str, choices=languages,
default='english')
parser.add_argument('-a', '--target_lang', help='If we should get depth images.', type=str, choices=languages,
default='english')
args = parser.parse_args()
# create speech recognizer/translator interface
# interface = SpeechRecognizerInterface(verbose=True, lang=args.lang)
interface = SpeechTranslatorInterface(verbose=True, from_lang=args.lang, target_lang=args.target_lang)
# run the interface
while True:
data = interface.run()
examples/interfaces/webcam.py
+41 -23
View File
@@ -1,34 +1,52 @@
#!/usr/bin/env python
"""Load the Webcam interface.
"""Run the Webcam interface.
Make sure that the webcam is connected before running this code. Note that it can take some time to initialize
the interface.
"""
from itertools import count
import argparse
import matplotlib.pyplot as plt
from pyrobolearn.tools.interfaces.camera.webcam import WebcamInterface
# create interface
interface = WebcamInterface(use_thread=True, sleep_dt=1./10, verbose=False)
# plotting using matplotlib in interactive mode
fig = plt.figure()
plot = None
plt.ion() # interactive mode on
# create parser
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool,
default=False)
args = parser.parse_args()
for _ in count():
# # if don't use thread call `step` or `run` (note that `run` returns the frame but not
# interface.step()
# get the frame and plot it with matplotlib
frame = interface.frame
if plot is None:
plot = plt.imshow(frame)
else:
plot.set_data(frame)
plt.pause(0.01)
# create webcam interface
if args.use_thread:
# create and run interface in a thread
interface = WebcamInterface(use_thread=True, sleep_dt=1./10, verbose=True)
raw_input('Press key to stop the webcam interface')
else:
# create interface
interface = WebcamInterface()
# check if the figure is closed, and if so, get out of the loop
if not plt.fignum_exists(fig.number):
break
# plotting using matplotlib in interactive mode
fig = plt.figure()
plot = None
plt.ion() # interactive mode on
plt.ioff() # interactive mode off
plt.show()
while True:
# if don't use thread call `step` or `run` (note that `run` returns the frame but not
interface.step()
# get the frame and plot it with matplotlib
frame = interface.frame
if plot is None:
plot = plt.imshow(frame)
else:
plot.set_data(frame)
plt.pause(0.01)
# check if the figure is closed, and if so, get out of the loop
if not plt.fignum_exists(fig.number):
break
plt.ioff() # interactive mode off
plt.show()
pyrobolearn/tools/interfaces/audio/alexa.py
+2
View File
@@ -0,0 +1,2 @@
# TODO
# Ref: https://developer.amazon.com/en-US/alexa/alexa-skills-kit/alexa-skill-python-tutorial
pyrobolearn/tools/interfaces/audio/audio.py
+2 -298
View File
@@ -1,10 +1,8 @@
#!/usr/bin/env python
"""Define the main basic Camera interface
This defines the main basic camera interface from which all other interfaces which uses a camera inherit from.
"""Define the main basic audio interface.
"""
import os
# TODO
from pyrobolearn.tools.interfaces.interface import Interface, InputInterface, OutputInterface, InputOutputInterface
@@ -19,59 +17,6 @@ except ImportError as e:
"pip install pyaudio\n"
raise ImportError(e.__str__() + string)
# Speech recognition
# Good tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones
try:
import speech_recognition as sr
except ImportError as e:
string = "\nHint: try to install speech_recognition by typing the following lines in the terminal: \n" \
"sudo apt-get install libpulse-dev" \
"pip install pocketsphinx\n" \
"pip install google-cloud-speech" \
"pip install SpeechRecognition"
raise ImportError(e.__str__() + string)
# Speech synthesis
# Good tutorial: https://pythonprogramminglanguage.com/text-to-speech/
# If Python3.3 or higher: https://pypi.org/project/google_speech/
try:
from gtts import gTTS
except ImportError as e:
string = "\nHint: try to install gTTS by typing: pip install gTTS"
raise ImportError(e.__str__() + string)
# # Another one is `pyttsx3`, which is the best offline module (the problem is that it only supports english)
# # Documentation (with examples): pyttsx3.readthedocs.io/en/latest/
# try:
# import pyttsx3
# except ImportError as e:
# string = "\nHint: try to install pyttsx3 by typing: pip install pyttsx3"
# raise ImportError(e.__str__() + string)
# what I also checked: `pyttsx` and `pyvona`
# import pyttsx
# import pyvona
# Translation
# Github repo: https://github.com/ssut/py-googletrans
# Tutorial: https://www.codeproject.com/Tips/1236705/How-to-Use-Google-Translator-in-Python
try:
from googletrans import Translator
except ImportError as e:
string = "\nHint: try to install googletrans by typing: pip install googletrans"
raise ImportError(e.__str__() + string)
# ChatterBot
# Github repo: https://github.com/gunthercox/ChatterBot
# Documentation: https://chatterbot.readthedocs.io/en/stable
# try:
# from chatterbot import ChatBot
# except ImportError as e:
# string = "\nHint: try to install chatterbot by typing: pip install chatterbot"
# raise ImportError(e.__str__() + string)
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
@@ -163,248 +108,7 @@ class InputOutputAudioInterface(InputOutputInterface):
pass
class SpeechRecognizerInterface(InputInterface):
r"""Speech Recognizer Interface
References:
[1] Tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones
"""
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech recognizer input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone() # device_index=-1
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.lang = languages[lang]
# string that is being said
self.data = ''
super(SpeechRecognizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# listen to speech through the microphone
with self.microphone as source:
self.recognizer.adjust_for_ambient_noise(source)
print("I am listening...")
audio = self.recognizer.listen(source) # listen to what is being said
# recognize speech (get the string from audio)
try:
print('Trying to understand what you just said...')
self.data = self.recognizer.recognize_google(audio, language=self.lang)
except sr.UnknownValueError:
print("Unable to recognize speech")
except sr.RequestError as e:
print("API unavailable".format(e))
if self.verbose:
print("You said: {}".format(self.data))
class SpeechSynthesizerInterface(OutputInterface):
r"""Speech Synthesizer Interface
References:
[1] tutorial: https://pythonprogramminglanguage.com/text-to-speech/
[2] If Python3.3 or higher: https://pypi.org/project/google_speech/
"""
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech synthesizer output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.lang = languages[lang]
self.updated = False
self.data = ''
super(SpeechSynthesizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
if self.updated:
# tts = text-to-speech
tts = gTTS(text=self.data, lang=self.lang)
tts.save('tmp.mp3')
os.system('mpg321 tmp.mp3')
os.system('rm tmp.mp3')
self.updated = False
def update(self, data):
"""Update the data."""
self.data = data
self.updated = True
class SpeechTranslatorInterface(InputOutputInterface):
r"""Speech Translator Interface
"""
def __init__(self, use_thread=False, target_lang='english', from_lang='auto', verbose=False):
"""
Initialize the speech translator input/output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.translator = Translator()
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR',
'auto': 'auto'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.target_lang = languages[target_lang]
self.from_lang = languages[from_lang]
self.updated = False
self.input_data = ''
self.data = ''
super(SpeechTranslatorInterface, self).__init__(use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# translate
if self.updated and self.target_lang != self.from_lang:
translated = self.translator.translate(self.input_data, dest=self.target_lang, src=self.from_lang)
self.data = translated.text
self.updated = False
def update(self, data):
"""Update the data."""
self.input_data = data
self.updated = True
class SpeechInterface(InputOutputInterface):
r"""Speech Interface
This class performs speech recognition, translation, and synthesization.
"""
def __init__(self, use_thread=False, target_lang='english', from_lang='english', verbose=False):
"""
Initialize the speech (recognizer, translator, and synthesizer) interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = SpeechRecognizerInterface(use_thread=False, lang=from_lang)
self.translator = None
if target_lang != from_lang:
self.translator = SpeechTranslatorInterface(use_thread=False, target_lang=target_lang,
from_lang=from_lang)
self.synthesizer = SpeechSynthesizerInterface(use_thread=False, lang=target_lang)
self.updated = False
self.input_data = ''
self.output_data = ''
super(SpeechInterface, self).__init__(use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
pass
def update(self, data):
"""Update the interface."""
pass
# Tests
if __name__ == '__main__':
interface = AudioInterface()
interface.print_info()
# recognize, translate and synthesize speech
english = set(['en', 'en-US', 'en-GB'])
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'}
# Check < https://gist.github.com/traysr/2001377 > for more
lang = languages['english']
recognizer = sr.Recognizer()
microphone = sr.Microphone() # device_index=-1
# print(microphone.list_microphone_names())
# listen to speech through the microphone
with microphone as source:
recognizer.adjust_for_ambient_noise(source)
print("Say something!")
audio = recognizer.listen(source)
# recognize speech
print('processing...')
string = ''
try:
string = recognizer.recognize_google(audio, language=lang)
except sr.UnknownValueError:
print("Unable to recognize speech")
except sr.RequestError as e:
print("API unavailable".format(e))
print("You said: " + string)
# translate it if other language than english
if lang not in english:
translator = Translator()
translated = translator.translate(string) # dest='en', src='auto')
print("which translates to: " + translated.text)
# produce speech
print('Let me try to repeat what you just said:')
tts = gTTS(text=string, lang=lang)
tts.save('tmp.mp3')
os.system('mpg321 tmp.mp3')
os.system('rm tmp.mp3')
# recognize speech using Sphinx
# try:
# print("Sphinx thinks you said '" + recognizer.recognize_sphinx(audio) + "'")
# except sr.UnknownValueError:
# print("Sphinx could not understand audio")
# except sr.RequestError as e:
# print("Sphinx error; {0}".format(e))
pyrobolearn/tools/interfaces/audio/google_assistant.py
+2
View File
@@ -0,0 +1,2 @@
# TODO
# Ref: https://developers.google.com/assistant/sdk/overview
pyrobolearn/tools/interfaces/audio/speech.py
+336
View File
@@ -0,0 +1,336 @@
#!/usr/bin/env python
"""Define the various speech interfaces allowing to perform speech recognition, translation, and synthesization.
"""
# TODO: chatbox, google assistant, alexa
import os # TODO: use subprocess instead
# import subprocess
from pyrobolearn.tools.interfaces.interface import InputInterface, OutputInterface, InputOutputInterface
# Speech recognition
# Good tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones
try:
import speech_recognition as sr
except ImportError as e:
string = "\nHint: try to install speech_recognition by typing the following lines in the terminal: \n" \
"sudo apt-get install libpulse-dev" \
"pip install pocketsphinx\n" \
"pip install google-cloud-speech" \
"pip install SpeechRecognition"
raise ImportError(e.__str__() + string)
# Speech synthesis
# Good tutorial: https://pythonprogramminglanguage.com/text-to-speech/
# If Python3.3 or higher: https://pypi.org/project/google_speech/
try:
from gtts import gTTS
except ImportError as e:
string = "\nHint: try to install gTTS by typing: `pip install gTTS`." \
"Also install `mpg321` using `sudo apt-get install mpg321`."
raise ImportError(e.__str__() + string)
# # Another one is `pyttsx3`, which is the best offline module (the problem is that it only supports english)
# # Documentation (with examples): pyttsx3.readthedocs.io/en/latest/
# try:
# import pyttsx3
# except ImportError as e:
# string = "\nHint: try to install pyttsx3 by typing: pip install pyttsx3"
# raise ImportError(e.__str__() + string)
# what I also checked: `pyttsx` and `pyvona`
# import pyttsx
# import pyvona
# Translation
# Github repo: https://github.com/ssut/py-googletrans
# Tutorial: https://www.codeproject.com/Tips/1236705/How-to-Use-Google-Translator-in-Python
try:
from googletrans import Translator
except ImportError as e:
string = "\nHint: try to install googletrans by typing: pip install googletrans"
raise ImportError(e.__str__() + string)
# ChatterBot
# Github repo: https://github.com/gunthercox/ChatterBot
# Documentation: https://chatterbot.readthedocs.io/en/stable
# try:
# from chatterbot import ChatBot
# except ImportError as e:
# string = "\nHint: try to install chatterbot by typing: pip install chatterbot"
# raise ImportError(e.__str__() + string)
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "GNU GPLv3"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class SpeechRecognizerInterface(InputInterface):
r"""Speech Recognizer Interface
References:
[1] Tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones
"""
available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian',
'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese',
'mandarin', 'polish', 'portuguese', 'russian', 'greek'}
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech recognizer input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone() # device_index=-1
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.lang = languages[lang]
# string that is being said
self.data = ''
super(SpeechRecognizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# listen to speech through the microphone
with self.microphone as source:
self.recognizer.adjust_for_ambient_noise(source)
if self.verbose:
print("Say something, I am listening!")
audio = self.recognizer.listen(source) # listen to what is being said
# recognize speech (get the string from audio)
try:
if self.verbose:
print('Please wait, trying to understand what you just said...')
self.data = self.recognizer.recognize_google(audio, language=self.lang)
if self.verbose:
print("You said: {}".format(''.join(self.data).encode('utf-8')))
return self.data
except sr.UnknownValueError:
print("Unable to recognize speech")
except sr.RequestError as e:
print("API unavailable".format(e))
return None
class SpeechSynthesizerInterface(OutputInterface):
r"""Speech Synthesizer Interface
References:
[1] tutorial: https://pythonprogramminglanguage.com/text-to-speech/
[2] If Python3.3 or higher: https://pypi.org/project/google_speech/
"""
available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian',
'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese',
'mandarin', 'polish', 'portuguese', 'russian', 'greek'}
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech synthesizer output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.lang = languages[lang]
self.updated = False
self._data = ''
super(SpeechSynthesizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
@property
def data(self):
return self._data
@data.setter
def data(self, data):
if isinstance(data, (str, unicode)):
self._data = data
self.updated = True
def run(self):
"""Run the interface."""
if self.updated:
if self.verbose:
print("Generating speech...")
# tts = text-to-speech
tts = gTTS(text=self.data, lang=self.lang)
tts.save('tmp.mp3')
os.system('mpg321 tmp.mp3 > /dev/null 2>&1') # TODO: use subprocess instead
os.system('rm tmp.mp3')
# subprocess.call(['mpg321 tmp.mp3'])
# subprocess.call(['rm tmp.mp3'])
self.updated = False
print("Speech generated!")
return self.data
def update(self, data):
"""Update the data."""
self.data = data
class TranslatorInterface(InputOutputInterface):
r"""Text Translator Interface
"""
available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian',
'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese',
'mandarin', 'polish', 'portuguese', 'russian', 'greek'}
def __init__(self, use_thread=False, target_lang='english', from_lang='auto', verbose=False):
"""
Initialize the speech translator input/output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.translator = Translator()
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR',
'auto': 'auto'}
# Check < https://gist.github.com/traysr/2001377 > for more
self.target_lang = languages[target_lang]
self.from_lang = languages[from_lang]
self.updated = False
self._input_data = ''
self.data = ''
super(TranslatorInterface, self).__init__(use_thread, verbose=verbose)
@property
def input_data(self):
return self._input_data
@input_data.setter
def input_data(self, data):
if isinstance(data, (str, unicode)):
self._input_data = data
self.updated = True
def run(self):
"""Run the interface."""
# translate
if self.updated and self.target_lang != self.from_lang:
if self.verbose:
print("Translating: {}".format(''.join(self.input_data).encode('utf-8')))
translated = self.translator.translate(self.input_data, dest=self.target_lang, src=self.from_lang)
self.data = translated.text
if self.verbose:
print("Translated text: {}".format(''.join(self.data).encode('utf-8')))
self.updated = False
return self.data
def update(self, data):
"""Update the data."""
self.input_data = data
class SpeechTranslatorInterface(InputOutputInterface):
r"""Speech Translator Interface
This class performs speech recognition, translation, and synthesization.
"""
available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian',
'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese',
'mandarin', 'polish', 'portuguese', 'russian', 'greek'}
def __init__(self, use_thread=False, target_lang='english', from_lang='english', verbose=False):
"""
Initialize the speech (recognizer, translator, and synthesizer) interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = SpeechRecognizerInterface(use_thread=False, lang=from_lang, verbose=verbose)
self.translator = None
if target_lang != from_lang:
self.translator = TranslatorInterface(use_thread=False, target_lang=target_lang,
from_lang=from_lang, verbose=verbose)
self.synthesizer = SpeechSynthesizerInterface(use_thread=False, lang=target_lang, verbose=verbose)
self.updated = False
self.input_data = ''
self.output_data = ''
super(SpeechTranslatorInterface, self).__init__(use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# run the recognizer
self.input_data = self.recognizer.run()
# run the translator
if self.translator is not None:
self.translator.input_data = self.input_data
self.output_data = self.translator.run()
else:
self.output_data = self.input_data
# run the synthesizer
if self.input_data is not None:
self.synthesizer.data = self.output_data
self.synthesizer.run()
return self.output_data
# Tests
if __name__ == '__main__':
print("Available languages are: {}".format(SpeechRecognizerInterface.available_languages))
# interface = SpeechRecognizerInterface(verbose=True, lang='english')
interface = SpeechTranslatorInterface(verbose=True, from_lang='french', target_lang='english')
while True:
data = interface.run()
pyrobolearn/tools/interfaces/camera/asus_xtion.py
+1 -1
View File
@@ -216,7 +216,7 @@ if __name__ == '__main__':
use_depth=use_depth, use_ir=use_ir)
# plotting using matplotlib in interactive mode
fig, axes = plt.subplots(1,2)
fig, axes = plt.subplots(1, 2)
plots = [None]*2
titles = []
if use_rgb:
pyrobolearn/tools/interfaces/camera/openpose.py
+14 -15
View File
@@ -36,10 +36,10 @@ class OpenPoseInterface(CameraInterface):
3D) pictures and map them to the human kinematic skeleton.
References:
[1] OpenPose: github.com/CMU-Perceptual-Computing-Lab/openpose
[2] PyOpenPose (official): github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/modules/python_module.md
[3] OpenPose output format: github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md
[4] PyOpenPose (python wrappers): github.com/FORTH-ModelBasedTracker/PyOpenPose
- [1] OpenPose: github.com/CMU-Perceptual-Computing-Lab/openpose
- [2] PyOpenPose (official): github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/modules/python_module.md
- [3] OpenPose output format: github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md
- [4] PyOpenPose (python wrappers): github.com/FORTH-ModelBasedTracker/PyOpenPose
"""
def __init__(self, camera=None, detect_face=False, detect_hands=False, openpose_path=None,
@@ -60,6 +60,14 @@ class OpenPoseInterface(CameraInterface):
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
# Check the path to the openpose folder (which contains various models and test images)
if openpose_path is None:
if 'OPENPOSE_PATH' not in os.environ:
raise ValueError("The OPENPOSE_PATH environment variable has not been set properly. Please "
"then provide the path to the openpose folder by specifying the `openpose_path` "
"argument")
openpose_path = os.environ['OPENPOSE_PATH']
self.openpose_path = openpose_path
# save variables
self.detect_face = detect_face
@@ -108,18 +116,9 @@ class OpenPoseInterface(CameraInterface):
['Mouth' + str(i-48) for i in range(48, 68)] + ['REye6'] + ['LEye6']
self.face_joint_names_to_ids = dict(zip(self.face_joints, range(len(self.face_joints))))
# Check the path to the openpose folder (which contains various models and test images)
if openpose_path is None:
if 'OPENPOSE_PATH' not in os.environ:
raise ValueError("The OPENPOSE_PATH environment variable has not been set properly. Please "
"then provide the path to the openpose folder by specifying the `openpose_path` "
"argument")
openpose_path = os.environ['OPENPOSE_PATH']
self.openpose_path = openpose_path
# specify the parameters
params = dict()
params["model_folder"] = path + "models/"
params["model_folder"] = openpose_path + "/models/"
if detect_face:
params["face"] = True
if detect_hands:
@@ -243,7 +242,7 @@ class OpenPoseInterface(CameraInterface):
# Tests
if __name__ == '__main__':
path = '/home/brian/repos/openpose/'
interface = OpenPoseInterface(openpose_path=path) # , use_thread=False, sleep_dt=1./10, verbose=True)
interface = OpenPoseInterface(openpose_path=path) # use_thread=True, sleep_dt=1./10, verbose=True)
while True:
frame, keypoints = interface.run()
pyrobolearn/tools/interfaces/interface.py
+8 -1
View File
@@ -55,6 +55,8 @@ class Interface(object):
VR/AR tools, phones, etc. For more info, see the `InputOutputInterface` class.
"""
use_thread = False
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the interface.
@@ -72,6 +74,8 @@ class Interface(object):
self.data = None
self.verbose = verbose
self.stop_thread = False
if self.use_thread:
self.thread = threading.Thread(target=self._run)
self.thread.start()
@@ -89,6 +93,8 @@ class Interface(object):
"""
if self.use_thread:
while True:
if self.stop_thread: # if the thread should stop
break
self.run(*args, **kwargs)
time.sleep(self.dt)
else:
@@ -105,7 +111,8 @@ class Interface(object):
"""
Stop and close the interface.
"""
pass
if self.use_thread:
self.stop_thread = True
#############
# Operators #