diff --git a/examples/interfaces/README.md b/examples/interfaces/README.md index 5ca8360..4bf31cc 100644 --- a/examples/interfaces/README.md +++ b/examples/interfaces/README.md @@ -4,8 +4,13 @@ In this folder, you will find examples on what interfaces you can use and on how You will also be able to connect an interface with an element of the world (in this case, a robot) using bridges, and see that different bridges can lead to different behaviors while getting the data from the same interface. -Here are few examples that depict the various interfaces: +Here are few examples that depict the various interfaces available to the user: 1. `mouse_keyboard.py`: use the mouse keyboard interface 2. `webcam.py`: use the webcam interface -3. `playstation.py`: use the Playstation joystick controller interface -4. `xbox.py`: use the Xbox joystick controller interface +3. `playstation.py`: use the Playstation game controller interface +4. `xbox.py`: use the Xbox game controller interface +5. `asus_xtion.py`: use the Asus Xtion interface +6. `openpose.py`: use the Openpose interface +7. `speech.py`: use the speech (recognizer/translator) interface + +**Note**: some interfaces require to install different libraries, and possibly to configure them. Check the raised errors. diff --git a/examples/interfaces/asus_xtion.py b/examples/interfaces/asus_xtion.py new file mode 100644 index 0000000..d51845c --- /dev/null +++ b/examples/interfaces/asus_xtion.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +"""Run the Asus Xtion interface. + +Make sure that the `openni` library is installed with all the correct environment variables set, and that the Asus +Xtion is connected before running this code. Note that it can take some time to initialize the interface. +""" + +import argparse +import matplotlib.pyplot as plt + +from pyrobolearn.tools.interfaces.camera.asus_xtion import AsusXtionInterface + + +# create parser +parser = argparse.ArgumentParser() +parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool, + default=False) +parser.add_argument('-r', '--use_rgb', help='If we should get RGB images. Note that RGB and IR images can not be ' + 'captured at the same time.', type=bool, + default=True) +parser.add_argument('-d', '--use_depth', help='If we should get depth images.', type=bool, + default=True) +parser.add_argument('-i', '--use_ir', help='If we should get IR images. Note that RGB and IR images can not be ' + 'captured at the same time.', type=bool, + default=False) +args = parser.parse_args() + + +# get which pictures to capture +use_rgb, use_depth, use_ir = args.use_rgb, args.use_depth, args.use_ir +if use_rgb and use_ir: + use_ir = False + + +# create Asus Xtion interface +interface = AsusXtionInterface(use_rgb=use_rgb, use_depth=use_depth, use_ir=use_ir) + + +# plotting using matplotlib in interactive mode +fig, axes = plt.subplots(1, 2) +plots = [None]*2 +titles = [] +if use_rgb: + titles.append('RGB') +if use_ir: + titles.append('IR') +if use_depth: + titles.append('Depth') + +plt.ion() # interactive mode on + +while True: + # if don't use thread call `step` or `run` + data = interface.run() + + # get the frame and plot it with matplotlib + if plots[0] is None: + for i in range(len(plots)): + plots[i] = axes[i].imshow(data[i]) + axes[i].set_title(titles[i]) + else: + for plot, img in zip(plots, data): + plot.set_data(img) + + # pause a bit + plt.pause(0.01) + + # check if the figure is closed, and if so, get out of the loop + if not plt.fignum_exists(fig.number): + break + +plt.ioff() # interactive mode off +plt.show() diff --git a/examples/interfaces/openpose.py b/examples/interfaces/openpose.py new file mode 100644 index 0000000..96e0bf7 --- /dev/null +++ b/examples/interfaces/openpose.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""Run the Openpose interface. + +Make sure that the webcam is connected, and that the openpose framework has been installed before running this code. +For this code to work, you have to specify the path to the openpose framework, or set the `OPENPOSE_PATH` environment +variable. Note that it can take some time to initialize the interface. +""" + +import cv2 +import argparse + +from pyrobolearn.tools.interfaces.camera.openpose import OpenPoseInterface + + +# create parser +parser = argparse.ArgumentParser() +parser.add_argument('-p', '--path', help='Absolute path to the openpose framework. If not specified, it will check ' + 'for the environment variable `OPENPOSE_PATH`.', type=str, default='') +parser.add_argument('-t', '--use_thread', help='If we should run the openpose interface in a thread.', type=bool, + default=False) +parser.add_argument('-f', '--detect_face', help='If we should detect the face with openpose.', type=bool, + default=True) +parser.add_argument('-a', '--detect_hands', help='If we should detect the hands with openpose.', type=bool, + default=False) +args = parser.parse_args() +path = None if args.path == '' else args.path + + +# create openpose interface +if args.use_thread: + # create and run interface in a thread + interface = OpenPoseInterface(openpose_path=path, detect_face=args.detect_face, detect_hands=args.detect_hands, + use_thread=True, sleep_dt=1. / 10, verbose=True) + raw_input('Press key to stop the openpose interface') +else: + # create interface + interface = OpenPoseInterface(openpose_path=path, detect_face=args.detect_face, detect_hands=args.detect_hands) + + # run interface + while True: + frame, keypoints = interface.run() + cv2.imshow("OpenPose 1.4.0 - Tutorial Python API", frame) + + # quit display if 'esc' button is pressed + key = cv2.waitKey(15) & 0xFF + if key == 27: + cv2.destroyWindow('frame') diff --git a/examples/interfaces/speech.py b/examples/interfaces/speech.py new file mode 100644 index 0000000..51b51d4 --- /dev/null +++ b/examples/interfaces/speech.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +"""Run the speech interface. + +This will perform speech recognition, translation, and synthesization. Make sure that your computer has a microphone +connected. + +In the future, interfaces using the Google assistant, Alexa, or a similar tool will be implemented. +""" + +import argparse + +from pyrobolearn.tools.interfaces.audio.speech import SpeechRecognizerInterface, SpeechTranslatorInterface + + +# get the available languages. +languages = SpeechRecognizerInterface.available_languages +print("Available languages are: {}".format(languages)) + +# create parser +parser = argparse.ArgumentParser() +# parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool, +# default=False) +parser.add_argument('-l', '--lang', help='The language that needs to be recognized.', type=str, choices=languages, + default='english') +parser.add_argument('-a', '--target_lang', help='If we should get depth images.', type=str, choices=languages, + default='english') +args = parser.parse_args() + + +# create speech recognizer/translator interface +# interface = SpeechRecognizerInterface(verbose=True, lang=args.lang) +interface = SpeechTranslatorInterface(verbose=True, from_lang=args.lang, target_lang=args.target_lang) + +# run the interface +while True: + data = interface.run() diff --git a/examples/interfaces/webcam.py b/examples/interfaces/webcam.py index 1c46d6d..3ad2c6d 100644 --- a/examples/interfaces/webcam.py +++ b/examples/interfaces/webcam.py @@ -1,34 +1,52 @@ #!/usr/bin/env python -"""Load the Webcam interface. +"""Run the Webcam interface. + +Make sure that the webcam is connected before running this code. Note that it can take some time to initialize +the interface. """ -from itertools import count +import argparse import matplotlib.pyplot as plt + from pyrobolearn.tools.interfaces.camera.webcam import WebcamInterface -# create interface -interface = WebcamInterface(use_thread=True, sleep_dt=1./10, verbose=False) -# plotting using matplotlib in interactive mode -fig = plt.figure() -plot = None -plt.ion() # interactive mode on +# create parser +parser = argparse.ArgumentParser() +parser.add_argument('-t', '--use_thread', help='If we should run the webcam interface in a thread.', type=bool, + default=False) +args = parser.parse_args() -for _ in count(): - # # if don't use thread call `step` or `run` (note that `run` returns the frame but not - # interface.step() - # get the frame and plot it with matplotlib - frame = interface.frame - if plot is None: - plot = plt.imshow(frame) - else: - plot.set_data(frame) - plt.pause(0.01) +# create webcam interface +if args.use_thread: + # create and run interface in a thread + interface = WebcamInterface(use_thread=True, sleep_dt=1./10, verbose=True) + raw_input('Press key to stop the webcam interface') +else: + # create interface + interface = WebcamInterface() - # check if the figure is closed, and if so, get out of the loop - if not plt.fignum_exists(fig.number): - break + # plotting using matplotlib in interactive mode + fig = plt.figure() + plot = None + plt.ion() # interactive mode on -plt.ioff() # interactive mode off -plt.show() + while True: + # if don't use thread call `step` or `run` (note that `run` returns the frame but not + interface.step() + + # get the frame and plot it with matplotlib + frame = interface.frame + if plot is None: + plot = plt.imshow(frame) + else: + plot.set_data(frame) + plt.pause(0.01) + + # check if the figure is closed, and if so, get out of the loop + if not plt.fignum_exists(fig.number): + break + + plt.ioff() # interactive mode off + plt.show() diff --git a/pyrobolearn/tools/interfaces/audio/alexa.py b/pyrobolearn/tools/interfaces/audio/alexa.py new file mode 100644 index 0000000..f3baf35 --- /dev/null +++ b/pyrobolearn/tools/interfaces/audio/alexa.py @@ -0,0 +1,2 @@ +# TODO +# Ref: https://developer.amazon.com/en-US/alexa/alexa-skills-kit/alexa-skill-python-tutorial diff --git a/pyrobolearn/tools/interfaces/audio/audio.py b/pyrobolearn/tools/interfaces/audio/audio.py index 56f46bf..aeafee8 100644 --- a/pyrobolearn/tools/interfaces/audio/audio.py +++ b/pyrobolearn/tools/interfaces/audio/audio.py @@ -1,10 +1,8 @@ #!/usr/bin/env python -"""Define the main basic Camera interface - -This defines the main basic camera interface from which all other interfaces which uses a camera inherit from. +"""Define the main basic audio interface. """ -import os +# TODO from pyrobolearn.tools.interfaces.interface import Interface, InputInterface, OutputInterface, InputOutputInterface @@ -19,59 +17,6 @@ except ImportError as e: "pip install pyaudio\n" raise ImportError(e.__str__() + string) -# Speech recognition -# Good tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones -try: - import speech_recognition as sr -except ImportError as e: - string = "\nHint: try to install speech_recognition by typing the following lines in the terminal: \n" \ - "sudo apt-get install libpulse-dev" \ - "pip install pocketsphinx\n" \ - "pip install google-cloud-speech" \ - "pip install SpeechRecognition" - raise ImportError(e.__str__() + string) - -# Speech synthesis -# Good tutorial: https://pythonprogramminglanguage.com/text-to-speech/ -# If Python3.3 or higher: https://pypi.org/project/google_speech/ -try: - from gtts import gTTS -except ImportError as e: - string = "\nHint: try to install gTTS by typing: pip install gTTS" - raise ImportError(e.__str__() + string) - -# # Another one is `pyttsx3`, which is the best offline module (the problem is that it only supports english) -# # Documentation (with examples): pyttsx3.readthedocs.io/en/latest/ -# try: -# import pyttsx3 -# except ImportError as e: -# string = "\nHint: try to install pyttsx3 by typing: pip install pyttsx3" -# raise ImportError(e.__str__() + string) - -# what I also checked: `pyttsx` and `pyvona` -# import pyttsx -# import pyvona - - -# Translation -# Github repo: https://github.com/ssut/py-googletrans -# Tutorial: https://www.codeproject.com/Tips/1236705/How-to-Use-Google-Translator-in-Python -try: - from googletrans import Translator -except ImportError as e: - string = "\nHint: try to install googletrans by typing: pip install googletrans" - raise ImportError(e.__str__() + string) - - -# ChatterBot -# Github repo: https://github.com/gunthercox/ChatterBot -# Documentation: https://chatterbot.readthedocs.io/en/stable -# try: -# from chatterbot import ChatBot -# except ImportError as e: -# string = "\nHint: try to install chatterbot by typing: pip install chatterbot" -# raise ImportError(e.__str__() + string) - __author__ = "Brian Delhaisse" __copyright__ = "Copyright 2018, PyRoboLearn" @@ -163,248 +108,7 @@ class InputOutputAudioInterface(InputOutputInterface): pass -class SpeechRecognizerInterface(InputInterface): - r"""Speech Recognizer Interface - - References: - [1] Tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones - """ - - def __init__(self, use_thread=False, lang='english', verbose=False): - """ - Initialize the speech recognizer input interface. - - Args: - use_thread (bool): If True, it will run the interface in a separate thread than the main one. - The interface will update its data automatically. - lang (str): language to recognize - verbose (bool): If True, it will print information about the state of the interface. This is let to the - programmer what he / she wishes to print. - """ - self.recognizer = sr.Recognizer() - self.microphone = sr.Microphone() # device_index=-1 - - languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', - 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', - 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', - 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'} - # Check < https://gist.github.com/traysr/2001377 > for more - self.lang = languages[lang] - - # string that is being said - self.data = '' - - super(SpeechRecognizerInterface, self).__init__(use_thread=use_thread, verbose=verbose) - - def run(self): - """Run the interface.""" - # listen to speech through the microphone - with self.microphone as source: - self.recognizer.adjust_for_ambient_noise(source) - print("I am listening...") - audio = self.recognizer.listen(source) # listen to what is being said - - # recognize speech (get the string from audio) - try: - print('Trying to understand what you just said...') - self.data = self.recognizer.recognize_google(audio, language=self.lang) - except sr.UnknownValueError: - print("Unable to recognize speech") - except sr.RequestError as e: - print("API unavailable".format(e)) - - if self.verbose: - print("You said: {}".format(self.data)) - - -class SpeechSynthesizerInterface(OutputInterface): - r"""Speech Synthesizer Interface - - References: - [1] tutorial: https://pythonprogramminglanguage.com/text-to-speech/ - [2] If Python3.3 or higher: https://pypi.org/project/google_speech/ - """ - - def __init__(self, use_thread=False, lang='english', verbose=False): - """ - Initialize the speech synthesizer output interface. - - Args: - use_thread (bool): If True, it will run the interface in a separate thread than the main one. - The interface will update its data automatically. - lang (str): language to recognize - verbose (bool): If True, it will print information about the state of the interface. This is let to the - programmer what he / she wishes to print. - """ - languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', - 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', - 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', - 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'} - # Check < https://gist.github.com/traysr/2001377 > for more - self.lang = languages[lang] - - self.updated = False - self.data = '' - - super(SpeechSynthesizerInterface, self).__init__(use_thread=use_thread, verbose=verbose) - - def run(self): - """Run the interface.""" - if self.updated: - # tts = text-to-speech - tts = gTTS(text=self.data, lang=self.lang) - tts.save('tmp.mp3') - os.system('mpg321 tmp.mp3') - os.system('rm tmp.mp3') - self.updated = False - - def update(self, data): - """Update the data.""" - self.data = data - self.updated = True - - -class SpeechTranslatorInterface(InputOutputInterface): - r"""Speech Translator Interface - """ - - def __init__(self, use_thread=False, target_lang='english', from_lang='auto', verbose=False): - """ - Initialize the speech translator input/output interface. - - Args: - use_thread (bool): If True, it will run the interface in a separate thread than the main one. - The interface will update its data automatically. - target_lang (str): language to translate to. - from_lang (str): language to translate from. - verbose (bool): If True, it will print information about the state of the interface. This is let to the - programmer what he / she wishes to print. - """ - self.translator = Translator() - - languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', - 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', - 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', - 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR', - 'auto': 'auto'} - # Check < https://gist.github.com/traysr/2001377 > for more - self.target_lang = languages[target_lang] - self.from_lang = languages[from_lang] - - self.updated = False - self.input_data = '' - self.data = '' - - super(SpeechTranslatorInterface, self).__init__(use_thread, verbose=verbose) - - def run(self): - """Run the interface.""" - # translate - if self.updated and self.target_lang != self.from_lang: - translated = self.translator.translate(self.input_data, dest=self.target_lang, src=self.from_lang) - self.data = translated.text - self.updated = False - - def update(self, data): - """Update the data.""" - self.input_data = data - self.updated = True - - -class SpeechInterface(InputOutputInterface): - r"""Speech Interface - - This class performs speech recognition, translation, and synthesization. - """ - - def __init__(self, use_thread=False, target_lang='english', from_lang='english', verbose=False): - """ - Initialize the speech (recognizer, translator, and synthesizer) interface. - - Args: - use_thread (bool): If True, it will run the interface in a separate thread than the main one. - The interface will update its data automatically. - target_lang (str): language to translate to. - from_lang (str): language to translate from. - verbose (bool): If True, it will print information about the state of the interface. This is let to the - programmer what he / she wishes to print. - """ - self.recognizer = SpeechRecognizerInterface(use_thread=False, lang=from_lang) - self.translator = None - if target_lang != from_lang: - self.translator = SpeechTranslatorInterface(use_thread=False, target_lang=target_lang, - from_lang=from_lang) - self.synthesizer = SpeechSynthesizerInterface(use_thread=False, lang=target_lang) - - self.updated = False - self.input_data = '' - self.output_data = '' - - super(SpeechInterface, self).__init__(use_thread, verbose=verbose) - - def run(self): - """Run the interface.""" - pass - - def update(self, data): - """Update the interface.""" - pass - - # Tests if __name__ == '__main__': interface = AudioInterface() interface.print_info() - - # recognize, translate and synthesize speech - english = set(['en', 'en-US', 'en-GB']) - languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', - 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', - 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', - 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'} - # Check < https://gist.github.com/traysr/2001377 > for more - lang = languages['english'] - - recognizer = sr.Recognizer() - microphone = sr.Microphone() # device_index=-1 - - # print(microphone.list_microphone_names()) - - # listen to speech through the microphone - with microphone as source: - recognizer.adjust_for_ambient_noise(source) - print("Say something!") - audio = recognizer.listen(source) - - # recognize speech - print('processing...') - string = '' - try: - string = recognizer.recognize_google(audio, language=lang) - except sr.UnknownValueError: - print("Unable to recognize speech") - except sr.RequestError as e: - print("API unavailable".format(e)) - - print("You said: " + string) - - # translate it if other language than english - if lang not in english: - translator = Translator() - translated = translator.translate(string) # dest='en', src='auto') - print("which translates to: " + translated.text) - - # produce speech - print('Let me try to repeat what you just said:') - tts = gTTS(text=string, lang=lang) - tts.save('tmp.mp3') - os.system('mpg321 tmp.mp3') - os.system('rm tmp.mp3') - - # recognize speech using Sphinx - # try: - # print("Sphinx thinks you said '" + recognizer.recognize_sphinx(audio) + "'") - # except sr.UnknownValueError: - # print("Sphinx could not understand audio") - # except sr.RequestError as e: - # print("Sphinx error; {0}".format(e)) diff --git a/pyrobolearn/tools/interfaces/audio/google_assistant.py b/pyrobolearn/tools/interfaces/audio/google_assistant.py new file mode 100644 index 0000000..7d8dbdd --- /dev/null +++ b/pyrobolearn/tools/interfaces/audio/google_assistant.py @@ -0,0 +1,2 @@ +# TODO +# Ref: https://developers.google.com/assistant/sdk/overview diff --git a/pyrobolearn/tools/interfaces/audio/speech.py b/pyrobolearn/tools/interfaces/audio/speech.py new file mode 100644 index 0000000..cda2107 --- /dev/null +++ b/pyrobolearn/tools/interfaces/audio/speech.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python +"""Define the various speech interfaces allowing to perform speech recognition, translation, and synthesization. +""" +# TODO: chatbox, google assistant, alexa + +import os # TODO: use subprocess instead +# import subprocess + +from pyrobolearn.tools.interfaces.interface import InputInterface, OutputInterface, InputOutputInterface + + +# Speech recognition +# Good tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones +try: + import speech_recognition as sr +except ImportError as e: + string = "\nHint: try to install speech_recognition by typing the following lines in the terminal: \n" \ + "sudo apt-get install libpulse-dev" \ + "pip install pocketsphinx\n" \ + "pip install google-cloud-speech" \ + "pip install SpeechRecognition" + raise ImportError(e.__str__() + string) + +# Speech synthesis +# Good tutorial: https://pythonprogramminglanguage.com/text-to-speech/ +# If Python3.3 or higher: https://pypi.org/project/google_speech/ +try: + from gtts import gTTS +except ImportError as e: + string = "\nHint: try to install gTTS by typing: `pip install gTTS`." \ + "Also install `mpg321` using `sudo apt-get install mpg321`." + raise ImportError(e.__str__() + string) + +# # Another one is `pyttsx3`, which is the best offline module (the problem is that it only supports english) +# # Documentation (with examples): pyttsx3.readthedocs.io/en/latest/ +# try: +# import pyttsx3 +# except ImportError as e: +# string = "\nHint: try to install pyttsx3 by typing: pip install pyttsx3" +# raise ImportError(e.__str__() + string) + +# what I also checked: `pyttsx` and `pyvona` +# import pyttsx +# import pyvona + + +# Translation +# Github repo: https://github.com/ssut/py-googletrans +# Tutorial: https://www.codeproject.com/Tips/1236705/How-to-Use-Google-Translator-in-Python +try: + from googletrans import Translator +except ImportError as e: + string = "\nHint: try to install googletrans by typing: pip install googletrans" + raise ImportError(e.__str__() + string) + + +# ChatterBot +# Github repo: https://github.com/gunthercox/ChatterBot +# Documentation: https://chatterbot.readthedocs.io/en/stable +# try: +# from chatterbot import ChatBot +# except ImportError as e: +# string = "\nHint: try to install chatterbot by typing: pip install chatterbot" +# raise ImportError(e.__str__() + string) + + +__author__ = "Brian Delhaisse" +__copyright__ = "Copyright 2018, PyRoboLearn" +__credits__ = ["Brian Delhaisse"] +__license__ = "GNU GPLv3" +__version__ = "1.0.0" +__maintainer__ = "Brian Delhaisse" +__email__ = "briandelhaisse@gmail.com" +__status__ = "Development" + + +class SpeechRecognizerInterface(InputInterface): + r"""Speech Recognizer Interface + + References: + [1] Tutorial: https://realpython.com/python-speech-recognition/#working-with-microphones + """ + + available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian', + 'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese', + 'mandarin', 'polish', 'portuguese', 'russian', 'greek'} + + def __init__(self, use_thread=False, lang='english', verbose=False): + """ + Initialize the speech recognizer input interface. + + Args: + use_thread (bool): If True, it will run the interface in a separate thread than the main one. + The interface will update its data automatically. + lang (str): language to recognize + verbose (bool): If True, it will print information about the state of the interface. This is let to the + programmer what he / she wishes to print. + """ + self.recognizer = sr.Recognizer() + self.microphone = sr.Microphone() # device_index=-1 + + languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', + 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', + 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', + 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'} + # Check < https://gist.github.com/traysr/2001377 > for more + self.lang = languages[lang] + + # string that is being said + self.data = '' + + super(SpeechRecognizerInterface, self).__init__(use_thread=use_thread, verbose=verbose) + + def run(self): + """Run the interface.""" + # listen to speech through the microphone + with self.microphone as source: + self.recognizer.adjust_for_ambient_noise(source) + if self.verbose: + print("Say something, I am listening!") + audio = self.recognizer.listen(source) # listen to what is being said + + # recognize speech (get the string from audio) + try: + if self.verbose: + print('Please wait, trying to understand what you just said...') + self.data = self.recognizer.recognize_google(audio, language=self.lang) + + if self.verbose: + print("You said: {}".format(''.join(self.data).encode('utf-8'))) + return self.data + except sr.UnknownValueError: + print("Unable to recognize speech") + except sr.RequestError as e: + print("API unavailable".format(e)) + + return None + + +class SpeechSynthesizerInterface(OutputInterface): + r"""Speech Synthesizer Interface + + References: + [1] tutorial: https://pythonprogramminglanguage.com/text-to-speech/ + [2] If Python3.3 or higher: https://pypi.org/project/google_speech/ + """ + + available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian', + 'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese', + 'mandarin', 'polish', 'portuguese', 'russian', 'greek'} + + def __init__(self, use_thread=False, lang='english', verbose=False): + """ + Initialize the speech synthesizer output interface. + + Args: + use_thread (bool): If True, it will run the interface in a separate thread than the main one. + The interface will update its data automatically. + lang (str): language to recognize + verbose (bool): If True, it will print information about the state of the interface. This is let to the + programmer what he / she wishes to print. + """ + languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', + 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', + 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', + 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR'} + # Check < https://gist.github.com/traysr/2001377 > for more + self.lang = languages[lang] + + self.updated = False + self._data = '' + + super(SpeechSynthesizerInterface, self).__init__(use_thread=use_thread, verbose=verbose) + + @property + def data(self): + return self._data + + @data.setter + def data(self, data): + if isinstance(data, (str, unicode)): + self._data = data + self.updated = True + + def run(self): + """Run the interface.""" + if self.updated: + if self.verbose: + print("Generating speech...") + # tts = text-to-speech + tts = gTTS(text=self.data, lang=self.lang) + tts.save('tmp.mp3') + os.system('mpg321 tmp.mp3 > /dev/null 2>&1') # TODO: use subprocess instead + os.system('rm tmp.mp3') + # subprocess.call(['mpg321 tmp.mp3']) + # subprocess.call(['rm tmp.mp3']) + self.updated = False + print("Speech generated!") + return self.data + + def update(self, data): + """Update the data.""" + self.data = data + + +class TranslatorInterface(InputOutputInterface): + r"""Text Translator Interface + """ + + available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian', + 'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese', + 'mandarin', 'polish', 'portuguese', 'russian', 'greek'} + + def __init__(self, use_thread=False, target_lang='english', from_lang='auto', verbose=False): + """ + Initialize the speech translator input/output interface. + + Args: + use_thread (bool): If True, it will run the interface in a separate thread than the main one. + The interface will update its data automatically. + target_lang (str): language to translate to. + from_lang (str): language to translate from. + verbose (bool): If True, it will print information about the state of the interface. This is let to the + programmer what he / she wishes to print. + """ + self.translator = Translator() + + languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB', + 'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de', + 'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN', + 'mandarin': 'zh-CN', 'polish': 'pl', 'portuguese': 'pt', 'russian': 'ru', 'greek': 'el-GR', + 'auto': 'auto'} + # Check < https://gist.github.com/traysr/2001377 > for more + self.target_lang = languages[target_lang] + self.from_lang = languages[from_lang] + + self.updated = False + self._input_data = '' + self.data = '' + + super(TranslatorInterface, self).__init__(use_thread, verbose=verbose) + + @property + def input_data(self): + return self._input_data + + @input_data.setter + def input_data(self, data): + if isinstance(data, (str, unicode)): + self._input_data = data + self.updated = True + + def run(self): + """Run the interface.""" + # translate + if self.updated and self.target_lang != self.from_lang: + if self.verbose: + print("Translating: {}".format(''.join(self.input_data).encode('utf-8'))) + translated = self.translator.translate(self.input_data, dest=self.target_lang, src=self.from_lang) + self.data = translated.text + if self.verbose: + print("Translated text: {}".format(''.join(self.data).encode('utf-8'))) + self.updated = False + return self.data + + def update(self, data): + """Update the data.""" + self.input_data = data + + +class SpeechTranslatorInterface(InputOutputInterface): + r"""Speech Translator Interface + + This class performs speech recognition, translation, and synthesization. + """ + + available_languages = {'french', 'english', 'american english', 'british english', 'indian english', 'italian', + 'japanese', 'korean', 'german', 'dutch', 'spanish', 'spanish (peru)', 'chinese', + 'mandarin', 'polish', 'portuguese', 'russian', 'greek'} + + def __init__(self, use_thread=False, target_lang='english', from_lang='english', verbose=False): + """ + Initialize the speech (recognizer, translator, and synthesizer) interface. + + Args: + use_thread (bool): If True, it will run the interface in a separate thread than the main one. + The interface will update its data automatically. + target_lang (str): language to translate to. + from_lang (str): language to translate from. + verbose (bool): If True, it will print information about the state of the interface. This is let to the + programmer what he / she wishes to print. + """ + self.recognizer = SpeechRecognizerInterface(use_thread=False, lang=from_lang, verbose=verbose) + self.translator = None + if target_lang != from_lang: + self.translator = TranslatorInterface(use_thread=False, target_lang=target_lang, + from_lang=from_lang, verbose=verbose) + self.synthesizer = SpeechSynthesizerInterface(use_thread=False, lang=target_lang, verbose=verbose) + + self.updated = False + self.input_data = '' + self.output_data = '' + + super(SpeechTranslatorInterface, self).__init__(use_thread, verbose=verbose) + + def run(self): + """Run the interface.""" + + # run the recognizer + self.input_data = self.recognizer.run() + + # run the translator + if self.translator is not None: + self.translator.input_data = self.input_data + self.output_data = self.translator.run() + else: + self.output_data = self.input_data + + # run the synthesizer + if self.input_data is not None: + self.synthesizer.data = self.output_data + self.synthesizer.run() + + return self.output_data + + +# Tests +if __name__ == '__main__': + + print("Available languages are: {}".format(SpeechRecognizerInterface.available_languages)) + + # interface = SpeechRecognizerInterface(verbose=True, lang='english') + interface = SpeechTranslatorInterface(verbose=True, from_lang='french', target_lang='english') + + while True: + data = interface.run() diff --git a/pyrobolearn/tools/interfaces/camera/asus_xtion.py b/pyrobolearn/tools/interfaces/camera/asus_xtion.py index 8f1570a..d90f2a7 100644 --- a/pyrobolearn/tools/interfaces/camera/asus_xtion.py +++ b/pyrobolearn/tools/interfaces/camera/asus_xtion.py @@ -216,7 +216,7 @@ if __name__ == '__main__': use_depth=use_depth, use_ir=use_ir) # plotting using matplotlib in interactive mode - fig, axes = plt.subplots(1,2) + fig, axes = plt.subplots(1, 2) plots = [None]*2 titles = [] if use_rgb: diff --git a/pyrobolearn/tools/interfaces/camera/openpose.py b/pyrobolearn/tools/interfaces/camera/openpose.py index 55e75dd..edec8a6 100644 --- a/pyrobolearn/tools/interfaces/camera/openpose.py +++ b/pyrobolearn/tools/interfaces/camera/openpose.py @@ -36,10 +36,10 @@ class OpenPoseInterface(CameraInterface): 3D) pictures and map them to the human kinematic skeleton. References: - [1] OpenPose: github.com/CMU-Perceptual-Computing-Lab/openpose - [2] PyOpenPose (official): github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/modules/python_module.md - [3] OpenPose output format: github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md - [4] PyOpenPose (python wrappers): github.com/FORTH-ModelBasedTracker/PyOpenPose + - [1] OpenPose: github.com/CMU-Perceptual-Computing-Lab/openpose + - [2] PyOpenPose (official): github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/modules/python_module.md + - [3] OpenPose output format: github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md + - [4] PyOpenPose (python wrappers): github.com/FORTH-ModelBasedTracker/PyOpenPose """ def __init__(self, camera=None, detect_face=False, detect_hands=False, openpose_path=None, @@ -60,6 +60,14 @@ class OpenPoseInterface(CameraInterface): verbose (bool): If True, it will print information about the state of the interface. This is let to the programmer what he / she wishes to print. """ + # Check the path to the openpose folder (which contains various models and test images) + if openpose_path is None: + if 'OPENPOSE_PATH' not in os.environ: + raise ValueError("The OPENPOSE_PATH environment variable has not been set properly. Please " + "then provide the path to the openpose folder by specifying the `openpose_path` " + "argument") + openpose_path = os.environ['OPENPOSE_PATH'] + self.openpose_path = openpose_path # save variables self.detect_face = detect_face @@ -108,18 +116,9 @@ class OpenPoseInterface(CameraInterface): ['Mouth' + str(i-48) for i in range(48, 68)] + ['REye6'] + ['LEye6'] self.face_joint_names_to_ids = dict(zip(self.face_joints, range(len(self.face_joints)))) - # Check the path to the openpose folder (which contains various models and test images) - if openpose_path is None: - if 'OPENPOSE_PATH' not in os.environ: - raise ValueError("The OPENPOSE_PATH environment variable has not been set properly. Please " - "then provide the path to the openpose folder by specifying the `openpose_path` " - "argument") - openpose_path = os.environ['OPENPOSE_PATH'] - self.openpose_path = openpose_path - # specify the parameters params = dict() - params["model_folder"] = path + "models/" + params["model_folder"] = openpose_path + "/models/" if detect_face: params["face"] = True if detect_hands: @@ -243,7 +242,7 @@ class OpenPoseInterface(CameraInterface): # Tests if __name__ == '__main__': path = '/home/brian/repos/openpose/' - interface = OpenPoseInterface(openpose_path=path) # , use_thread=False, sleep_dt=1./10, verbose=True) + interface = OpenPoseInterface(openpose_path=path) # use_thread=True, sleep_dt=1./10, verbose=True) while True: frame, keypoints = interface.run() diff --git a/pyrobolearn/tools/interfaces/interface.py b/pyrobolearn/tools/interfaces/interface.py index dcf142a..bd14621 100644 --- a/pyrobolearn/tools/interfaces/interface.py +++ b/pyrobolearn/tools/interfaces/interface.py @@ -55,6 +55,8 @@ class Interface(object): VR/AR tools, phones, etc. For more info, see the `InputOutputInterface` class. """ + use_thread = False + def __init__(self, use_thread=False, sleep_dt=0, verbose=False): """ Initialize the interface. @@ -72,6 +74,8 @@ class Interface(object): self.data = None self.verbose = verbose + self.stop_thread = False + if self.use_thread: self.thread = threading.Thread(target=self._run) self.thread.start() @@ -89,6 +93,8 @@ class Interface(object): """ if self.use_thread: while True: + if self.stop_thread: # if the thread should stop + break self.run(*args, **kwargs) time.sleep(self.dt) else: @@ -105,7 +111,8 @@ class Interface(object): """ Stop and close the interface. """ - pass + if self.use_thread: + self.stop_thread = True ############# # Operators #