wassname/pyrobolearn
1
0
Fork 0
mirror of https://github.com/wassname/pyrobolearn.git synced 2026-09-09 11:31:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

update tools

Browse Source
This commit is contained in:
Brian Delhaisse
2019-03-27 15:23:33 +01:00
parent 106d2b744d
commit 01eafc916e
41 changed files with 674 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyrobolearn/tools/__init__.py
+2 -2
View File
@@ -1,6 +1,6 @@
# import interfaces
# import interfaces
from . import interfaces
# import bridges
# import bridges
from . import bridges
pyrobolearn/tools/bridges/__init__.py
+4 -4
View File
@@ -7,13 +7,13 @@ from .bridge import Bridge
from .mouse_keyboard import *
# Bridge for audio interfaces
# from audio import *
from . import audio
# Bridge for camera interfaces
# from camera import *
from . import camera
# Bridge for controller interfaces
# from controllers import *
from . import controllers
# Bridge for VR interfaces
# from vr import *
from . import vr
pyrobolearn/tools/bridges/audio/__init__.py
+3 -3
View File
@@ -1,8 +1,8 @@
## Bridge for audio interfaces ##
# Bridge for audio interfaces #
# Bridge between audio and robot
from robots import *
from . import robots
# Bridge between audio and world
from world import *
from . import world
pyrobolearn/tools/bridges/audio/robots/__init__.py
+5 -5
View File
@@ -1,6 +1,6 @@
# bridge between audio and wheeled robot
from bridge_speech_wheeled import *
# bridge between audio and rototary wing UAV
from bridge_speech_rotatory_uav import *
# # bridge between audio and wheeled robot
# from .bridge_speech_wheeled import *
#
# # bridge between audio and rototary wing UAV
# from .bridge_speech_rotatory_uav import *
pyrobolearn/tools/bridges/audio/robots/bridge_speech_rotatory_uav.py
+14 -2
View File
@@ -1,10 +1,22 @@
# Bridges between audio interface and rotatory wing robots
#!/usr/bin/env python
"""Bridges between audio interface and rotatory wing robots
"""
from pyrobolearn.robots import RotaryWingUAV
from pyrobolearn.tools.interfaces.audio import SpeechRecognizerInterface
from pyrobolearn.tools.interfaces.audio.audio import SpeechRecognizerInterface
from pyrobolearn.tools.bridges.bridge import Bridge
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class BridgeSpeechRecognizerRotatoryUAV(Bridge):
r"""Bridge Speech Wheeled Robot
pyrobolearn/tools/bridges/audio/robots/bridge_speech_wheeled.py
+13 -2
View File
@@ -1,11 +1,22 @@
# Bridges between audio interface and wheeled robots
#!/usr/bin/env python
"""Bridges between audio interface and wheeled robots
"""
import numpy as np
from pyrobolearn.robots import WheeledRobot, AckermannWheeledRobot
from pyrobolearn.tools.interfaces.audio import SpeechRecognizerInterface
from pyrobolearn.tools.interfaces.audio.audio import SpeechRecognizerInterface
from pyrobolearn.tools.bridges.bridge import Bridge
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class BridgeSpeechRecognizerWheeledRobot(Bridge):
r"""Bridge Speech Wheeled Robot
pyrobolearn/tools/bridges/bridge.py
+12
View File
@@ -8,6 +8,7 @@ bridges inherit from.
from pyrobolearn.tools.interfaces import Interface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
@@ -43,6 +44,13 @@ class Bridge(object):
"""
def __init__(self, interface, priority=None):
"""
Initialize the bridge.
Args:
interface (Interface): interface instance.
priority (None, int): priority number.
"""
self.interface = interface
self.priority = priority
@@ -52,20 +60,24 @@ class Bridge(object):
@property
def interface(self):
"""Return the interface instance associated to the bridge."""
return self._interface
@interface.setter
def interface(self, interface):
"""Set the interface associated to the bridge."""
if not isinstance(interface, Interface):
raise TypeError("Expecting interface to be an instance of Interface, instead got {}".format(interface))
self._interface = interface
@property
def priority(self):
"""Return the priority number."""
return self._priority
@priority.setter
def priority(self, priority):
"""Set the priority number."""
if priority is not None:
if not isinstance(priority, int):
raise TypeError("Expecting the priority to be an integer, instead got {}".format(priority))
pyrobolearn/tools/bridges/camera/__init__.py
+3 -3
View File
@@ -1,8 +1,8 @@
## Bridge for camera interfaces ##
# Bridge for camera interfaces #
# Bridge between camera and robot
from robots import *
from . import robots
# Bridge between camera and world
from world import *
from . import world
pyrobolearn/tools/bridges/controllers/__init__.py
+3 -3
View File
@@ -1,8 +1,8 @@
## Bridge for controller interfaces ##
# Bridge for controller interfaces #
# Bridge between controller and robot
from robots import *
from . import robots
# Bridge between controller and world
from world import *
from . import world
pyrobolearn/tools/bridges/controllers/robots/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
# bridge between controller and wheeled robot
from bridge_controller_wheeled import *
from .bridge_controller_wheeled import *
pyrobolearn/tools/bridges/controllers/robots/bridge_controller_wheeled.py
+17 -6
View File
@@ -1,11 +1,22 @@
# Bridges between controller interface and wheeled robots
#!/usr/bin/env python
"""Bridges between controller interface and wheeled robots
"""
from pyrobolearn.robots import WheeledRobot, AckermannWheeledRobot
from pyrobolearn.tools.interfaces.controllers import XboxControllerInterface, XboxOneControllerInterface
from pyrobolearn.tools.interfaces.controllers.xbox import XboxControllerInterface, XboxOneControllerInterface
from pyrobolearn.tools.bridges.bridge import Bridge
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class BridgeXboxWheeledRobot(Bridge):
r"""Bridge Xbox Wheeled Robot
@@ -25,7 +36,7 @@ class BridgeXboxWheeledRobot(Bridge):
self.speed = 1.
def step(self):
x,y = self.interface.LJ
x, y = self.interface.LJ
class BridgeXboxOneWheeledRobot(Bridge):
@@ -46,7 +57,7 @@ class BridgeXboxOneWheeledRobot(Bridge):
self.speed = 1.
def step(self):
x,y = self.interface.LJ
x, y = self.interface.LJ
class BridgeXboxOneAckermannWheeledRobot(Bridge):
@@ -65,7 +76,7 @@ class BridgeXboxOneAckermannWheeledRobot(Bridge):
self.speed = 1.
def step(self):
x,y = self.interface.LJ
x, y = self.interface.LJ
self.robot.set_steering(-x / 2.)
self.robot.drive_forward(y * self.speed)
pyrobolearn/tools/bridges/mouse_keyboard/bridge_mousekeyboard_imitation_task.py
+1
View File
@@ -19,6 +19,7 @@ from pyrobolearn.tools.interfaces import MouseKeyboardInterface
from pyrobolearn.tools.bridges.mouse_keyboard.bridge_mousekeyboard_world import BridgeMouseKeyboardWorld
# from pyrobolearn.tasks.imitation_task import ILTask # Warning: circular dependency
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
pyrobolearn/tools/bridges/vr/__init__.py
+3 -3
View File
@@ -1,8 +1,8 @@
## Bridge for VR interfaces ##
# Bridge for VR interfaces #
# Bridge between VR and robot
from robots import *
from . import robots
# Bridge between VR and world
from world import *
from . import world
pyrobolearn/tools/interfaces/__init__.py
+8 -8
View File
@@ -6,25 +6,25 @@ from .interface import *
from .mouse_keyboard import *
# audio
# from audio import *
from . import audio
# camera
# from camera import *
from . import camera
# controllers
# from controllers import *
from . import controllers
# bci
# from bci import *
from . import bci
# sensor suits
# from suits import *
from . import suits
# sensors (in general, EMG, etc)
# from sensors import *
from . import sensors
# VR interfaces
# from vr import *
from . import vr
# robot interfaces
# from robots import *
from . import robots
pyrobolearn/tools/interfaces/audio/__init__.py
+2 -1
View File
@@ -1,3 +1,4 @@
# import audio interfaces
from audio import *
# from audio import *
# from . import audio
pyrobolearn/tools/interfaces/audio/audio.py
+83 -13
View File
@@ -1,3 +1,9 @@
#!/usr/bin/env python
"""Define the main basic Camera interface
This defines the main basic camera interface from which all other interfaces which uses a camera inherit from.
"""
import os
from pyrobolearn.tools.interfaces.interface import Interface, InputInterface, OutputInterface, InputOutputInterface
@@ -67,6 +73,16 @@ except ImportError as e:
# raise ImportError(e.__str__() + string)
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class AudioInterface(Interface):
r"""Audio Interface
@@ -81,12 +97,14 @@ class AudioInterface(Interface):
"""
def __init__(self):
"""Initialize the audio interface."""
super(AudioInterface, self).__init__()
self.port = pyaudio.PyAudio()
self.stream = self.port.open(format=pyaudio.paInt16, channels=2, rate=44100, input=True,
frames_per_buffer=1024) #input_device_index=)
frames_per_buffer=1024) # input_device_index=)
def printInfo(self):
def print_info(self):
"""Print information about the audio interface."""
for i in range(self.port.get_device_count()):
info = self.port.get_device_info_by_index(i)
print("###############################################################")
@@ -96,19 +114,23 @@ class AudioInterface(Interface):
info['defaultHighInputLatency']))
print("Output latency (low, high): {}, {}".format(info['defaultLowOutputLatency'],
info['defaultHighOutputLatency']))
print("Is an input device? {}".format(self.isInputDevice(info)))
print("Is an output device? {}".format(self.isOutputDevice(info)))
print("Is an input device? {}".format(self.is_input_device(info)))
print("Is an output device? {}".format(self.is_output_device(info)))
def isInputDevice(self, info):
return (info['maxInputChannels'] != 0)
@staticmethod
def is_input_device(info):
return info['maxInputChannels'] != 0
def isOutputDevice(self, info):
return (info['maxOutputChannels'] != 0)
@staticmethod
def is_output_device(info):
return info['maxOutputChannels'] != 0
def step(self):
"""Perform a step with the interface."""
data = self.stream.read()
def __del__(self):
"""Delete the audio interface."""
self.stream.stop_stream()
self.stream.close()
@@ -149,6 +171,16 @@ class SpeechRecognizerInterface(InputInterface):
"""
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech recognizer input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone() # device_index=-1
@@ -165,6 +197,7 @@ class SpeechRecognizerInterface(InputInterface):
super(SpeechRecognizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# listen to speech through the microphone
with self.microphone as source:
self.recognizer.adjust_for_ambient_noise(source)
@@ -193,7 +226,16 @@ class SpeechSynthesizerInterface(OutputInterface):
"""
def __init__(self, use_thread=False, lang='english', verbose=False):
"""
Initialize the speech synthesizer output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
lang (str): language to recognize
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
'indian english': 'en-IN', 'italian': 'it', 'japanese': 'ja', 'korean': 'ko', 'german': 'de',
'dutch': 'nl', 'spanish': 'es', 'spanish (peru)': 'es-PE', 'chinese': 'zh-CN',
@@ -207,6 +249,7 @@ class SpeechSynthesizerInterface(OutputInterface):
super(SpeechSynthesizerInterface, self).__init__(use_thread=use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
if self.updated:
# tts = text-to-speech
tts = gTTS(text=self.data, lang=self.lang)
@@ -216,6 +259,7 @@ class SpeechSynthesizerInterface(OutputInterface):
self.updated = False
def update(self, data):
"""Update the data."""
self.data = data
self.updated = True
@@ -224,7 +268,18 @@ class SpeechTranslatorInterface(InputOutputInterface):
r"""Speech Translator Interface
"""
def __init__(self, use_thread=False, target_lang='english', from_lang='auto'):
def __init__(self, use_thread=False, target_lang='english', from_lang='auto', verbose=False):
"""
Initialize the speech translator input/output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.translator = Translator()
languages = {'french': 'fr', 'english': 'en', 'american english': 'en-US', 'british english': 'en-GB',
@@ -240,9 +295,10 @@ class SpeechTranslatorInterface(InputOutputInterface):
self.input_data = ''
self.data = ''
super(SpeechTranslatorInterface, self).__init__(use_thread)
super(SpeechTranslatorInterface, self).__init__(use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
# translate
if self.updated and self.target_lang != self.from_lang:
translated = self.translator.translate(self.input_data, dest=self.target_lang, src=self.from_lang)
@@ -250,6 +306,7 @@ class SpeechTranslatorInterface(InputOutputInterface):
self.updated = False
def update(self, data):
"""Update the data."""
self.input_data = data
self.updated = True
@@ -260,7 +317,18 @@ class SpeechInterface(InputOutputInterface):
This class performs speech recognition, translation, and synthesization.
"""
def __init__(self, use_thread=False, target_lang='english', from_lang='english'):
def __init__(self, use_thread=False, target_lang='english', from_lang='english', verbose=False):
"""
Initialize the speech (recognizer, translator, and synthesizer) interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
target_lang (str): language to translate to.
from_lang (str): language to translate from.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.recognizer = SpeechRecognizerInterface(use_thread=False, lang=from_lang)
self.translator = None
if target_lang != from_lang:
@@ -272,19 +340,21 @@ class SpeechInterface(InputOutputInterface):
self.input_data = ''
self.output_data = ''
super(SpeechInterface, self).__init__(use_thread)
super(SpeechInterface, self).__init__(use_thread, verbose=verbose)
def run(self):
"""Run the interface."""
pass
def update(self, data):
"""Update the interface."""
pass
# Tests
if __name__ == '__main__':
interface = AudioInterface()
interface.printInfo()
interface.print_info()
# recognize, translate and synthesize speech
english = set(['en', 'en-US', 'en-GB'])
pyrobolearn/tools/interfaces/audio/microphone.py
+1 -1
View File
@@ -1,5 +1,5 @@
from audio import InputAudioInterface
from pyrobolearn.tools.interfaces.audio import InputAudioInterface
class MicrophoneInterface(InputAudioInterface):
pyrobolearn/tools/interfaces/audio/speaker.py
+1 -1
View File
@@ -1,5 +1,5 @@
from audio import OutputAudioInterface
from pyrobolearn.tools.interfaces.audio import OutputAudioInterface
class SpeakerInterface(OutputAudioInterface):
pyrobolearn/tools/interfaces/camera/__init__.py
+20 -15
View File
@@ -1,18 +1,23 @@
# General import
from camera import CameraInterface
from .camera import CameraInterface
# Webcam
# from webcam import WebcamInterface
# Asus Xtion
# from asus_xtion import AsusXtionInterface
# Kinect
# from kinect import *
# FER
# from fer import FERInterface
# OpenPose
# from openpose import OpenPoseInterface
# # Webcam
# # from webcam import WebcamInterface
# from . import webcam
#
# # Asus Xtion
# # from asus_xtion import AsusXtionInterface
# from . import asus_xtion
#
# # Kinect
# # from kinect import *
# from . import kinect
#
# # FER
# # from fer import FERInterface
# from . import fer
#
# # OpenPose
# # from openpose import OpenPoseInterface
# from . import openpose
pyrobolearn/tools/interfaces/camera/asus_xtion.py
+36 -5
View File
@@ -1,3 +1,6 @@
#!/usr/bin/env python
"""Define the Asus Xtion input interface.
"""
import numpy as np
@@ -29,10 +32,19 @@ except ImportError as e:
'check the README in the `libfreenect/OpenNI2-FreenectDriver` folder')
from camera import CameraInterface
from pyrobolearn.tools.interfaces.camera import CameraInterface
# check https://github.com/roboticslab-uc3m/installation-guides/blob/master/install-openni-nite.md
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class AsusXtionInterface(CameraInterface):
r"""Asus Xtion Interface
@@ -50,6 +62,22 @@ class AsusXtionInterface(CameraInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0., verbose=False, use_rgb=True, use_depth=True, use_ir=False):
"""
Initialize the Asus Xtion input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring or
setting the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
use_rgb (bool): If True, it will get the RGB camera image. Note that if this is enabled, you can not
get IR images at the same time.
use_depth (bool): If True, it will get the depth camera image.
use_ir (bool): If True, it will get the Infrared image. Note that if this is enabled, you can not get
RGB images at the same time.
"""
# quick check
if use_rgb and use_ir:
@@ -189,11 +217,14 @@ if __name__ == '__main__':
# plotting using matplotlib in interactive mode
fig, axes = plt.subplots(1,2)
plots = [None]*2
plots = [None]*2
titles = []
if use_rgb: titles.append('RGB')
if use_ir: titles.append('IR')
if use_depth: titles.append('Depth')
if use_rgb:
titles.append('RGB')
if use_ir:
titles.append('IR')
if use_depth:
titles.append('Depth')
plt.ion() # interactive mode on
for _ in count():
pyrobolearn/tools/interfaces/camera/camera.py
+12 -1
View File
@@ -6,7 +6,6 @@ This defines the main basic camera interface from which all other interfaces whi
from pyrobolearn.tools.interfaces.interface import InputInterface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
@@ -25,4 +24,16 @@ class CameraInterface(InputInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the camera input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring or
setting the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(CameraInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
self.frame = None
pyrobolearn/tools/interfaces/camera/description.md
-2
View File
@@ -1,2 +0,0 @@
In this folder, we use the Python Wrapper for OpenPose, and apply it on pictures
streamed from a webcam or kinect.
pyrobolearn/tools/interfaces/camera/fer.py
+28 -3
View File
@@ -1,5 +1,19 @@
#!/usr/bin/env python
"""Provide the Facial Expression Recognition Interface
"""
# TODO
from camera import CameraInterface
from pyrobolearn.tools.interfaces.camera import CameraInterface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class FERInterface(CameraInterface):
@@ -12,5 +26,16 @@ class FERInterface(CameraInterface):
[3] https://github.com/a514514772/Real-Time-Facial-Expression-Recognition-with-DeepLearning
"""
def __init__(self):
super(FERInterface, self).__init__()
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the FER input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring the
next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(FERInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
pyrobolearn/tools/interfaces/camera/kinect.py
+94 -6
View File
@@ -1,3 +1,6 @@
#!/usr/bin/env python
"""Provide the Kinect input interface.
"""
# import kinect library
@@ -17,11 +20,12 @@
# - https://stackoverflow.com/questions/19181332/libfreenect-vs-openni
import numpy as np
import cv2
# by default, use `openni` (optionally with the freenect driver) as it seems to be the most complete library
KINECT_LIBRARY = 'freenect'
if KINECT_LIBRARY[-8:] == 'freenect': # 'libfreenect' or 'freenect'
if KINECT_LIBRARY[-8:] == 'freenect': # 'libfreenect' or 'freenect'
# References:
# - OpenKinect: https://openkinect.org/wiki/Getting_Started
# - tuto: naman5.wordpress.com/2014/06/24/experimenting-with-kinect-using-opencv-python-and-open-kinect-libfreenect/
@@ -99,7 +103,17 @@ else:
# import interface
from camera import CameraInterface
from pyrobolearn.tools.interfaces.camera import CameraInterface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class KinectInterface(CameraInterface):
@@ -131,6 +145,17 @@ class KinectInterface(CameraInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0., verbose=False):
"""
Initialize the Kinect input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring
the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(KinectInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
@@ -148,25 +173,54 @@ class FreenectKinectInterface(KinectInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0., verbose=False):
"""
Initialize the Kinect input interface using the `freenect` library.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring
the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
# data
self.rgb = None
self.depth = None
super(FreenectKinectInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
def get_image(self, convertTo=None): # cv2.COLOR_RGB2BGR):
def get_image(self, convert_to=None): # cv2.COLOR_RGB2BGR):
"""Get the RGB image.
Args:
convert_to (int): if the picture must be converted to another format using `cv2.COLOR`
Returns:
np.array[width, height, 3]: RGB image.
"""
array, _ = freenect.sync_get_video()
if convertTo is not None:
array = cv2.cvtColor(array, convertTo)
if convert_to is not None:
array = cv2.cvtColor(array, convert_to)
return array
def get_depth(self):
"""Get the depth image.
Returns:
np.array[width, height]: depth image.
"""
array, _ = freenect.sync_get_depth()
array = array.astype(np.uint8)
return array
def run(self):
"""Run the interface; get the RGB and depth images.
Returns:
np.array[width, height, 3]: RGB image
np.array[width, height]: depth image
"""
self.rgb = self.get_image()
self.depth = self.get_depth()
@@ -185,7 +239,17 @@ class OpenNIKinectInterface(KinectInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0., verbose=False):
"""
Initialize the Kinect input interface using the `openni` library.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring
the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
# initialize openni2; you can give the path to the library as an argument. Otherwise, it will look for
# OPENNI2_REDIST and OPENNI2_REDIST64 environment variables.
openni2.initialize()
@@ -224,6 +288,12 @@ class OpenNIKinectInterface(KinectInterface):
super(OpenNIKinectInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
def run(self):
"""Run the interface; get the RGB and depth images.
Returns:
np.array[width, height, 3]: RGB image
np.array[width, height]: depth image
"""
# read frames
rgb_frame = self.rgb_stream.read_frame()
depth_frame = self.depth_stream.read_frame()
@@ -243,6 +313,7 @@ class OpenNIKinectInterface(KinectInterface):
return self.rgb, self.depth
def __del__(self):
"""Delete the Kinect interface."""
# close all the streams
self.rgb_stream.close()
self.depth_stream.close()
@@ -262,7 +333,18 @@ class KinectSkeletonTrackingInterface(KinectInterface):
"""
def __init__(self, use_thread=False, sleep_dt=0., verbose=False, track_hand=False):
"""
Initialize the Kinect input interface using the `openni` library.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring
the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
track_hand (bool): If True, it will track the hands.
"""
# initialize openni2 and nite2; you can give the path to the library as an argument.
# Otherwise, it will look for OPENNI2_REDIST / OPENNI2_REDIST64 and NITE2_REDIST / NITE2_REDIST64 environment
# variables.
@@ -311,6 +393,11 @@ class KinectSkeletonTrackingInterface(KinectInterface):
verbose=verbose)
def run(self):
"""Run the interface; get the skeleton data.
Returns:
dict: skeleton data
"""
# read frame
frame = self.tracker.read_frame()
@@ -338,12 +425,13 @@ class KinectSkeletonTrackingInterface(KinectInterface):
return self.data
def __del__(self):
"""Delete the Kinect interface."""
# unload nite2 and openni2
nite2.unload()
openni2.unload()
class ROSKinectInterface(CameraInterface):
class ROSKinectInterface(KinectInterface):
r"""ROS Kinect Interface
References:
pyrobolearn/tools/interfaces/camera/openpose.py
+30 -3
View File
@@ -15,8 +15,8 @@ except ImportError as e:
'`pyrobolearn/scripts/install_openpose.sh` to install the library and the associated '
'python wrapper.')
from camera import CameraInterface
from webcam import WebcamInterface
from pyrobolearn.tools.interfaces.camera import CameraInterface
from pyrobolearn.tools.interfaces.camera.webcam import WebcamInterface
__author__ = "Brian Delhaisse"
@@ -44,6 +44,22 @@ class OpenPoseInterface(CameraInterface):
def __init__(self, camera=None, detect_face=False, detect_hands=False, openpose_path=None,
use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the openpose camera input interface.
Args:
camera (None, CameraInterface): camera interface to get the images from.
detect_face (bool): if True, it will also detect the face keypoints.
detect_hands (bool): if True, it will also detect the hand keypoints.
openpose_path (str, None): path to the Openpose folder. If None, it will get the `OPENPOSE_PATH` bash
environment variable.
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring the
next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
# save variables
self.detect_face = detect_face
@@ -52,7 +68,7 @@ class OpenPoseInterface(CameraInterface):
# Check the given camera
if camera is None:
# If None, get pictures from a webcam
camera = WebcamInterface(use_thread=False, convertTo=None, verbose=False)
camera = WebcamInterface(use_thread=False, convert_to=None, verbose=False)
self.camera_in_openpose = True
else:
self.camera_in_openpose = False
@@ -125,6 +141,7 @@ class OpenPoseInterface(CameraInterface):
@property
def num_bodies(self):
"""Get the number of detected bodies."""
return len(self.datum.poseKeypoints)
@property
@@ -134,41 +151,51 @@ class OpenPoseInterface(CameraInterface):
@property
def left_hand_keypoints(self):
"""Get the left hand keypoints."""
return self.datum.handKeypoints[0]
@property
def right_hand_keypoints(self):
"""Get the right hand keypoints."""
return self.datum.handKeypoints[1]
@property
def hand_keypoints(self):
"""Get the hand keypoints."""
return self.left_hand_keypoints, self.right_hand_keypoints
@property
def face_keypoints(self):
"""Get the face keypoints."""
return self.datum.faceKeypoints
@property
def keypoints(self):
"""Get the keypoints."""
return self.body_keypoints, self.face_keypoints, self.left_hand_keypoints, self.right_hand_keypoints
@property
def heatmap(self):
"""Get the heatmap."""
return None
@property
def input_image(self):
"""Get the input image."""
return self.datum.cvInputData
@property
def output_image(self):
"""Get the output image."""
return self.datum.cvOutputData
@property
def num_gpus(self):
"""Return the number of GPUs."""
return pyopenpose.get_gpu_number()
def run(self, input_frame=None):
"""Run the interface."""
if input_frame is None:
# read image
if self.camera_in_openpose:
pyrobolearn/tools/interfaces/camera/webcam.py
+35 -9
View File
@@ -4,11 +4,10 @@
This provides the main interface to get pictures from the specified webcam.
"""
import cv2 # OpenCV to capture image from webcam
import os
import cv2 # OpenCV to capture image from webcam
from camera import CameraInterface
from pyrobolearn.tools.interfaces.camera import CameraInterface
# to close correctly the webcam once we run
os.environ["OPENCV_VIDEOIO_PRIORITY_MSMF"] = "0"
@@ -38,23 +37,48 @@ class WebcamInterface(CameraInterface):
[1] https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_gui/py_video_display/py_video_display.html
"""
def __init__(self, webcamId=0, saveVideo=False, filename='output.avi', fps=20, frameSize=(640,480), codec='XVID',
convertTo=cv2.COLOR_BGR2RGB, use_thread=False, sleep_dt=0, verbose=False):
def __init__(self, webcam_id=0, filename=None, fps=20, frame_size=(640, 480), codec='XVID',
convert_to=cv2.COLOR_BGR2RGB, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the webcam input interface.
Args:
webcam_id (int): webcam id. It allows to select which webcam to use if multiple webcams are present.
filename (str, None): If a string is given it will save the video at the specified filename. If None,
it won't save any videos.
fps (int): if we save a video, number of frames per second to record.
frame_size (tuple of int): if we save a video, the size of the frame (width, height).
codec (str): if we save a video, codec to be used.
convert_to (int, None): what format we should convert the images to (use `cv2.COLOR_*`). If None, it
won't convert the input image.
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring the
next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
# create video capture object
self.capture = cv2.VideoCapture(webcamId)
self.capture = cv2.VideoCapture(webcam_id)
# create video writer
if saveVideo:
if filename is not None:
# define codec (DIVX, XVID, MJPG, X264, WMV1, WMV2)
codec = cv2.VideoWriter_fourcc(*codec)
# check if we need to add an extension to the given filename
tmp = filename.split('/')[-1].split('.')
if len(tmp) == 1:
filename += '.avi'
# define video writer
self.writer = cv2.VideoWriter(filename, codec, fps, frameSize)
self.writer = cv2.VideoWriter(filename, codec, fps, frame_size)
else:
self.writer = None
# variables
self.convertTo = convertTo
self.convertTo = convert_to
self.verbose = False
# camera image
@@ -64,6 +88,7 @@ class WebcamInterface(CameraInterface):
super(WebcamInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
def run(self): # , display=True, convertToGray=False):
"""Run the interface."""
# get the frame from the webcam
return_code, frame = self.capture.read()
@@ -97,6 +122,7 @@ class WebcamInterface(CameraInterface):
return frame
def __del__(self):
"""Delete the interface."""
self.capture.release()
if self.writer is not None:
self.writer.release()
pyrobolearn/tools/interfaces/controllers/__init__.py
+8 -6
View File
@@ -1,9 +1,11 @@
# import general game controller interface
from controller import GameControllerInterface
from .controller import GameControllerInterface
# Xbox controller interface
from xbox import *
# Playstation controller interface
from playstation import *
# # Xbox controller interface
# # from .xbox import *
# from . import xbox
#
# # Playstation controller interface
# # from .playstation import *
# from . import playstation
pyrobolearn/tools/interfaces/controllers/controller.py
+17 -1
View File
@@ -1,10 +1,26 @@
#!/usr/bin/env python
"""Provide the abstract game controller interface class.
All the game controller interfaces inherit from the `GameControllerInterface` class defined here.
Dependencies:
- `pyrobolearn.tools.interfaces.InputInterface`
"""
from pyrobolearn.tools.interfaces.interface import InputOutputInterface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class GameControllerInterface(InputOutputInterface):
r"""Game Controller Interface
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
pyrobolearn/tools/interfaces/controllers/playstation.py
+11 -10
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python
"""Define the PlayStation controller interface
"""Provide the PlayStation controller interfaces.
This provides the interfaces for the PlayStation controllers (PS3 and PS4) using the `inputs` library.
"""
@@ -9,7 +9,7 @@ try:
except ImportError as e:
raise ImportError(repr(e) + '\nTry to install `inputs`: pip install inputs')
from controller import GameControllerInterface
from pyrobolearn.tools.interfaces.controllers.controller import GameControllerInterface
__author__ = "Brian Delhaisse"
@@ -51,10 +51,10 @@ class PSControllerInterface(GameControllerInterface):
# translation
buttons = ['BTN_SOUTH', 'BTN_EAST', 'BTN_WEST', 'BTN_NORTH', 'BTN_THUMBL', 'BTN_THUMBR', 'BTN_TL', 'BTN_TL2',
'BTN_TR', 'BTN_TR2', 'BTN_START', 'BTN_SELECT', 'BTN_MODE', 'ABS_HAT0X', 'ABS_HAT0Y', 'ABS_Z', 'ABS_RZ',
'ABS_X', 'ABS_Y', 'ABS_RX', 'ABS_RY']
ps4_buttons = ['X', 'O', 'S', 'T', 'LJB', 'RJB', 'L1', 'L2', 'R1', 'R2', 'options', 'share', 'PS', 'L', 'R', 'RT',
'LJX', 'LJY', 'RJX', 'RJY']
'BTN_TR', 'BTN_TR2', 'BTN_START', 'BTN_SELECT', 'BTN_MODE', 'ABS_HAT0X', 'ABS_HAT0Y', 'ABS_Z',
'ABS_RZ', 'ABS_X', 'ABS_Y', 'ABS_RX', 'ABS_RY']
ps4_buttons = ['X', 'O', 'S', 'T', 'LJB', 'RJB', 'L1', 'L2', 'R1', 'R2', 'options', 'share', 'PS', 'L', 'R',
'RT', 'LJX', 'LJY', 'RJX', 'RJY']
self.map = dict(zip(buttons, ps4_buttons))
self.inv_map = dict(zip(ps4_buttons, buttons))
@@ -169,21 +169,21 @@ class PSControllerInterface(GameControllerInterface):
print("Pushed button {} - state = {}".format(self.last_updated_button,
self.buttons[self.last_updated_button]))
def setLeftVibration(self, time_msec):
def set_left_vibration(self, time_msec):
"""Set the vibration for the left motor"""
self.gamepad.set_vibration(1, 0, time_msec)
# display info
if self.verbose:
print("Set vibration to the left motor for {} msec".format(time_msec))
def setRightVibration(self, time_msec):
def set_right_vibration(self, time_msec):
"""Set the vibration for the right motor"""
self.gamepad.set_vibration(0, 1, time_msec)
# display info
if self.verbose:
print("Set vibration to the right motor for {} msec".format(time_msec))
def setVibration(self, time_msec):
def set_vibration(self, time_msec):
"""Set the vibration for both motors"""
self.gamepad.set_vibration(1, 1, time_msec)
# display info
@@ -257,7 +257,8 @@ class PS4ControllerInterface(PSControllerInterface):
def __init__(self, use_thread=False):
super(PS4ControllerInterface, self).__init__(use_thread=use_thread,
controller_name='Sony Interactive Entertainment Wireless Controller')
controller_name='Sony Interactive Entertainment Wireless '
'Controller')
# Tests
pyrobolearn/tools/interfaces/controllers/xbox.py
+21 -6
View File
@@ -1,3 +1,8 @@
#!/usr/bin/env python
"""Provide the Xbox controller interfaces.
This provides the interfaces for the PlayStation controllers (Xbox 360 and Xbox One) using the `inputs` library.
"""
try:
from inputs import devices
@@ -11,7 +16,17 @@ try:
except ImportError as e:
raise ImportError(repr(e) + '\nTry to install `inputs`: pip install inputs')
from controller import GameControllerInterface
from pyrobolearn.tools.interfaces.controllers.controller import GameControllerInterface
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class XboxControllerInterface(GameControllerInterface):
@@ -163,8 +178,8 @@ class XboxControllerInterface(GameControllerInterface):
###########
def run(self):
#print('running')
events = self.gamepad.read() #blocking=False) # get_gamepad()
# print('running')
events = self.gamepad.read() # blocking=False) # get_gamepad()
for event in events:
event_type, code, state = event.ev_type, event.code, event.state
self.__setitem(event_type, self.map.get(code), state)
@@ -174,21 +189,21 @@ class XboxControllerInterface(GameControllerInterface):
print("Pushed button {} - state = {}".format(self.last_updated_button,
self.buttons[self.last_updated_button]))
def setLeftVibration(self, time_msec):
def set_left_vibration(self, time_msec):
"""Set the vibration for the left motor"""
self.gamepad.set_vibration(1, 0, time_msec)
# display info
if self.verbose:
print("Set vibration to the left motor for {} msec".format(time_msec))
def setRightVibration(self, time_msec):
def set_right_vibration(self, time_msec):
"""Set the vibration for the right motor"""
self.gamepad.set_vibration(0, 1, time_msec)
# display info
if self.verbose:
print("Set vibration to the right motor for {} msec".format(time_msec))
def setVibration(self, time_msec):
def set_vibration(self, time_msec):
"""Set the vibration for both motors"""
self.gamepad.set_vibration(1, 1, time_msec)
# display info
pyrobolearn/tools/interfaces/interface.py
+56
View File
@@ -9,6 +9,8 @@ The code that connects the interfaces to the simulator or elements inside this l
in `pyrobolearn/tools/bridges`.
"""
# TODO: when using thread makes it in a safe way and in an asynchronous manner
import threading
import time
@@ -54,6 +56,17 @@ class Interface(object):
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring or
setting the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
self.use_thread = use_thread
self.dt = sleep_dt
self.data = None
@@ -94,6 +107,16 @@ class Interface(object):
"""
pass
#############
# Operators #
#############
def __repr__(self):
return self.__class__.__name__
def __str__(self):
return self.__class__.__name__
def __call__(self):
self.step()
@@ -109,6 +132,17 @@ class InputInterface(Interface):
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the input interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring the
next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(InputInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
@@ -121,6 +155,17 @@ class OutputInterface(Interface):
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before setting the
next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(OutputInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
@@ -137,4 +182,15 @@ class InputOutputInterface(Interface):
"""
def __init__(self, use_thread=False, sleep_dt=0, verbose=False):
"""
Initialize the input + output interface.
Args:
use_thread (bool): If True, it will run the interface in a separate thread than the main one.
The interface will update its data automatically.
sleep_dt (float): If :attr:`use_thread` is True, it will sleep the specified amount before acquiring /
setting the next sample.
verbose (bool): If True, it will print information about the state of the interface. This is let to the
programmer what he / she wishes to print.
"""
super(InputOutputInterface, self).__init__(use_thread=use_thread, sleep_dt=sleep_dt, verbose=verbose)
pyrobolearn/tools/interfaces/mouse_keyboard/mousekeyboard.py
+6 -1
View File
@@ -16,7 +16,7 @@ from pyrobolearn.simulators import Simulator
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "(c) Brian Delhaisse"
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
@@ -127,15 +127,18 @@ class MouseKeyboardInterface(InputInterface):
@property
def mouse_pressed(self):
"""Return True if one of the mouse buttons has been pressed."""
return self.left_click_pressed or self.right_click_pressed
@mouse_pressed.setter
def mouse_pressed(self, pressed):
"""Set if the mouse has been pressed."""
self.left_click_pressed = pressed
self.right_click_pressed = pressed
@property
def mouse_down(self):
"""Check if one of the mouse buttons is down."""
return self.left_click_down or self.right_click_down
###########
@@ -143,6 +146,7 @@ class MouseKeyboardInterface(InputInterface):
###########
def check_key_events(self):
"""Check the key events."""
# get key events
events = self.simulator.get_keyboard_events()
@@ -161,6 +165,7 @@ class MouseKeyboardInterface(InputInterface):
self.key_pressed.append(key)
def check_mouse_events(self):
"""Check the mouse events."""
# get mouse events
events = self.simulator.get_mouse_events()
pyrobolearn/tools/interfaces/robots/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
# General import
from robot import RobotInterface
from .robot import RobotInterface
pyrobolearn/tools/interfaces/sensors/__init__.py
+2 -3
View File
@@ -1,7 +1,6 @@
# general imports
from sensor import *
from .sensor import *
# EMG sensor
from emg import EMGInterface
from . import emg
pyrobolearn/tools/interfaces/sensors/emg.py
+1 -1
View File
@@ -1,5 +1,5 @@
from sensor import BioSensorInterface
from pyrobolearn.tools.interfaces.sensors import BioSensorInterface
class EMGInterface(BioSensorInterface):
pyrobolearn/tools/interfaces/sensors/sensor.py
+1 -1
View File
@@ -13,4 +13,4 @@ class BioSensorInterface(InputInterface):
r"""Bio-Sensor Interface
"""
pass
pass
pyrobolearn/tools/interfaces/suits/__init__.py
+2 -2
View File
@@ -1,6 +1,6 @@
# general import
from suit import SuitInterface
from .suit import SuitInterface
# Xsens suit
from xsens import XsensSuitInterface
from . import xsens
pyrobolearn/tools/interfaces/suits/xsens.py
+1 -1
View File
@@ -1,5 +1,5 @@
from suit import SuitInterface
from pyrobolearn.tools.interfaces.suits import SuitInterface
class XsensSuitInterface(SuitInterface):
pyrobolearn/tools/interfaces/vr/__init__.py
+7 -7
View File
@@ -1,11 +1,11 @@
# general interface
from vr import VRInterface
from .vr import VRInterface
## VR interfaces ##
# VR interfaces #
# Oculus
from oculus import OculusInterface
# HTC
from htc import HTCViveInterface
# # Oculus
# from . import oculus
#
# # HTC
# from . import htc
pyrobolearn/tools/interfaces/vr/oculus.py
+106 -91
View File
@@ -1,11 +1,15 @@
# Define the OculusTouch class which communicates with Unity (on Windows) using TCP.
# Currently, Oculus has only support for Windows systems. However, several
# libraries such as ROS only runs on Linux systems. Thus, we can run Unity on
# a Windows system, associate the Unity scripts with the Oculus GameObjects, and then
# run this file on a Unix system (such as Linux or MacOSX) which will communicate by TCP.
# The scripts for Unity can be found in the `unity-scripts` folder.
#
# Currently, this code is the server while the code running in Unity on Windows is the client.
#!/usr/bin/env python
"""Define the OculusTouch class which communicates with Unity (on Windows) using TCP.
Currently, Oculus has only support for Windows systems. However, several libraries such as ROS only runs on
Linux systems. Thus, we can run Unity on a Windows system, associate the Unity scripts with the Oculus GameObjects,
and then run this file on a Unix system (such as Linux or MacOSX) which will communicate by TCP.
The scripts for Unity can be found in the `oculus_windows/unity-scripts/` folder.
Currently, this code is the server while the code running in Unity on Windows is the client.
"""
# TODO: refactor the code, and use the openvr library
import Queue
import socket
@@ -18,6 +22,15 @@ from pyrobolearn.utils.bullet_utils import RGBAColor
from pyrobolearn.tools.interfaces.vr import VRInterface
# from pyrobolearn.worlds.world import BasicWorld
__author__ = "Brian Delhaisse"
__copyright__ = "Copyright 2018, PyRoboLearn"
__credits__ = ["Brian Delhaisse"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Brian Delhaisse"
__email__ = "briandelhaisse@gmail.com"
__status__ = "Development"
class OculusInterface(VRInterface):
r"""Oculus VR Interface
@@ -53,7 +66,7 @@ class OculusInterface(VRInterface):
super(OculusInterface, self).__init__()
self.MSG_SIZE = 0
self.use_headset = use_headset # TODO: if we use the headset, we automatically use threads
self.use_headset = use_headset # TODO: if we use the headset, we automatically use threads
self.use_threading = use_threading
if use_controllers:
@@ -65,16 +78,16 @@ class OculusInterface(VRInterface):
self.task = None
# create visual spheres in the world for the hands
self.worldCamera = self.world.get_main_camera()
V, P, Vp, V_inv, P_inv, Vp_inv = self.worldCamera.get_matrices(True)
camera = self.worldCamera.get_debug_visualizer_camera(convert=False)
self.world_camera = self.world.get_main_camera()
V, P, Vp, V_inv, P_inv, Vp_inv = self.world_camera.get_matrices(True)
camera = self.world_camera.get_debug_visualizer_camera(convert=False)
width, height = camera[:2]
posL = np.array([width / 2 - 20, height / 2, 0.95, 1])
posR = np.array([width / 2 + 20, height / 2, 0.95, 1])
posL = self.worldCamera.screen_to_world(posL, Vp_inv, P_inv, V_inv)[:3]
posR = self.worldCamera.screen_to_world(posR, Vp_inv, P_inv, V_inv)[:3]
self.leftSphere = self.world.loadVisualSphere(posL, radius=0.1, color=RGBAColor.red) # red
self.rightSphere = self.world.loadVisualSphere(posR, radius=0.1, color=RGBAColor.blue) # blue
left_pos = np.array([width / 2 - 20, height / 2, 0.95, 1])
right_pos = np.array([width / 2 + 20, height / 2, 0.95, 1])
left_pos = self.world_camera.screen_to_world(left_pos, Vp_inv, P_inv, V_inv)[:3]
right_pos = self.world_camera.screen_to_world(right_pos, Vp_inv, P_inv, V_inv)[:3]
self.leftSphere = self.world.loadVisualSphere(left_pos, radius=0.1, color=RGBAColor.red) # red
self.rightSphere = self.world.loadVisualSphere(right_pos, radius=0.1, color=RGBAColor.blue) # blue
# Check IP: get IP of this computer if not provided
if ip:
@@ -87,7 +100,7 @@ class OculusInterface(VRInterface):
self.queue = Queue.Queue(10)
self.threads = []
if self.use_headset:
thread = Thread(target=self.runThread, args=(ip, port + 1))
thread = Thread(target=self.run_thread, args=(ip, port + 1))
self.threads.append(thread)
# Connection over the network for joysticks
@@ -96,16 +109,16 @@ class OculusInterface(VRInterface):
socket.SOCK_STREAM) # TCP = SOCK_STREAM
self.server_address = (ip, port)
self.sock.bind(self.server_address)
self.sock.listen(1) ## Only for TCP
self.sock.listen(1) # Only for TCP
print('Waiting for connection...')
self.connection, self.client_address = self.sock.accept()
# VR
self.head, self.leftHand, self.rightHand = [], [], []
self.leftJoystick, self.rightJoystick = [], []
self.leftVibration, self.rightVibration = 0, 0
self.vibrationTime = 1
self.prevOculusHeadPos = None
self.left_joystick, self.right_joystick = [], []
self.left_vibration, self.right_vibration = 0, 0
self.vibration_time = 1
self.prev_oculus_head_pos = None
# Camera images
if rate is None:
@@ -114,8 +127,8 @@ class OculusInterface(VRInterface):
self.width, self.height = 400, 400
self.encode_params = [int(cv2.IMWRITE_JPEG_QUALITY), 80]
self.leftCollided = False
self.rightCollided = False
self.left_collided = False
self.right_collided = False
def recv(self):
data = self.connection.recvfrom(self.MSG_SIZE)
@@ -132,15 +145,15 @@ class OculusInterface(VRInterface):
elif name == "R": # right hand: position + quaternion
self.rightHand = [values[:3], values[3:]]
elif name == 'JL': # left joystick [touch, button, lateral, forward]
self.leftJoystick = [values[0], values[1], values[-2:]]
self.left_joystick = [values[0], values[1], values[-2:]]
elif name == 'JR': # right joystick: [touch, button, lateral, forward]
self.rightJoystick = [values[0], values[1], values[-2:]]
self.right_joystick = [values[0], values[1], values[-2:]]
# move the camera by rotating
yaw, pitch = values[-2:]
#self.worldCamera.add_yaw_pitch(yaw, pitch, radian=False)
#print(pitch, yaw)
pos = self.worldCamera.target_position
dist = self.worldCamera.dist
# self.worldCamera.add_yaw_pitch(yaw, pitch, radian=False)
# print(pitch, yaw)
pos = self.world_camera.target_position
dist = self.world_camera.dist
elif name == 'BA': # button A: [touch, button]
pass
elif name == 'BB': # button B: [touch, button]
@@ -160,76 +173,78 @@ class OculusInterface(VRInterface):
elif name == 'BRH': # button right hand trigger: trigger
pass
### update world ###
# update world #
headWorldPos = self.worldCamera.position
targetPos = self.worldCamera.target_position
forwardVec, upVec, lateralVec = self.worldCamera.get_vectors()
if self.prevOculusHeadPos is None:
self.prevOculusHeadPos = self.head[0]
head_world_pos = self.world_camera.position
target_pos = self.world_camera.target_position
forward_vec, up_vec, lateral_vec = self.world_camera.get_vectors()
if self.prev_oculus_head_pos is None:
self.prev_oculus_head_pos = self.head[0]
# update world camera position and orientation based on Oculus headset and joysticks
#R = np.array(self.sim.getMatrixFromQuaternion(self.head[1])).reshape(3,3)
#targetPos = R.dot(targetPos)
#targetPos += (self.head[0] - self.prevOculusHeadPos)
lateral, forward = self.leftJoystick[-1] # move the camera by translating
targetPos += 0.1 * (forward * forwardVec + lateral * lateralVec)
self.worldCamera.target_position = targetPos
# # update world camera position and orientation based on Oculus headset and joysticks
# R = np.array(self.sim.getMatrixFromQuaternion(self.head[1])).reshape(3,3)
# target_pos = R.dot(target_pos)
# target_pos += (self.head[0] - self.prevOculusHeadPos)
lateral, forward = self.left_joystick[-1] # move the camera by translating
target_pos += 0.1 * (forward * forward_vec + lateral * lateral_vec)
self.world_camera.target_position = target_pos
# update hand positions in world
leftHandWorldPos = headWorldPos + (self.leftHand[0] - self.head[0])
rightHandWorldPos = headWorldPos + (self.rightHand[0] - self.head[0])
#self.world.move_object(self.leftSphere, self.leftHand[0], (0, 0, 0, 1))
#self.world.move_object(self.rightSphere, self.rightHand[0], (0, 0, 0, 1))
self.world.move_object(self.leftSphere, leftHandWorldPos, (0, 0, 0, 1))
self.world.move_object(self.rightSphere, rightHandWorldPos, (0, 0, 0, 1))
left_hand_world_pos = head_world_pos + (self.leftHand[0] - self.head[0])
right_hand_world_pos = head_world_pos + (self.rightHand[0] - self.head[0])
# self.world.move_object(self.leftSphere, self.leftHand[0], (0, 0, 0, 1))
# self.world.move_object(self.rightSphere, self.rightHand[0], (0, 0, 0, 1))
self.world.move_object(self.leftSphere, left_hand_world_pos, (0, 0, 0, 1))
self.world.move_object(self.rightSphere, right_hand_world_pos, (0, 0, 0, 1))
# change color if hands collide with an object
self.leftCollided = self.updateSphereColor(self.leftSphere, self.leftCollided,
RGBAColor.orange, RGBAColor.red)
self.rightCollided = self.updateSphereColor(self.rightSphere, self.rightCollided,
RGBAColor.green, RGBAColor.blue)
self.left_collided = self.update_sphere_color(self.leftSphere, self.left_collided,
RGBAColor.orange, RGBAColor.red)
self.right_collided = self.update_sphere_color(self.rightSphere, self.right_collided,
RGBAColor.green, RGBAColor.blue)
# get pictures for the eyes
if self.use_headset and (self.cnt % self.rate) == 0:
self.cnt = 0
# get picture for the eyes
leftPic = self.getEyeRGBImage(headWorldPos, targetPos, lateralVec, beta=-0.32)
rightPic = self.getEyeRGBImage(headWorldPos, targetPos, lateralVec, beta=0.32)
#if self.use_threading:
left_pic = self.get_eye_rgb_image(head_world_pos, target_pos, lateral_vec, beta=-0.32)
right_pic = self.get_eye_rgb_image(head_world_pos, target_pos, lateral_vec, beta=0.32)
# if self.use_threading:
# add them to the queue
self.queue.put((leftPic, rightPic))
#else:
self.queue.put((left_pic, right_pic))
# else:
# # compress pictures and send them over the network
# self.compressAndSendPicture(leftPic, self.connection)
# self.compressAndSendPicture(rightPic, self.connection)
# self.compress_and_send_picture(left_pic, self.connection)
# self.compress_and_send_picture(right_pic, self.connection)
self.cnt += 1
def updateSphereColor(self, sphere, hasCollidedPreviously, collisionColor, freeColor):
def update_sphere_color(self, sphere, has_collided_previously, collision_color, free_color):
"""Update sphere color."""
aabb = self.world.get_object_aabb(sphere)
if len(self.world.get_object_ids_in_aabb(aabb[0], aabb[1])) > 1:
update = not hasCollidedPreviously
update = not has_collided_previously
else:
update = hasCollidedPreviously
update = has_collided_previously
if update:
hasCollidedPreviously = not hasCollidedPreviously
if hasCollidedPreviously:
self.world.change_object_color(sphere, color=collisionColor)
has_collided_previously = not has_collided_previously
if has_collided_previously:
self.world.change_object_color(sphere, color=collision_color)
else:
self.world.change_object_color(sphere, color=freeColor)
self.world.change_object_color(sphere, color=free_color)
return hasCollidedPreviously
return has_collided_previously
def runThread(self, ip, port):
def run_thread(self, ip, port):
"""Run thread."""
# create socket for image
sock = socket.socket(socket.AF_INET, # Internet
socket.SOCK_STREAM) # UDP = SOCK_DGRAM / TCP = SOCK_STREAM
server_address = (ip, port) # ip and port
server_address = (ip, port) # ip and port
sock.bind(self.server_address)
sock.listen(1) ## Only for TCP
sock.listen(1) # Only for TCP
print('Thread: waiting for connection...')
connection, client_address = sock.accept()
print('Thread: connected')
@@ -240,27 +255,27 @@ class OculusInterface(VRInterface):
images = self.queue.get(block=True)
# compress pictures and send them over the network
for image in images:
self.compressAndSendPicture(image, connection)
#time.sleep(1./60)
self.compress_and_send_picture(image, connection)
# time.sleep(1./60)
def getEyeRGBImage(self, headWorldPos, targetPos, lateralVec, beta=0.32):
eyePos = headWorldPos + beta * lateralVec
eyeTargetPos = targetPos + beta * lateralVec
V = self.sim.computeViewMatrix(cameraEyePosition=eyePos, cameraTargetPosition=eyeTargetPos,
def get_eye_rgb_image(self, head_world_pos, target_pos, lateral_vec, beta=0.32):
eye_pos = head_world_pos + beta * lateral_vec
eye_target_pos = target_pos + beta * lateral_vec
V = self.sim.computeViewMatrix(cameraEyePosition=eye_pos, cameraTargetPosition=eye_target_pos,
cameraUpVector=(0,0,1))
pic = np.array(self.sim.get_camera_image(self.width, self.height, viewMatrix=V)[2])
pic = pic.reshape(self.width, self.height, 4)[:, :, :3]
return pic
def compressAndSendPicture(self, image, connection):
retval, image = cv2.imencode('.jpg', image, self.encode_params)
def compress_and_send_picture(self, image, connection):
return_value, image = cv2.imencode('.jpg', image, self.encode_params)
image = image.tostring()
connection.sendall(struct.pack('<i', len(image)))
connection.sendall(image)
def send(self):
msg = "VL=" + format(self.leftVibration, '03') + "," + format(self.vibrationTime, '03') + \
";VR=" + format(self.rightVibration, '03') + "," + format(self.vibrationTime, '03')
msg = "VL=" + format(self.left_vibration, '03') + "," + format(self.vibration_time, '03') + \
";VR=" + format(self.right_vibration, '03') + "," + format(self.vibration_time, '03')
self.connection.sendall(msg)
def step(self):
@@ -270,18 +285,18 @@ class OculusInterface(VRInterface):
# alias
update = step
def printState(self):
def print_state(self):
print("Head: {}".format(self.head))
print("Left hand: {}".format(self.leftHand))
print("Right hand: {}".format(self.rightHand))
def setVibration(self, left=0, right=0, vibrationTime=1):
def set_vibration(self, left=0, right=0, vibration_time=1):
"""
Set the level of vibration on the corresponding oculus touch for the specified number of iterations/time.
The level of vibration of each controller is between 0 and 255.
"""
self.leftVibration, self.rightVibration = min(int(left), 255), min(int(right), 255)
self.vibrationTime = min(int(vibrationTime), 200)
self.left_vibration, self.right_vibration = min(int(left), 255), min(int(right), 255)
self.vibration_time = min(int(vibration_time), 200)
def __del__(self):
# stop threads
@@ -295,14 +310,14 @@ class OculusInterface(VRInterface):
# Test
if __name__ == "__main__":
import pybullet as p
from pybullet_envs.bullet.bullet_client import BulletClient
from pyrobolearn.simulators import BulletSim
from pyrobolearn.worlds import BasicWorld
import time
import numpy as np
from itertools import count
# create simulator
sim = BulletClient(connection_mode=p.GUI)
sim = BulletSim()
# create world
world = BasicWorld(sim)
@@ -313,7 +328,7 @@ if __name__ == "__main__":
# run simulation
for t in count():
interface.step()
#interface.printState()
# interface.print_state()
# step in the simulation
world.step()
time.sleep(1./60)
pyrobolearn/tools/interfaces/vr/vr.py
+3
View File
@@ -1,4 +1,7 @@
import openvr
from pyrobolearn.tools.interfaces.interface import InputOutputInterface