solving compatbility issues with baselines + update envs, rewards, terminal conditions, states/actions

examples/README.rst

@@ -21,4 +21,4 @@ You can check the following folders on:
 - ``rewards``: how to use the reward functions.
 - ``environments``: provide a full example on how to create an environment from scratch in PRL.
 - ``imitation``: how to use imitation learning with the framework.
-- ``gym/cartpole``: policies that are trained with different algorithms on the gym Cartpole environment.
+- ``reinforcement``: how to use reinforcement learning with the framework.

examples/kinematics/inverse/moving_sphere.py

@@ -72,4 +72,5 @@ for t in count():
     robot.set_joint_positions(q, joint_ids=joint_ids)
 
     # step in simulation
+    robot.step()
     world.step(sleep_dt=dt)

examples/reinforcement/README.md

@@ -0,0 +1,9 @@
+## Reinforcement learning task
+
+In this folder, you can run reinforcement learning tasks.
+
+- In the `gym` subfolder, you can run `gym` environments using the models and algorithms available from the PRL 
+  frameworks.
+- In the `baselines` subfolder, you can `PRL` environments using the neural networks models and algorithms provided by 
+  the `stable_baselines` library.
+- Other example files provide PRL environments along with models and algorithms provided by PRL.

examples/reinforcement/baselines/README.rst

@@ -0,0 +1,14 @@
+Baselines
+---------
+
+This folder contains examples when using PRL environments and algorithms defined in the ``stable_baselines`` Python
+library.
+
+Few notes with respect to that:
+
+1. ``stable_baselines`` uses the ``TensorFlow`` backend, and a ``DummyVecEnv`` has to be provided to the algorithms.
+2. Normally, in PRL, the actions can be defined outside the environments and it is the policy that is responsible to
+   apply the action in the world. However, in ``OpenAI gym``, it is the environment that has the ``action_space`` and
+   apply the ``action``. To accommodate with that, the action can also be defined and provided to the PRL environment.
+3. When using PRL with ``stable_baselines``, make sure that each states have the same dimensions; i.e. we can not
+   return a 1D vector state with a 2D matrix state at the same time (at least, not currently).

examples/reinforcement/baselines/acrobot_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Acrobot' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('Acrobot-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/acrobot_prl.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""Example on how to use the PRL 'Acrobot' environment using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+import gym
+
+import pyrobolearn as prl
+from pyrobolearn.envs.control.acrobot import AcrobotEnv
+
+# create env, state, and action from gym
+sim = prl.simulators.Bullet(render=True)
+env = AcrobotEnv(sim)
+print("State and action space: {} and {}".format(env.state.space, env.action.space))
+print("State and action merged space: {} and {}".format(env.state.merged_space, env.action.merged_space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+# env.render()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    # env.render()

examples/reinforcement/baselines/cartpole_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/pendulum_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Pendulum' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('Pendulum-v0')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/pendulum_prl.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""Example on how to use the PRL 'Acrobot' environment using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+import gym
+
+import pyrobolearn as prl
+from pyrobolearn.envs.control.pendulum import InvertedPendulumSwingUpEnv
+
+# create env, state, and action from gym
+sim = prl.simulators.Bullet(render=True)
+env = InvertedPendulumSwingUpEnv(sim)
+print("State and action space: {} and {}".format(env.state.space, env.action.space))
+print("State and action merged space: {} and {}".format(env.state.merged_space, env.action.merged_space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+# env.render()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    # env.render()