solving compatbility issues with baselines + update envs, rewards, terminal conditions, states/actions

examples/reinforcement/README.md

@@ -0,0 +1,9 @@
+## Reinforcement learning task
+
+In this folder, you can run reinforcement learning tasks.
+
+- In the `gym` subfolder, you can run `gym` environments using the models and algorithms available from the PRL 
+  frameworks.
+- In the `baselines` subfolder, you can `PRL` environments using the neural networks models and algorithms provided by 
+  the `stable_baselines` library.
+- Other example files provide PRL environments along with models and algorithms provided by PRL.

examples/reinforcement/baselines/README.rst

@@ -0,0 +1,14 @@
+Baselines
+---------
+
+This folder contains examples when using PRL environments and algorithms defined in the ``stable_baselines`` Python
+library.
+
+Few notes with respect to that:
+
+1. ``stable_baselines`` uses the ``TensorFlow`` backend, and a ``DummyVecEnv`` has to be provided to the algorithms.
+2. Normally, in PRL, the actions can be defined outside the environments and it is the policy that is responsible to
+   apply the action in the world. However, in ``OpenAI gym``, it is the environment that has the ``action_space`` and
+   apply the ``action``. To accommodate with that, the action can also be defined and provided to the PRL environment.
+3. When using PRL with ``stable_baselines``, make sure that each states have the same dimensions; i.e. we can not
+   return a 1D vector state with a 2D matrix state at the same time (at least, not currently).

examples/reinforcement/baselines/acrobot_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Acrobot' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('Acrobot-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/acrobot_prl.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""Example on how to use the PRL 'Acrobot' environment using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+import gym
+
+import pyrobolearn as prl
+from pyrobolearn.envs.control.acrobot import AcrobotEnv
+
+# create env, state, and action from gym
+sim = prl.simulators.Bullet(render=True)
+env = AcrobotEnv(sim)
+print("State and action space: {} and {}".format(env.state.space, env.action.space))
+print("State and action merged space: {} and {}".format(env.state.merged_space, env.action.merged_space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+# env.render()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    # env.render()

examples/reinforcement/baselines/cartpole_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/pendulum_gym.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Pendulum' OpenAI Gym environments in PRL using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+from pyrobolearn.envs import gym  # this is a thin wrapper around the gym library
+
+# create env, state, and action from gym
+env = gym.make('Pendulum-v0')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    env.render()

examples/reinforcement/baselines/pendulum_prl.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""Example on how to use the PRL 'Acrobot' environment using the `stable_baselines` library.
+"""
+
+from stable_baselines.common.policies import MlpPolicy
+from stable_baselines.common.vec_env import DummyVecEnv
+from stable_baselines import PPO2
+
+import gym
+
+import pyrobolearn as prl
+from pyrobolearn.envs.control.pendulum import InvertedPendulumSwingUpEnv
+
+# create env, state, and action from gym
+sim = prl.simulators.Bullet(render=True)
+env = InvertedPendulumSwingUpEnv(sim)
+print("State and action space: {} and {}".format(env.state.space, env.action.space))
+print("State and action merged space: {} and {}".format(env.state.merged_space, env.action.merged_space))
+
+# The algorithms require a vectorized environment to run
+env = DummyVecEnv([lambda: env])
+
+model = PPO2(MlpPolicy, env, verbose=1)
+model.learn(total_timesteps=10000)
+
+obs = env.reset()
+# env.render()
+for i in range(1000):
+    action, _states = model.predict(obs)
+    obs, rewards, dones, info = env.step(action)
+    # env.render()

examples/reinforcement/gym/cartpole/README.md

@@ -0,0 +1,3 @@
+## Cartpole gym example
+
+In this folder, you can try different policies and RL algorithms to train them. Just run the file `python <policy>_<algo>.py`.

examples/reinforcement/gym/cartpole/linear_bo.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a linear policy trained with
+the Bayesian Optimization algorithm.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import BO
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = LinearPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+algo = BO(task, policy)
+rewards = algo.train(num_steps=1000, num_episodes=30, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(rewards)
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/linear_cem.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a linear policy trained with
+the Cross-Entropy Method algorithm.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import CEM
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = LinearPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+algo = CEM(task, policy, population_size=20, elite_fraction=0.2)
+avg_rewards, max_rewards = algo.train(num_steps=1000, num_episodes=30, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(avg_rewards, label='avg')
+plt.plot(max_rewards, label='max')
+plt.legend()
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/linear_cmaes.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a linear policy trained with
+the CMA-ES algorithm.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import CMAES
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = LinearPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+algo = CMAES(task, policy, population_size=20)
+avg_rewards, max_rewards = algo.train(num_steps=1000, num_episodes=30, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(avg_rewards, label='avg')
+plt.plot(max_rewards, label='max')
+plt.legend()
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/linear_fd.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a linear policy trained with
+the finite difference algorithm.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import FD
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = LinearPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+# Note: the hyperparameters can be a little bit tricky to optimize...
+algo = FD(task, policy, std_dev=0.01, learning_rate=0.01, difference_type='central', normalize_grad=True)
+rewards = algo.train(num_steps=1000, num_rollouts=5, num_episodes=50, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(rewards)
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/linear_power.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a linear policy trained with
+the PoWER RL algorithm.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import LinearPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import PoWER
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = LinearPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+# Note: depends a lot on the initialization of the policy
+algo = PoWER(task, policy)
+rewards = algo.train(num_steps=1000, num_rollouts=10, num_episodes=300, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(rewards)
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/neat_neat.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using NEAT.
+"""
+
+import matplotlib.pyplot as plt
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import NEATPolicy
+from pyrobolearn.tasks import RLTask
+from pyrobolearn.algos import NEAT
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = NEATPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, use_terminating_condition=True, render=True)
+
+# create RL algo
+algo = NEAT(task, policy, population_size=20)
+avg_rewards, max_rewards = algo.train(num_steps=1000, num_episodes=30, verbose=True)
+
+# plot
+plt.figure()
+plt.plot(avg_rewards, label='avg')
+plt.plot(max_rewards, label='max')
+plt.legend()
+plt.show()
+
+# test optimized policy
+reward = algo.test(num_steps=1000, use_terminating_condition=True, render=True)
+print("Final reward obtained on the test: {}".format(reward))

examples/reinforcement/gym/cartpole/random_none.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+"""Example on how to use the 'Cartpole' OpenAI Gym environments in PyRoboLearn using a random policy
+"""
+
+from pyrobolearn.envs import gym
+from pyrobolearn.policies import RandomPolicy
+from pyrobolearn.tasks import RLTask
+
+
+# create env, state, and action from gym
+env = gym.make('CartPole-v1')
+state, action = env.state, env.action
+print("State and action space: {} and {}".format(state.space, action.space))
+
+# create policy
+policy = RandomPolicy(state, action)
+
+# create task and run it
+task = RLTask(env, policy)
+task.run(num_steps=1000, dt=0.02, use_terminating_condition=False, render=True)