add rewards examples

examples/rewards/README.rst

@@ -0,0 +1,73 @@
+Rewards
+=======
+
+In this folder, we provide examples on how to use reward/cost functions which are provided to reinforcement learning environments.
+We show the available operations you can use on these.
+
+The reward function might be defined as [1]_:
+
+- :math:`r: \mathcal{S} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}`, it returns the reward value :math:`r(s)`.
+- :math:`r: \mathcal{S} \times \mathcal{A} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}` and action :math:`a \in \mathcal{A}`, it returns the reward value :math:`r(s,a)`.
+- :math:`r: \mathcal{S} \times \mathcal{A} \times \mathcal{S} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}`, action :math:`a \in \mathcal{A}`, and next state :math:`s' \in \mathcal{S}`, it returns the reward value :math:`r(s,a,s')`.
+
+Note that the cost function is just minus the reward function, i.e. it is given by :math:`c(s,a,s') = -r(s,a,s')`.
+
+In PRL, all reward functions inherit from the abstract ``Reward`` class defined in `pyrobolearn/rewards/reward.py <https://github.com/robotlearn/pyrobolearn/tree/master/pyrobolearn/rewards>`_, and several methods and operations are provided.
+You can for instance:
+
+* provide the ``State`` and/or ``Action`` instances to some reward functions that will compute the reward value based on their value.
+* access to the range of the reward function.
+* add, multiply, divide, subtract, and apply basic functions such as :math:`\exp`, :math:`\cos`, :math:`\sin`, and others on reward functions. The resulting range is automatically scaled based on the operations.
+* define your own rewards/costs and reuse them in your code.
+
+Here is a short snippet showing the basic usage of reward functions:
+
+.. code-block:: python
+    :linenos:
+
+    import pyrobolearn as prl
+    from pyrobolearn.rewards import FixedReward, YourReward
+
+    # define the simulator and world (and load what you want in it)
+    sim = ...
+    world = ...
+    ...
+
+    # define your state / action for your reward function
+    state = ...
+    action = ...
+
+    # define the reward function
+    reward = 2 * FixedReward(3) + 0.5 * YourReward(state, action)
+
+    # print the range of the reward function
+    print(reward.range)
+
+    # compute the reward value
+    value = reward()
+    print(value)
+
+    # update the state for instance
+    state()     # this will modify the internal state data
+
+    # recompute the reward value
+    value = reward()
+    print(value)    # you will normally get a different value
+
+    # you can give the reward function to your RL environment
+    # which will use it when calling `env.step()`.
+    env = prl.envs.Env(world, state, reward)
+
+
+Examples
+~~~~~~~~
+
+Here are few examples that you can find in this folder that better demonstrate how to use the reward functions:
+
+1. ``basics.py``: demonstrate the various features (operations) you can use with the ``Reward`` class.
+2. ``manipulator.py``: show how the distance cost decreases as you move the manipulator (with your mouse) closer to the target object in the world.
+3. ``forward_progress.py``: show how the reward function that measures how much a robot has moved forward increases / decreases based on the robot velocity. Use the arrow keys on your keyboard to move the robot, and observe how the computed reward value changes.
+
+References:
+
+.. [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 1998

examples/rewards/basics.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""Demonstrate the various features (operations) you can use with the ``Reward`` class.
+
+To illustrate the various operations we use the ``FixedReward``.
+
+See the other examples to see how to use more complex reward functions.
+"""
+
+import numpy as np
+
+# You can import in two ways reward functions
+import pyrobolearn as prl                       # import PRL
+from pyrobolearn.rewards import FixedReward     # specify which function you want to import
+
+# You can also import all the reward functions and the mathematical functions but I usually avoid it because
+# we do not know while reading the code where the various classes and other functionalities come from
+# from pyrobolearn.rewards import *
+
+
+# define two fixed reward functions
+r1 = prl.rewards.FixedReward(value=3)
+r2 = FixedReward(value=2, range=(-2, 2))
+
+# print their value by calling them
+print("\nInitial rewards: r1 = {}, and r2 = {}".format(r1, r2))
+print("Initial reward value: r1() = {}, and r2() = {}".format(r1(), r2()))
+print("Initial reward range: range(r1) = {}, and range(r2) = {}".format(r1.range, r2.range))
+
+# try to define a fixed reward function where the initial value is not in the defined range
+try:
+    r3 = FixedReward(value=2, range=(-1, 1))
+except ValueError as e:
+    print("\nTrying `r3=FixedReward(value=2, range=(-1,1))` results in an error: \n" + str(e) + "\n")
+
+
+# perform some mathematical operations on them
+r4 = 2 * prl.rewards.cos(r1) - 3 * r2
+
+# print its value and range
+print("Perform mathematical operations on r1 and r2:")
+print("r4 = 2 * cos(r1) - 3 * r2 = {}".format(r4()))
+print("2 * cos(3) - 3 * 2 = {}".format(2 * np.cos(3) - 3 * 2))
+print("range(r4) = {}".format(r4.range))
+
+
+# try to perform an operation which it not authorized
+try:
+    r5 = r1 / r2        # the range of r2 is [-2, 2] and thus there is a chance it could be 0 at one point
+except ValueError as e:
+    print("\nTrying `r5 = r1 / r2` results in an error: \n" + str(e) + "\n")
+
+
+# The range of r2 is [-2, 2], if a reward function computes a reward value which is not in the range,
+# it will automatically be clipped.
+r2.value = 3        # you can only set the value for the FixedReward
+print("Setting r2.value=3 while its range is [-2, 2], and computing the reward will result in: r2={}".format(r2()))

examples/rewards/forward_progress.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""Demonstrate how the reward function that measures how much a robot has moved forward increases / decreases based on
+the robot velocity. Use the arrow keys on your keyboard to move the robot, and observe how the computed reward value
+changes.
+"""
+
+from itertools import count
+import pyrobolearn as prl
+
+
+# Create simulator
+sim = prl.simulators.Bullet()
+
+# create world
+world = prl.worlds.BasicWorld(sim)
+
+# create wheeled robot
+robot = world.load_robot('epuck')
+
+# create interface and bridge to control the robot with the keyboard
+interface = prl.tools.interfaces.MouseKeyboardInterface()
+bridge = prl.tools.bridges.BridgeMouseKeyboardDifferentialWheeledRobot(robot=robot, interface=interface)
+
+# create state
+state = prl.states.BasePositionState(robot)
+
+# create reward
+reward = 1000 * prl.rewards.ForwardProgressReward(state=state, direction=(1, 0, 0))
+
+
+# run simulation
+for t in count():
+    # perform a step with the bridge and interface
+    bridge.step(update_interface=True)
+
+    # update state: in this case it will get the base position state and will save it in the state instance
+    state()
+
+    # compute reward: this will look in the previously given state instance its current state data
+    print("Reward value = {}".format(reward()))
+
+    # perform a step in the simulator
+    world.step(sim.dt)

examples/rewards/manipulator.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Demonstrate how the distance cost decreases as you move the manipulator (with your mouse) closer to the target
+object in the world.
+"""
+
+from itertools import count
+import pyrobolearn as prl
+
+
+# Create simulator
+sim = prl.simulators.Bullet()
+
+# create world
+world = prl.worlds.BasicWorld(sim)
+
+# create robot
+robot = world.load_robot('kuka_iiwa')
+print(robot)
+
+# desired position
+sphere = world.load_visual_sphere([0.5, 0., 0.5], radius=0.05, color=(1, 0, 0, 0.5), return_body=True)
+
+# create state
+state = prl.states.LinkPositionState(robot, link_ids=robot.end_effectors)
+
+# create reward
+# reward = prl.rewards.DistanceCost()
+
+# run simulation
+for t in count():
+    # update state
+    state()
+
+    # compute reward
+    # print("Reward value = {}".format(reward()))
+
+    # perform a step in the simulator
+    world.step(sim.dt)