diff --git a/examples/rewards/README.rst b/examples/rewards/README.rst new file mode 100644 index 0000000..7662883 --- /dev/null +++ b/examples/rewards/README.rst @@ -0,0 +1,73 @@ +Rewards +======= + +In this folder, we provide examples on how to use reward/cost functions which are provided to reinforcement learning environments. +We show the available operations you can use on these. + +The reward function might be defined as [1]_: + +- :math:`r: \mathcal{S} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}`, it returns the reward value :math:`r(s)`. +- :math:`r: \mathcal{S} \times \mathcal{A} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}` and action :math:`a \in \mathcal{A}`, it returns the reward value :math:`r(s,a)`. +- :math:`r: \mathcal{S} \times \mathcal{A} \times \mathcal{S} \rightarrow \mathbb{R}`: given the state :math:`s \in \mathcal{S}`, action :math:`a \in \mathcal{A}`, and next state :math:`s' \in \mathcal{S}`, it returns the reward value :math:`r(s,a,s')`. + +Note that the cost function is just minus the reward function, i.e. it is given by :math:`c(s,a,s') = -r(s,a,s')`. + +In PRL, all reward functions inherit from the abstract ``Reward`` class defined in `pyrobolearn/rewards/reward.py `_, and several methods and operations are provided. +You can for instance: + +* provide the ``State`` and/or ``Action`` instances to some reward functions that will compute the reward value based on their value. +* access to the range of the reward function. +* add, multiply, divide, subtract, and apply basic functions such as :math:`\exp`, :math:`\cos`, :math:`\sin`, and others on reward functions. The resulting range is automatically scaled based on the operations. +* define your own rewards/costs and reuse them in your code. + +Here is a short snippet showing the basic usage of reward functions: + +.. code-block:: python + :linenos: + + import pyrobolearn as prl + from pyrobolearn.rewards import FixedReward, YourReward + + # define the simulator and world (and load what you want in it) + sim = ... + world = ... + ... + + # define your state / action for your reward function + state = ... + action = ... + + # define the reward function + reward = 2 * FixedReward(3) + 0.5 * YourReward(state, action) + + # print the range of the reward function + print(reward.range) + + # compute the reward value + value = reward() + print(value) + + # update the state for instance + state() # this will modify the internal state data + + # recompute the reward value + value = reward() + print(value) # you will normally get a different value + + # you can give the reward function to your RL environment + # which will use it when calling `env.step()`. + env = prl.envs.Env(world, state, reward) + + +Examples +~~~~~~~~ + +Here are few examples that you can find in this folder that better demonstrate how to use the reward functions: + +1. ``basics.py``: demonstrate the various features (operations) you can use with the ``Reward`` class. +2. ``manipulator.py``: show how the distance cost decreases as you move the manipulator (with your mouse) closer to the target object in the world. +3. ``forward_progress.py``: show how the reward function that measures how much a robot has moved forward increases / decreases based on the robot velocity. Use the arrow keys on your keyboard to move the robot, and observe how the computed reward value changes. + +References: + +.. [1] "Reinforcement Learning: An Introduction", Sutton and Barto, 1998 \ No newline at end of file diff --git a/examples/rewards/basics.py b/examples/rewards/basics.py new file mode 100644 index 0000000..7ce4dd4 --- /dev/null +++ b/examples/rewards/basics.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +"""Demonstrate the various features (operations) you can use with the ``Reward`` class. + +To illustrate the various operations we use the ``FixedReward``. + +See the other examples to see how to use more complex reward functions. +""" + +import numpy as np + +# You can import in two ways reward functions +import pyrobolearn as prl # import PRL +from pyrobolearn.rewards import FixedReward # specify which function you want to import + +# You can also import all the reward functions and the mathematical functions but I usually avoid it because +# we do not know while reading the code where the various classes and other functionalities come from +# from pyrobolearn.rewards import * + + +# define two fixed reward functions +r1 = prl.rewards.FixedReward(value=3) +r2 = FixedReward(value=2, range=(-2, 2)) + +# print their value by calling them +print("\nInitial rewards: r1 = {}, and r2 = {}".format(r1, r2)) +print("Initial reward value: r1() = {}, and r2() = {}".format(r1(), r2())) +print("Initial reward range: range(r1) = {}, and range(r2) = {}".format(r1.range, r2.range)) + +# try to define a fixed reward function where the initial value is not in the defined range +try: + r3 = FixedReward(value=2, range=(-1, 1)) +except ValueError as e: + print("\nTrying `r3=FixedReward(value=2, range=(-1,1))` results in an error: \n" + str(e) + "\n") + + +# perform some mathematical operations on them +r4 = 2 * prl.rewards.cos(r1) - 3 * r2 + +# print its value and range +print("Perform mathematical operations on r1 and r2:") +print("r4 = 2 * cos(r1) - 3 * r2 = {}".format(r4())) +print("2 * cos(3) - 3 * 2 = {}".format(2 * np.cos(3) - 3 * 2)) +print("range(r4) = {}".format(r4.range)) + + +# try to perform an operation which it not authorized +try: + r5 = r1 / r2 # the range of r2 is [-2, 2] and thus there is a chance it could be 0 at one point +except ValueError as e: + print("\nTrying `r5 = r1 / r2` results in an error: \n" + str(e) + "\n") + + +# The range of r2 is [-2, 2], if a reward function computes a reward value which is not in the range, +# it will automatically be clipped. +r2.value = 3 # you can only set the value for the FixedReward +print("Setting r2.value=3 while its range is [-2, 2], and computing the reward will result in: r2={}".format(r2())) diff --git a/examples/rewards/forward_progress.py b/examples/rewards/forward_progress.py new file mode 100644 index 0000000..14a9af7 --- /dev/null +++ b/examples/rewards/forward_progress.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +"""Demonstrate how the reward function that measures how much a robot has moved forward increases / decreases based on +the robot velocity. Use the arrow keys on your keyboard to move the robot, and observe how the computed reward value +changes. +""" + +from itertools import count +import pyrobolearn as prl + + +# Create simulator +sim = prl.simulators.Bullet() + +# create world +world = prl.worlds.BasicWorld(sim) + +# create wheeled robot +robot = world.load_robot('epuck') + +# create interface and bridge to control the robot with the keyboard +interface = prl.tools.interfaces.MouseKeyboardInterface() +bridge = prl.tools.bridges.BridgeMouseKeyboardDifferentialWheeledRobot(robot=robot, interface=interface) + +# create state +state = prl.states.BasePositionState(robot) + +# create reward +reward = 1000 * prl.rewards.ForwardProgressReward(state=state, direction=(1, 0, 0)) + + +# run simulation +for t in count(): + # perform a step with the bridge and interface + bridge.step(update_interface=True) + + # update state: in this case it will get the base position state and will save it in the state instance + state() + + # compute reward: this will look in the previously given state instance its current state data + print("Reward value = {}".format(reward())) + + # perform a step in the simulator + world.step(sim.dt) diff --git a/examples/rewards/manipulator.py b/examples/rewards/manipulator.py new file mode 100644 index 0000000..76cc7ba --- /dev/null +++ b/examples/rewards/manipulator.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +"""Demonstrate how the distance cost decreases as you move the manipulator (with your mouse) closer to the target +object in the world. +""" + +from itertools import count +import pyrobolearn as prl + + +# Create simulator +sim = prl.simulators.Bullet() + +# create world +world = prl.worlds.BasicWorld(sim) + +# create robot +robot = world.load_robot('kuka_iiwa') +print(robot) + +# desired position +sphere = world.load_visual_sphere([0.5, 0., 0.5], radius=0.05, color=(1, 0, 0, 0.5), return_body=True) + +# create state +state = prl.states.LinkPositionState(robot, link_ids=robot.end_effectors) + +# create reward +# reward = prl.rewards.DistanceCost() + +# run simulation +for t in count(): + # update state + state() + + # compute reward + # print("Reward value = {}".format(reward())) + + # perform a step in the simulator + world.step(sim.dt)