[rllib] MAML Agent (#8862)

* Halfway done with transferring MAML to new Ray

* MAML Beta Out

* Debugging MAML atm

* Distributed Execution

* Pendulum Mass Working

* All experiments complete

* Cleaned up codebase

* Travis CI

* Travis CI

* Tests

* Merged conflicts

* Fixed variance bug conflict

* Comment resolved

* Apply suggestions from code review

fixed test_maml

* Update rllib/agents/maml/tests/test_maml.py

* asdf

* Fix testing

Co-authored-by: Sven Mika <sven@anyscale.io>

rllib/examples/env/ant_rand_goal.py

@@ -0,0 +1,73 @@
+import numpy as np
+import gym
+from gym.envs.mujoco.mujoco_env import MujocoEnv
+
+
+class AntRandGoalEnv(gym.utils.EzPickle, MujocoEnv):
+    """Ant Environment that randomizes goals as tasks
+
+    Goals are randomly sampled 2D positions
+    """
+
+    def __init__(self):
+        self.set_task(self.sample_tasks(1)[0])
+        MujocoEnv.__init__(self, "ant.xml", 5)
+        gym.utils.EzPickle.__init__(self)
+
+    def sample_tasks(self, n_tasks):
+        # Samples a goal position (2x1 position ector)
+        a = np.random.random(n_tasks) * 2 * np.pi
+        r = 3 * np.random.random(n_tasks)**0.5
+        return np.stack((r * np.cos(a), r * np.sin(a)), axis=-1)
+
+    def set_task(self, task):
+        """
+        Args:
+            task: task of the meta-learning environment
+        """
+        self.goal_pos = task
+
+    def get_task(self):
+        """
+        Returns:
+            task: task of the meta-learning environment
+        """
+        return self.goal_pos
+
+    def step(self, a):
+        self.do_simulation(a, self.frame_skip)
+        xposafter = self.get_body_com("torso")
+        goal_reward = -np.sum(np.abs(
+            xposafter[:2] - self.goal_pos))  # make it happy, not suicidal
+        ctrl_cost = .1 * np.square(a).sum()
+        contact_cost = 0.5 * 1e-3 * np.sum(
+            np.square(np.clip(self.sim.data.cfrc_ext, -1, 1)))
+        # survive_reward = 1.0
+        survive_reward = 0.0
+        reward = goal_reward - ctrl_cost - contact_cost + survive_reward
+        # notdone = np.isfinite(state).all() and 1.0 >= state[2] >= 0.
+        # done = not notdone
+        done = False
+        ob = self._get_obs()
+        return ob, reward, done, dict(
+            reward_forward=goal_reward,
+            reward_ctrl=-ctrl_cost,
+            reward_contact=-contact_cost,
+            reward_survive=survive_reward)
+
+    def _get_obs(self):
+        return np.concatenate([
+            self.sim.data.qpos.flat,
+            self.sim.data.qvel.flat,
+            np.clip(self.sim.data.cfrc_ext, -1, 1).flat,
+        ])
+
+    def reset_model(self):
+        qpos = self.init_qpos + self.np_random.uniform(
+            size=self.model.nq, low=-.1, high=.1)
+        qvel = self.init_qvel + self.np_random.randn(self.model.nv) * .1
+        self.set_state(qpos, qvel)
+        return self._get_obs()
+
+    def viewer_setup(self):
+        self.viewer.cam.distance = self.model.stat.extent * 0.5

rllib/examples/env/halfcheetah_rand_direc.py

@@ -0,0 +1,62 @@
+import numpy as np
+import gym
+from gym.envs.mujoco.mujoco_env import MujocoEnv
+
+
+class HalfCheetahRandDirecEnv(MujocoEnv, gym.utils.EzPickle):
+    """HalfCheetah Environment with two diff tasks, moving forwards or backwards
+
+    Direction is defined as a scalar: +1.0 (forwards) or -1.0 (backwards)
+    """
+
+    def __init__(self, goal_direction=None):
+        self.goal_direction = goal_direction if goal_direction else 1.0
+        MujocoEnv.__init__(self, "half_cheetah.xml", 5)
+        gym.utils.EzPickle.__init__(self, goal_direction)
+
+    def sample_tasks(self, n_tasks):
+        # For fwd/bwd env, goal direc is backwards if - 1.0, forwards if + 1.0
+        return np.random.choice((-1.0, 1.0), (n_tasks, ))
+
+    def set_task(self, task):
+        """
+        Args:
+            task: task of the meta-learning environment
+        """
+        self.goal_direction = task
+
+    def get_task(self):
+        """
+        Returns:
+            task: task of the meta-learning environment
+        """
+        return self.goal_direction
+
+    def step(self, action):
+        xposbefore = self.sim.data.qpos[0]
+        self.do_simulation(action, self.frame_skip)
+        xposafter = self.sim.data.qpos[0]
+        ob = self._get_obs()
+        reward_ctrl = -0.5 * 0.1 * np.square(action).sum()
+        reward_run = self.goal_direction * (xposafter - xposbefore) / self.dt
+        reward = reward_ctrl + reward_run
+        done = False
+        return ob, reward, done, dict(
+            reward_run=reward_run, reward_ctrl=reward_ctrl)
+
+    def _get_obs(self):
+        return np.concatenate([
+            self.sim.data.qpos.flat[1:],
+            self.sim.data.qvel.flat,
+        ])
+
+    def reset_model(self):
+        qpos = self.init_qpos + self.np_random.uniform(
+            low=-.1, high=.1, size=self.model.nq)
+        qvel = self.init_qvel + self.np_random.randn(self.model.nv) * .1
+        self.set_state(qpos, qvel)
+        obs = self._get_obs()
+        return obs
+
+    def viewer_setup(self):
+        self.viewer.cam.distance = self.model.stat.extent * 0.5

rllib/examples/env/pendulum_mass.py

@@ -0,0 +1,28 @@
+import numpy as np
+import gym
+from gym.envs.classic_control.pendulum import PendulumEnv
+
+
+class PendulumMassEnv(PendulumEnv, gym.utils.EzPickle):
+    """PendulumMassEnv varies the weight of the pendulum
+
+    Tasks are defined to be weight uniformly sampled between [0.5,2]
+    """
+
+    def sample_tasks(self, n_tasks):
+        # Mass is a random float between 0.5 and 2
+        return np.random.uniform(low=0.5, high=2.0, size=(n_tasks, ))
+
+    def set_task(self, task):
+        """
+        Args:
+            task: task of the meta-learning environment
+        """
+        self.m = task
+
+    def get_task(self):
+        """
+        Returns:
+            task: task of the meta-learning environment
+        """
+        return self.m