[rllib] Add downscale and frameskip options for Montezumas (#908)

* up

* update

* fix

* update

* update

* update

* api break

* Update run_multi_node_tests.sh

* fix

python/ray/rllib/ppo/env.py

@@ -10,13 +10,15 @@ from ray.rllib.models import ModelCatalog
 
 class BatchedEnv(object):
     """This holds multiple gym envs and performs steps on all of them."""
-    def __init__(self, name, batchsize):
+    def __init__(self, name, batchsize, options):
         self.envs = [gym.make(name) for _ in range(batchsize)]
         self.observation_space = self.envs[0].observation_space
         self.action_space = self.envs[0].action_space
         self.batchsize = batchsize
         self.preprocessor = ModelCatalog.get_preprocessor(
-            name, self.envs[0].observation_space.shape)
+            name, self.envs[0].observation_space.shape, options["model"])
+        self.extra_frameskip = options.get("extra_frameskip", 1)
+        assert self.extra_frameskip >= 1
 
     def reset(self):
         observations = [
@@ -33,7 +35,12 @@ class BatchedEnv(object):
                 observations.append(np.zeros(self.shape))
                 rewards.append(0.0)
                 continue
-            observation, reward, done, info = self.envs[i].step(action)
+            reward = 0.0
+            for j in range(self.extra_frameskip):
+                observation, r, done, info = self.envs[i].step(action)
+                reward += r
+                if done:
+                    break
             if render:
                 self.envs[0].render()
             observations.append(self.preprocessor.transform(observation))