[autoscaler]: automatically pull new docker image (#6111)

* Docker: automatically pull new image

* Fix missing value in schema

* Address review comments

python/ray/autoscaler/autoscaler.py

@@ -106,6 +106,7 @@ CLUSTER_CONFIG_SCHEMA = {
         {
             "image": (str, OPTIONAL),  # e.g. tensorflow/tensorflow:1.5.0-py3
             "container_name": (str, OPTIONAL),  # e.g., ray_docker
+            "pull_before_run": (bool, OPTIONAL),  # run `docker pull` first
             # shared options for starting head/worker docker
             "run_options": (list, OPTIONAL),
 

python/ray/autoscaler/aws/example-full.yaml

@@ -25,6 +25,9 @@ autoscaling_mode: default
 docker:
     image: "" # e.g., tensorflow/tensorflow:1.5.0-py3
     container_name: "" # e.g. ray_docker
+    # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
+    # if no cached version is present.
+    pull_before_run: True
     run_options: []  # Extra options to pass into "docker run"
 
     # Example of running a GPU head with CPU workers

python/ray/autoscaler/docker.py

@@ -17,6 +17,7 @@ def dockerize_if_needed(config):
         return config
 
     docker_image = config["docker"].get("image")
+    docker_pull = config["docker"].get("pull_before_run", True)
     cname = config["docker"].get("container_name")
     run_options = config["docker"].get("run_options", [])
 
@@ -37,6 +38,10 @@ def dockerize_if_needed(config):
         assert cname, "Must provide container name!"
     docker_mounts = {dst: dst for dst in config["file_mounts"]}
 
+    if docker_pull:
+        docker_pull_cmd = "docker pull {}".format(docker_image)
+        config["initialization_commands"].append(docker_pull_cmd)
+
     head_docker_start = docker_start_cmds(ssh_user, head_docker_image,
                                           docker_mounts, cname,
                                           run_options + head_run_options)

python/ray/autoscaler/gcp/example-full.yaml

@@ -25,6 +25,9 @@ autoscaling_mode: default
 docker:
     image: "" # e.g., tensorflow/tensorflow:1.5.0-py3
     container_name: "" # e.g. ray_docker
+    # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
+    # if no cached version is present.
+    pull_before_run: True
     run_options: []  # Extra options to pass into "docker run"
 
 

python/ray/autoscaler/local/example-full.yaml

@@ -28,6 +28,9 @@ idle_timeout_minutes: 5
 docker:
     image: "" # e.g., tensorflow/tensorflow:1.5.0-py3
     container_name: "" # e.g. ray_docker
+    # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
+    # if no cached version is present.
+    pull_before_run: True
     run_options: []  # Extra options to pass into "docker run"
 
 # Local specific configuration.