diff --git a/python/ray/autoscaler/autoscaler.py b/python/ray/autoscaler/autoscaler.py index 092d80c42..1bb96018a 100644 --- a/python/ray/autoscaler/autoscaler.py +++ b/python/ray/autoscaler/autoscaler.py @@ -106,6 +106,7 @@ CLUSTER_CONFIG_SCHEMA = { { "image": (str, OPTIONAL), # e.g. tensorflow/tensorflow:1.5.0-py3 "container_name": (str, OPTIONAL), # e.g., ray_docker + "pull_before_run": (bool, OPTIONAL), # run `docker pull` first # shared options for starting head/worker docker "run_options": (list, OPTIONAL), diff --git a/python/ray/autoscaler/aws/example-full.yaml b/python/ray/autoscaler/aws/example-full.yaml index 67654d53a..cde18a454 100644 --- a/python/ray/autoscaler/aws/example-full.yaml +++ b/python/ray/autoscaler/aws/example-full.yaml @@ -25,6 +25,9 @@ autoscaling_mode: default docker: image: "" # e.g., tensorflow/tensorflow:1.5.0-py3 container_name: "" # e.g. ray_docker + # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image + # if no cached version is present. + pull_before_run: True run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers diff --git a/python/ray/autoscaler/docker.py b/python/ray/autoscaler/docker.py index c1b384dee..8088e5387 100644 --- a/python/ray/autoscaler/docker.py +++ b/python/ray/autoscaler/docker.py @@ -17,6 +17,7 @@ def dockerize_if_needed(config): return config docker_image = config["docker"].get("image") + docker_pull = config["docker"].get("pull_before_run", True) cname = config["docker"].get("container_name") run_options = config["docker"].get("run_options", []) @@ -37,6 +38,10 @@ def dockerize_if_needed(config): assert cname, "Must provide container name!" docker_mounts = {dst: dst for dst in config["file_mounts"]} + if docker_pull: + docker_pull_cmd = "docker pull {}".format(docker_image) + config["initialization_commands"].append(docker_pull_cmd) + head_docker_start = docker_start_cmds(ssh_user, head_docker_image, docker_mounts, cname, run_options + head_run_options) diff --git a/python/ray/autoscaler/gcp/example-full.yaml b/python/ray/autoscaler/gcp/example-full.yaml index a89804dba..8ee88bd11 100644 --- a/python/ray/autoscaler/gcp/example-full.yaml +++ b/python/ray/autoscaler/gcp/example-full.yaml @@ -25,6 +25,9 @@ autoscaling_mode: default docker: image: "" # e.g., tensorflow/tensorflow:1.5.0-py3 container_name: "" # e.g. ray_docker + # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image + # if no cached version is present. + pull_before_run: True run_options: [] # Extra options to pass into "docker run" diff --git a/python/ray/autoscaler/local/example-full.yaml b/python/ray/autoscaler/local/example-full.yaml index e63646675..f119af6c7 100644 --- a/python/ray/autoscaler/local/example-full.yaml +++ b/python/ray/autoscaler/local/example-full.yaml @@ -28,6 +28,9 @@ idle_timeout_minutes: 5 docker: image: "" # e.g., tensorflow/tensorflow:1.5.0-py3 container_name: "" # e.g. ray_docker + # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image + # if no cached version is present. + pull_before_run: True run_options: [] # Extra options to pass into "docker run" # Local specific configuration.