From 2b0e93586f9fab473af30f8f578286164ab57c4a Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Tue, 21 Jan 2020 13:54:04 -0800
Subject: [PATCH] [autoscaler] Auto-replace "DEFAULT" with most recent DLAMI
 (#6848)

* try_this

* fix

* actual fix

* default
---
 doc/source/autoscaling.rst          |  2 ++
 python/ray/autoscaler/aws/config.py | 47 +++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/doc/source/autoscaling.rst b/doc/source/autoscaling.rst
index d930c62e6..f566b2159 100644
--- a/doc/source/autoscaling.rst
+++ b/doc/source/autoscaling.rst
@@ -34,6 +34,8 @@ Test that it works by running the following commands from your local machine:
     # Tear down the cluster.
     $ ray down ray/python/ray/autoscaler/aws/example-full.yaml
 
+.. tip:: For the AWS node configuration, you can set ``"ImageId: DEFAULT"`` to automatically use the newest `Deep Learning AMI <https://aws.amazon.com/machine-learning/amis/>`_ for your region. For example, ``head_node: {InstanceType: c5.xlarge, ImageId: DEFAULT}``.
+
 GCP
 ~~~
 
diff --git a/python/ray/autoscaler/aws/config.py b/python/ray/autoscaler/aws/config.py
index 7df4c11a1..efadbdd8b 100644
--- a/python/ray/autoscaler/aws/config.py
+++ b/python/ray/autoscaler/aws/config.py
@@ -17,6 +17,22 @@ DEFAULT_RAY_INSTANCE_PROFILE = RAY + "-v1"
 DEFAULT_RAY_IAM_ROLE = RAY + "-v1"
 SECURITY_GROUP_TEMPLATE = RAY + "-{}"
 
+DEFAULT_AMI_NAME = "AWS Deep Learning AMI (Ubuntu 18.04) V26.0"
+
+# Obtained from https://aws.amazon.com/marketplace/pp/B07Y43P7X5 on 1/20/2020.
+DEFAULT_AMI = {
+    "us-east-2": "ami-0bb99846db2df6e38",  # US East (Ohio)
+    "us-east-1": "ami-0698bcaf8bd9ef56d",  # US East (N. Virginia)
+    "us-west-1": "ami-074c29e29c500f623",  # US West (N. California)
+    "us-west-2": "ami-010a96c958f9ee5cf",  # US West (Oregon)
+    "ca-central-1": "ami-086b864eabf2da9b1",  # Canada (Central)
+    "eu-central-1": "ami-0dcdcc4bc9e75005f",  # EU (Frankfurt)
+    "eu-west-1": "ami-071e6d171b20431fb",  # EU (Ireland)
+    "eu-west-2": "ami-0470e741c969b62fc",  # EU (London)
+    "eu-west-3": "ami-064f884d98b90a453",  # EU (Paris)
+    "sa-east-1": "ami-054e94fd9b444491d",  # SA (Sao Paulo)
+}
+
 assert StrictVersion(boto3.__version__) >= StrictVersion("1.4.8"), \
     "Boto3 version >= 1.4.8 required, try `pip install -U boto3`"
 
@@ -49,6 +65,9 @@ def bootstrap_aws(config):
     # the group, and also SSH access from outside.
     config = _configure_security_group(config)
 
+    # Provide a helpful message for missing AMI.
+    _check_ami(config)
+
     return config
 
 
@@ -250,6 +269,34 @@ def _configure_security_group(config):
     return config
 
 
+def _check_ami(config):
+    """Provide helpful message for missing ImageId for node configuration."""
+
+    region = config["provider"]["region"]
+    default_ami = DEFAULT_AMI.get(region)
+    if not default_ami:
+        # If we do not provide a default AMI for the given region, noop.
+        return
+
+    if config["head_node"].get("ImageId", "").lower() == "DEFAULT":
+        config["head_node"]["ImageId"] = default_ami
+        logger.info("_check_ami: head node ImageId specified as 'DEFAULT'. "
+                    "Using '{ami_id}', which is the default {ami_name} "
+                    "for your region ({region}).".format(
+                        ami_id=default_ami,
+                        ami_name=DEFAULT_AMI_NAME,
+                        region=region))
+
+    if config["worker_nodes"].get("ImageId", "").lower() == "DEFAULT":
+        config["worker_nodes"]["ImageId"] = default_ami
+        logger.info("_check_ami: worker nodes ImageId specified as 'DEFAULT'. "
+                    "Using '{ami_id}', which is the default {ami_name} "
+                    "for your region ({region}).".format(
+                        ami_id=default_ami,
+                        ami_name=DEFAULT_AMI_NAME,
+                        region=region))
+
+
 def _get_vpc_id_or_die(config, subnet_id):
     ec2 = _resource("ec2", config)
     subnet = list(