From 42c9fa19d1bc02c82070be7c6f40ed79dfaa8408 Mon Sep 17 00:00:00 2001
From: Joseph Lucas <JosephTLucas@users.noreply.github.com>
Date: Sun, 17 May 2020 15:18:00 -0400
Subject: [PATCH] [autoscaler] Ray Up url-arg (#8279)

---
 python/ray/scripts/scripts.py            | 12 ++++++++++++
 python/ray/tests/test_autoscaler_yaml.py | 17 +++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py
index e3b52e4b3..c19ece32f 100644
--- a/python/ray/scripts/scripts.py
+++ b/python/ray/scripts/scripts.py
@@ -7,6 +7,8 @@ import os
 import subprocess
 import sys
 import time
+import urllib
+import urllib.parse
 
 import ray.services as services
 from ray.autoscaler.commands import (
@@ -566,6 +568,16 @@ def create_or_update(cluster_config_file, min_workers, max_workers, no_restart,
     if restart_only or no_restart:
         assert restart_only != no_restart, "Cannot set both 'restart_only' " \
             "and 'no_restart' at the same time!"
+    if urllib.parse.urlparse(cluster_config_file).scheme in ("http", "https"):
+        try:
+            response = urllib.request.urlopen(cluster_config_file, timeout=5)
+            content = response.read()
+            file_name = cluster_config_file.split("/")[-1]
+            with open(file_name, "wb") as f:
+                f.write(content)
+            cluster_config_file = file_name
+        except urllib.error.HTTPError as e:
+            logger.info("Error downloading file: ", e)
     create_or_update_cluster(cluster_config_file, min_workers, max_workers,
                              no_restart, restart_only, yes, cluster_name)
 
diff --git a/python/ray/tests/test_autoscaler_yaml.py b/python/ray/tests/test_autoscaler_yaml.py
index b25bb9dc9..b83e76014 100644
--- a/python/ray/tests/test_autoscaler_yaml.py
+++ b/python/ray/tests/test_autoscaler_yaml.py
@@ -2,6 +2,8 @@ import jsonschema
 import os
 import unittest
 import yaml
+import urllib
+import tempfile
 
 from ray.autoscaler.autoscaler import fillout_defaults, validate_config
 from ray.test_utils import recursive_fnmatch
@@ -25,6 +27,21 @@ class AutoscalingConfigTest(unittest.TestCase):
             except Exception:
                 self.fail("Config did not pass validation test!")
 
+    def testValidateNetworkConfig(self):
+        web_yaml = "https://raw.githubusercontent.com/ray-project/ray/" \
+            "master/python/ray/autoscaler/aws/example-full.yaml"
+        response = urllib.request.urlopen(web_yaml, timeout=5)
+        content = response.read()
+        with tempfile.TemporaryFile() as f:
+            f.write(content)
+            f.seek(0)
+            config = yaml.safe_load(f)
+        config = fillout_defaults(config)
+        try:
+            validate_config(config)
+        except Exception:
+            self.fail("Config did not pass validation test!")
+
     def _test_invalid_config(self, config_path):
         with open(os.path.join(RAY_PATH, config_path)) as f:
             config = yaml.safe_load(f)