[autoscaler] Ray Up url-arg (#8279)

python/ray/scripts/scripts.py

@@ -7,6 +7,8 @@ import os
 import subprocess
 import sys
 import time
+import urllib
+import urllib.parse
 
 import ray.services as services
 from ray.autoscaler.commands import (
@@ -566,6 +568,16 @@ def create_or_update(cluster_config_file, min_workers, max_workers, no_restart,
     if restart_only or no_restart:
         assert restart_only != no_restart, "Cannot set both 'restart_only' " \
             "and 'no_restart' at the same time!"
+    if urllib.parse.urlparse(cluster_config_file).scheme in ("http", "https"):
+        try:
+            response = urllib.request.urlopen(cluster_config_file, timeout=5)
+            content = response.read()
+            file_name = cluster_config_file.split("/")[-1]
+            with open(file_name, "wb") as f:
+                f.write(content)
+            cluster_config_file = file_name
+        except urllib.error.HTTPError as e:
+            logger.info("Error downloading file: ", e)
     create_or_update_cluster(cluster_config_file, min_workers, max_workers,
                              no_restart, restart_only, yes, cluster_name)
 

python/ray/tests/test_autoscaler_yaml.py

@@ -2,6 +2,8 @@ import jsonschema
 import os
 import unittest
 import yaml
+import urllib
+import tempfile
 
 from ray.autoscaler.autoscaler import fillout_defaults, validate_config
 from ray.test_utils import recursive_fnmatch
@@ -25,6 +27,21 @@ class AutoscalingConfigTest(unittest.TestCase):
             except Exception:
                 self.fail("Config did not pass validation test!")
 
+    def testValidateNetworkConfig(self):
+        web_yaml = "https://raw.githubusercontent.com/ray-project/ray/" \
+            "master/python/ray/autoscaler/aws/example-full.yaml"
+        response = urllib.request.urlopen(web_yaml, timeout=5)
+        content = response.read()
+        with tempfile.TemporaryFile() as f:
+            f.write(content)
+            f.seek(0)
+            config = yaml.safe_load(f)
+        config = fillout_defaults(config)
+        try:
+            validate_config(config)
+        except Exception:
+            self.fail("Config did not pass validation test!")
+
     def _test_invalid_config(self, config_path):
         with open(os.path.join(RAY_PATH, config_path)) as f:
             config = yaml.safe_load(f)