Extract out "fillout_defaults" from other config preparation actions (#8996)

python/ray/autoscaler/commands.py

@@ -16,7 +16,7 @@ except ImportError:  # py2
     from pipes import quote
 
 from ray.autoscaler.util import validate_config, hash_runtime_conf, \
-    hash_launch_conf, fillout_defaults
+    hash_launch_conf, prepare_config
 from ray.autoscaler.node_provider import get_node_provider, NODE_PROVIDERS
 from ray.autoscaler.tags import TAG_RAY_NODE_TYPE, TAG_RAY_LAUNCH_CONFIG, \
     TAG_RAY_NODE_NAME, NODE_TYPE_WORKER, NODE_TYPE_HEAD
@@ -44,7 +44,7 @@ def create_or_update_cluster(config_file, override_min_workers,
 
 
 def _bootstrap_config(config):
-    config = fillout_defaults(config)
+    config = prepare_config(config)
 
     hasher = hashlib.sha1()
     hasher.update(json.dumps([config], sort_keys=True).encode("utf-8"))
@@ -74,7 +74,7 @@ def teardown_cluster(config_file, yes, workers_only, override_cluster_name,
     config = yaml.safe_load(open(config_file).read())
     if override_cluster_name is not None:
         config["cluster_name"] = override_cluster_name
-    config = fillout_defaults(config)
+    config = prepare_config(config)
     validate_config(config)
 
     confirm("This will destroy your cluster", yes)

python/ray/autoscaler/util.py

@@ -54,11 +54,16 @@ def validate_config(config):
         raise jsonschema.ValidationError(message=e.message) from None
 
 
+def prepare_config(config):
+    with_defaults = fillout_defaults(config)
+    merge_setup_commands(with_defaults)
+    dockerize_if_needed(with_defaults)
+    return with_defaults
+
+
 def fillout_defaults(config):
     defaults = get_default_config(config["provider"])
     defaults.update(config)
-    merge_setup_commands(defaults)
-    dockerize_if_needed(defaults)
     defaults["auth"] = defaults.get("auth", {})
     return defaults
 

python/ray/tests/aws/utils/helpers.py

@@ -2,7 +2,7 @@ import os
 import yaml
 import ray
 
-from ray.autoscaler.commands import fillout_defaults, validate_config
+from ray.autoscaler.commands import prepare_config, validate_config
 from ray.tests.aws.utils.constants import DEFAULT_CLUSTER_NAME
 
 
@@ -17,7 +17,7 @@ def load_aws_example_config_file(file_name):
 
 
 def bootstrap_aws_config(config):
-    config = fillout_defaults(config)
+    config = prepare_config(config)
     validate_config(config)
     config["cluster_name"] = DEFAULT_CLUSTER_NAME
     return ray.autoscaler.aws.config.bootstrap_aws(config)

python/ray/tests/test_autoscaler.py

@@ -10,7 +10,7 @@ from jsonschema.exceptions import ValidationError
 
 import ray
 import ray.services as services
-from ray.autoscaler.util import fillout_defaults, validate_config
+from ray.autoscaler.util import prepare_config, validate_config
 from ray.autoscaler.load_metrics import LoadMetrics
 from ray.autoscaler.autoscaler import StandardAutoscaler
 from ray.autoscaler.tags import TAG_RAY_NODE_TYPE, TAG_RAY_NODE_STATUS, \
@@ -341,7 +341,7 @@ class AutoscalingTest(unittest.TestCase):
             "region": "us-east-1",
             "availability_zone": "us-east-1a",
         }
-        config = fillout_defaults(config)
+        config = prepare_config(config)
         try:
             validate_config(config)
         except ValidationError:
@@ -747,7 +747,7 @@ class AutoscalingTest(unittest.TestCase):
     def testReportsConfigFailures(self):
         config = copy.deepcopy(SMALL_CLUSTER)
         config["provider"]["type"] = "external"
-        config = fillout_defaults(config)
+        config = prepare_config(config)
         config["provider"]["type"] = "mock"
         config_path = self.write_config(config)
         self.provider = MockProvider()

python/ray/tests/test_autoscaler_yaml.py

@@ -5,7 +5,7 @@ import yaml
 import urllib
 import tempfile
 
-from ray.autoscaler.util import fillout_defaults, validate_config
+from ray.autoscaler.util import prepare_config, validate_config
 from ray.test_utils import recursive_fnmatch
 
 RAY_PATH = os.path.abspath(os.path.join(__file__, "../../"))
@@ -21,7 +21,7 @@ class AutoscalingConfigTest(unittest.TestCase):
         for config_path in CONFIG_PATHS:
             with open(config_path) as f:
                 config = yaml.safe_load(f)
-            config = fillout_defaults(config)
+            config = prepare_config(config)
             try:
                 validate_config(config)
             except Exception:
@@ -36,7 +36,7 @@ class AutoscalingConfigTest(unittest.TestCase):
             f.write(content)
             f.seek(0)
             config = yaml.safe_load(f)
-        config = fillout_defaults(config)
+        config = prepare_config(config)
         try:
             validate_config(config)
         except Exception: