From 8f9b46596835ff73adb5590353b795c362c0f754 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Wed, 17 Jun 2020 16:40:37 -0700 Subject: [PATCH] Extract out "fillout_defaults" from other config preparation actions (#8996) --- python/ray/autoscaler/commands.py | 6 +++--- python/ray/autoscaler/util.py | 9 +++++++-- python/ray/tests/aws/utils/helpers.py | 4 ++-- python/ray/tests/test_autoscaler.py | 6 +++--- python/ray/tests/test_autoscaler_yaml.py | 6 +++--- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/python/ray/autoscaler/commands.py b/python/ray/autoscaler/commands.py index e8260545d..00968be57 100644 --- a/python/ray/autoscaler/commands.py +++ b/python/ray/autoscaler/commands.py @@ -16,7 +16,7 @@ except ImportError: # py2 from pipes import quote from ray.autoscaler.util import validate_config, hash_runtime_conf, \ - hash_launch_conf, fillout_defaults + hash_launch_conf, prepare_config from ray.autoscaler.node_provider import get_node_provider, NODE_PROVIDERS from ray.autoscaler.tags import TAG_RAY_NODE_TYPE, TAG_RAY_LAUNCH_CONFIG, \ TAG_RAY_NODE_NAME, NODE_TYPE_WORKER, NODE_TYPE_HEAD @@ -44,7 +44,7 @@ def create_or_update_cluster(config_file, override_min_workers, def _bootstrap_config(config): - config = fillout_defaults(config) + config = prepare_config(config) hasher = hashlib.sha1() hasher.update(json.dumps([config], sort_keys=True).encode("utf-8")) @@ -74,7 +74,7 @@ def teardown_cluster(config_file, yes, workers_only, override_cluster_name, config = yaml.safe_load(open(config_file).read()) if override_cluster_name is not None: config["cluster_name"] = override_cluster_name - config = fillout_defaults(config) + config = prepare_config(config) validate_config(config) confirm("This will destroy your cluster", yes) diff --git a/python/ray/autoscaler/util.py b/python/ray/autoscaler/util.py index 7c56cca31..1febddbb1 100644 --- a/python/ray/autoscaler/util.py +++ b/python/ray/autoscaler/util.py @@ -54,11 +54,16 @@ def validate_config(config): raise jsonschema.ValidationError(message=e.message) from None +def prepare_config(config): + with_defaults = fillout_defaults(config) + merge_setup_commands(with_defaults) + dockerize_if_needed(with_defaults) + return with_defaults + + def fillout_defaults(config): defaults = get_default_config(config["provider"]) defaults.update(config) - merge_setup_commands(defaults) - dockerize_if_needed(defaults) defaults["auth"] = defaults.get("auth", {}) return defaults diff --git a/python/ray/tests/aws/utils/helpers.py b/python/ray/tests/aws/utils/helpers.py index dbbfb0a23..f1f5f382f 100644 --- a/python/ray/tests/aws/utils/helpers.py +++ b/python/ray/tests/aws/utils/helpers.py @@ -2,7 +2,7 @@ import os import yaml import ray -from ray.autoscaler.commands import fillout_defaults, validate_config +from ray.autoscaler.commands import prepare_config, validate_config from ray.tests.aws.utils.constants import DEFAULT_CLUSTER_NAME @@ -17,7 +17,7 @@ def load_aws_example_config_file(file_name): def bootstrap_aws_config(config): - config = fillout_defaults(config) + config = prepare_config(config) validate_config(config) config["cluster_name"] = DEFAULT_CLUSTER_NAME return ray.autoscaler.aws.config.bootstrap_aws(config) diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index d5b332881..2d5cd76e7 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -10,7 +10,7 @@ from jsonschema.exceptions import ValidationError import ray import ray.services as services -from ray.autoscaler.util import fillout_defaults, validate_config +from ray.autoscaler.util import prepare_config, validate_config from ray.autoscaler.load_metrics import LoadMetrics from ray.autoscaler.autoscaler import StandardAutoscaler from ray.autoscaler.tags import TAG_RAY_NODE_TYPE, TAG_RAY_NODE_STATUS, \ @@ -341,7 +341,7 @@ class AutoscalingTest(unittest.TestCase): "region": "us-east-1", "availability_zone": "us-east-1a", } - config = fillout_defaults(config) + config = prepare_config(config) try: validate_config(config) except ValidationError: @@ -747,7 +747,7 @@ class AutoscalingTest(unittest.TestCase): def testReportsConfigFailures(self): config = copy.deepcopy(SMALL_CLUSTER) config["provider"]["type"] = "external" - config = fillout_defaults(config) + config = prepare_config(config) config["provider"]["type"] = "mock" config_path = self.write_config(config) self.provider = MockProvider() diff --git a/python/ray/tests/test_autoscaler_yaml.py b/python/ray/tests/test_autoscaler_yaml.py index 4e8802c3c..da17e5d2e 100644 --- a/python/ray/tests/test_autoscaler_yaml.py +++ b/python/ray/tests/test_autoscaler_yaml.py @@ -5,7 +5,7 @@ import yaml import urllib import tempfile -from ray.autoscaler.util import fillout_defaults, validate_config +from ray.autoscaler.util import prepare_config, validate_config from ray.test_utils import recursive_fnmatch RAY_PATH = os.path.abspath(os.path.join(__file__, "../../")) @@ -21,7 +21,7 @@ class AutoscalingConfigTest(unittest.TestCase): for config_path in CONFIG_PATHS: with open(config_path) as f: config = yaml.safe_load(f) - config = fillout_defaults(config) + config = prepare_config(config) try: validate_config(config) except Exception: @@ -36,7 +36,7 @@ class AutoscalingConfigTest(unittest.TestCase): f.write(content) f.seek(0) config = yaml.safe_load(f) - config = fillout_defaults(config) + config = prepare_config(config) try: validate_config(config) except Exception: