wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-04 11:54:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

[autoscaler] Split autoscaler interface public private (#10898)

Browse Source
This commit is contained in:
Eric Liang
2020-09-18 18:16:23 -07:00
committed by GitHub
parent 9a07c7b963
commit 6a227ae501
56 changed files with 436 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tests/aws/conftest.py
+1 -1
View File
@@ -1,6 +1,6 @@
import pytest
from ray.autoscaler.aws.config import _resource_cache
from ray.autoscaler._private.aws.config import _resource_cache
from botocore.stub import Stubber
python/ray/tests/aws/utils/constants.py
+10 -9
View File
@@ -4,14 +4,14 @@ from datetime import datetime
# Override global constants used in AWS autoscaler config artifact names.
# This helps ensure that any unmocked test doesn't alter non-test artifacts.
ray.autoscaler.aws.config.RAY = \
ray.autoscaler._private.aws.config.RAY = \
"ray-autoscaler-aws-test"
ray.autoscaler.aws.config.DEFAULT_RAY_INSTANCE_PROFILE = \
ray.autoscaler.aws.config.RAY + "-v1"
ray.autoscaler.aws.config.DEFAULT_RAY_IAM_ROLE = \
ray.autoscaler.aws.config.RAY + "-v1"
ray.autoscaler.aws.config.SECURITY_GROUP_TEMPLATE = \
ray.autoscaler.aws.config.RAY + "-{}"
ray.autoscaler._private.aws.config.DEFAULT_RAY_INSTANCE_PROFILE = \
ray.autoscaler._private.aws.config.RAY + "-v1"
ray.autoscaler._private.aws.config.DEFAULT_RAY_IAM_ROLE = \
ray.autoscaler._private.aws.config.RAY + "-v1"
ray.autoscaler._private.aws.config.SECURITY_GROUP_TEMPLATE = \
ray.autoscaler._private.aws.config.RAY + "-{}"
# Default IAM instance profile to expose to tests.
DEFAULT_INSTANCE_PROFILE = {
@@ -35,7 +35,7 @@ DEFAULT_INSTANCE_PROFILE = {
# Default EC2 key pair to expose to tests.
DEFAULT_KEY_PAIR = {
"KeyFingerprint": "00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00",
"KeyName": ray.autoscaler.aws.config.RAY + "_us-west-2",
"KeyName": ray.autoscaler._private.aws.config.RAY + "_us-west-2",
}
# Primary EC2 subnet to expose to tests.
@@ -69,7 +69,8 @@ DEFAULT_CLUSTER_NAME = "test-cluster-name"
# (prior to inbound rule configuration).
DEFAULT_SG = {
"Description": "Auto-created security group for Ray workers",
"GroupName": ray.autoscaler.aws.config.RAY + "-" + DEFAULT_CLUSTER_NAME,
"GroupName": ray.autoscaler._private.aws.config.RAY + "-" +
DEFAULT_CLUSTER_NAME,
"OwnerId": "test-owner",
"GroupId": "sg-1234abcd",
"VpcId": DEFAULT_SUBNET["VpcId"],
python/ray/tests/aws/utils/helpers.py
+3 -2
View File
@@ -2,11 +2,12 @@ import os
import yaml
import ray
from ray.autoscaler.commands import prepare_config, validate_config
from ray.autoscaler._private.commands import prepare_config, validate_config
from ray.tests.aws.utils.constants import DEFAULT_CLUSTER_NAME
def get_aws_example_config_file_path(file_name):
import ray.autoscaler.aws
return os.path.join(
os.path.dirname(ray.autoscaler.aws.__file__), file_name)
@@ -20,7 +21,7 @@ def bootstrap_aws_config(config):
config = prepare_config(config)
validate_config(config)
config["cluster_name"] = DEFAULT_CLUSTER_NAME
return ray.autoscaler.aws.config.bootstrap_aws(config)
return ray.autoscaler._private.aws.config.bootstrap_aws(config)
def bootstrap_aws_example_config_file(file_name):
python/ray/tests/aws/utils/mocks.py
+1 -1
View File
@@ -1,4 +1,4 @@
from ray.autoscaler.aws.config import key_pair
from ray.autoscaler._private.aws.config import key_pair
from ray.tests.aws.utils.constants import DEFAULT_KEY_PAIR
python/ray/tests/aws/utils/stubs.py
+1 -1
View File
@@ -12,7 +12,7 @@ def configure_iam_role_default(iam_client_stub):
iam_client_stub.add_response(
"get_instance_profile",
expected_params={
"InstanceProfileName": ray.autoscaler.aws.config.
"InstanceProfileName": ray.autoscaler._private.aws.config.
DEFAULT_RAY_INSTANCE_PROFILE
},
service_response={"InstanceProfile": DEFAULT_INSTANCE_PROFILE})
python/ray/tests/test_autoscaler.py
+9 -9
View File
@@ -10,14 +10,14 @@ from jsonschema.exceptions import ValidationError
import ray
import ray.services as services
from ray.autoscaler.util import prepare_config, validate_config
from ray.autoscaler.commands import get_or_create_head_node
from ray.autoscaler.docker import DOCKER_MOUNT_PREFIX
from ray.autoscaler.load_metrics import LoadMetrics
from ray.autoscaler.autoscaler import StandardAutoscaler
from ray.autoscaler._private.util import prepare_config, validate_config
from ray.autoscaler._private.commands import get_or_create_head_node
from ray.autoscaler._private.docker import DOCKER_MOUNT_PREFIX
from ray.autoscaler._private.load_metrics import LoadMetrics
from ray.autoscaler._private.autoscaler import StandardAutoscaler
from ray.autoscaler.tags import TAG_RAY_NODE_KIND, TAG_RAY_NODE_STATUS, \
STATUS_UP_TO_DATE, STATUS_UPDATE_FAILED, TAG_RAY_USER_NODE_TYPE
from ray.autoscaler.node_provider import NODE_PROVIDERS, NodeProvider
from ray.autoscaler.node_provider import _NODE_PROVIDERS, NodeProvider
from ray.test_utils import RayTestTimeoutException
import pytest
@@ -319,14 +319,14 @@ class LoadMetricsTest(unittest.TestCase):
class AutoscalingTest(unittest.TestCase):
def setUp(self):
NODE_PROVIDERS["mock"] = \
_NODE_PROVIDERS["mock"] = \
lambda config: self.create_provider
self.provider = None
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
self.provider = None
del NODE_PROVIDERS["mock"]
del _NODE_PROVIDERS["mock"]
shutil.rmtree(self.tmpdir)
ray.shutdown()
@@ -1327,7 +1327,7 @@ class AutoscalingTest(unittest.TestCase):
f"{DOCKER_MOUNT_PREFIX}/home/test-folder/")
# Simulate a second `ray up` call
from ray.autoscaler import util
from ray.autoscaler._private import util
util._hash_cache = {}
runner = MockProcessRunner()
lm = LoadMetrics()
python/ray/tests/test_autoscaler_yaml.py
+1 -1
View File
@@ -6,7 +6,7 @@ import unittest
import urllib
import yaml
from ray.autoscaler.util import prepare_config, validate_config
from ray.autoscaler._private.util import prepare_config, validate_config
from ray.test_utils import recursive_fnmatch
RAY_PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
python/ray/tests/test_cli.py
+1 -1
View File
@@ -34,7 +34,7 @@ from click.testing import CliRunner
from testfixtures import Replacer
from testfixtures.popen import MockPopen, PopenBehaviour
import ray.autoscaler.aws.config as aws_config
import ray.autoscaler._private.aws.config as aws_config
import ray.scripts.scripts as scripts
python/ray/tests/test_command_runner.py
+2 -2
View File
@@ -1,9 +1,9 @@
import pytest
from ray.tests.test_autoscaler import MockProvider, MockProcessRunner
from ray.autoscaler.command_runner import CommandRunnerInterface, \
from ray.autoscaler._private.command_runner import CommandRunnerInterface, \
SSHCommandRunner, _with_environment_variables, DockerCommandRunner, \
KubernetesCommandRunner
from ray.autoscaler.docker import DOCKER_MOUNT_PREFIX
from ray.autoscaler._private.docker import DOCKER_MOUNT_PREFIX
from getpass import getuser
import hashlib
python/ray/tests/test_coordinator_server.py
+17 -17
View File
@@ -3,10 +3,10 @@ import unittest
import socket
import json
from ray.autoscaler.node_provider import _NODE_PROVIDERS, _get_node_provider
from ray.autoscaler.local.coordinator_server import OnPremCoordinatorServer
from ray.autoscaler.node_provider import NODE_PROVIDERS, get_node_provider
from ray.autoscaler.local.node_provider import LocalNodeProvider
from ray.autoscaler.local.coordinator_node_provider import (
from ray.autoscaler._private.local.node_provider import LocalNodeProvider
from ray.autoscaler._private.local.coordinator_node_provider import (
CoordinatorSenderNodeProvider)
from ray.autoscaler.tags import (TAG_RAY_NODE_KIND, TAG_RAY_CLUSTER_NAME,
TAG_RAY_NODE_NAME, NODE_KIND_WORKER,
@@ -35,10 +35,10 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
"""Check correct import when coordinator_address is in config yaml."""
provider_config = {"coordinator_address": "fake_address:1234"}
coordinator_node_provider = NODE_PROVIDERS.get("local")(
coordinator_node_provider = _NODE_PROVIDERS.get("local")(
provider_config)
assert coordinator_node_provider is CoordinatorSenderNodeProvider
local_node_provider = NODE_PROVIDERS.get("local")({})
local_node_provider = _NODE_PROVIDERS.get("local")({})
assert local_node_provider is LocalNodeProvider
def testClusterStateInit(self):
@@ -59,8 +59,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
},
}
provider_config = cluster_config["provider"]
node_provider = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider = _get_node_provider(provider_config,
cluster_config["cluster_name"])
assert isinstance(node_provider, LocalNodeProvider)
expected_workers = {}
expected_workers[provider_config["head_ip"]] = {
@@ -85,8 +85,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
# Test removing workers updates the cluster state.
del expected_workers[provider_config["worker_ips"][0]]
removed_ip = provider_config["worker_ips"].pop()
node_provider = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider = _get_node_provider(provider_config,
cluster_config["cluster_name"])
workers = json.loads(open(state_save_path).read())
assert workers == expected_workers
@@ -98,8 +98,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
"state": "terminated",
}
provider_config["worker_ips"].append(removed_ip)
node_provider = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider = _get_node_provider(provider_config,
cluster_config["cluster_name"])
workers = json.loads(open(state_save_path).read())
assert workers == expected_workers
@@ -162,8 +162,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
"worker_nodes": {},
}
provider_config = cluster_config["provider"]
node_provider_1 = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider_1 = _get_node_provider(provider_config,
cluster_config["cluster_name"])
assert isinstance(node_provider_1, CoordinatorSenderNodeProvider)
assert not node_provider_1.non_terminated_nodes({})
@@ -189,8 +189,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
# Add another cluster.
cluster_config["cluster_name"] = "random_name_2"
provider_config = cluster_config["provider"]
node_provider_2 = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider_2 = _get_node_provider(provider_config,
cluster_config["cluster_name"])
assert not node_provider_2.non_terminated_nodes({})
assert not node_provider_2.is_running(self.list_of_node_ips[1])
assert node_provider_2.is_terminated(self.list_of_node_ips[1])
@@ -211,8 +211,8 @@ class OnPremCoordinatorServerTest(unittest.TestCase):
# Add another cluster (should fail because we only have two nodes).
cluster_config["cluster_name"] = "random_name_3"
provider_config = cluster_config["provider"]
node_provider_3 = get_node_provider(provider_config,
cluster_config["cluster_name"])
node_provider_3 = _get_node_provider(provider_config,
cluster_config["cluster_name"])
assert not node_provider_3.non_terminated_nodes(head_node_tags)
head_node_tags[TAG_RAY_NODE_NAME] = "ray-{}-head".format(
cluster_config["cluster_name"])
python/ray/tests/test_resource_demand_scheduler.py
+9 -8
View File
@@ -8,13 +8,14 @@ import unittest
import ray
from ray.tests.test_autoscaler import SMALL_CLUSTER, MockProvider, \
MockProcessRunner
from ray.autoscaler.autoscaler import StandardAutoscaler
from ray.autoscaler.load_metrics import LoadMetrics
from ray.autoscaler.node_provider import NODE_PROVIDERS
from ray.autoscaler.commands import get_or_create_head_node
from ray.autoscaler.tags import TAG_RAY_USER_NODE_TYPE, TAG_RAY_NODE_KIND
from ray.autoscaler.resource_demand_scheduler import _utilization_score, \
from ray.autoscaler.node_provider import _NODE_PROVIDERS
from ray.autoscaler._private.autoscaler import StandardAutoscaler
from ray.autoscaler._private.load_metrics import LoadMetrics
from ray.autoscaler._private.commands import get_or_create_head_node
from ray.autoscaler._private.resource_demand_scheduler import \
_utilization_score, \
get_bin_pack_residual, get_nodes_for, ResourceDemandScheduler
from ray.autoscaler.tags import TAG_RAY_USER_NODE_TYPE, TAG_RAY_NODE_KIND
from ray.test_utils import same_elements
from time import sleep
@@ -221,14 +222,14 @@ class LoadMetricsTest(unittest.TestCase):
class AutoscalingTest(unittest.TestCase):
def setUp(self):
NODE_PROVIDERS["mock"] = \
_NODE_PROVIDERS["mock"] = \
lambda config: self.create_provider
self.provider = None
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
self.provider = None
del NODE_PROVIDERS["mock"]
del _NODE_PROVIDERS["mock"]
shutil.rmtree(self.tmpdir)
ray.shutdown()