From c644650818d47959b8fb483b2159d692a55cac8e Mon Sep 17 00:00:00 2001 From: Ian Rodney Date: Tue, 1 Sep 2020 11:25:37 -0700 Subject: [PATCH] [docker] Run docker stop in teardown_cluster (#10407) --- python/ray/autoscaler/commands.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python/ray/autoscaler/commands.py b/python/ray/autoscaler/commands.py index dae20fa3e..dd7a00599 100644 --- a/python/ray/autoscaler/commands.py +++ b/python/ray/autoscaler/commands.py @@ -344,9 +344,41 @@ def teardown_cluster(config_file: str, yes: bool, workers_only: bool, return head + workers + def run_docker_stop(node, container_name): + try: + updater = NodeUpdaterThread( + node_id=node, + provider_config=config["provider"], + provider=provider, + auth_config=config["auth"], + cluster_name=config["cluster_name"], + file_mounts=config["file_mounts"], + initialization_commands=[], + setup_commands=[], + ray_start_commands=[], + runtime_hash="", + file_mounts_contents_hash="", + is_head_node=False, + docker_config=config.get("docker")) + _exec( + updater, + f"docker stop {container_name}", + False, + False, + run_env="host") + except Exception: + cli_logger.warning(f"Docker stop failed on {node}") + cli_logger.old_warning(logger, f"Docker stop failed on {node}") + # Loop here to check that both the head and worker nodes are actually # really gone A = remaining_nodes() + + container_name = config.get("docker", {}).get("container_name") + if container_name: + for node in A: + run_docker_stop(node, container_name) + with LogTimer("teardown_cluster: done."): while A: cli_logger.old_info(