Minimal version of piping autoscaler events to driver logs (#13434)

python/ray/autoscaler/_private/autoscaler.py

@@ -4,6 +4,7 @@ from urllib3.exceptions import MaxRetryError
 import copy
 import logging
 import math
+import operator
 import os
 import subprocess
 import threading
@@ -19,6 +20,7 @@ from ray.autoscaler.tags import (
     TAG_RAY_USER_NODE_TYPE, STATUS_UNINITIALIZED, STATUS_WAITING_FOR_SSH,
     STATUS_SYNCING_FILES, STATUS_SETTING_UP, STATUS_UP_TO_DATE,
     NODE_KIND_WORKER, NODE_KIND_UNMANAGED, NODE_KIND_HEAD)
+from ray.autoscaler._private.event_summarizer import EventSummarizer
 from ray.autoscaler._private.legacy_info_string import legacy_log_info_string
 from ray.autoscaler._private.providers import _get_node_provider
 from ray.autoscaler._private.updater import NodeUpdaterThread
@@ -73,7 +75,8 @@ class StandardAutoscaler:
                  max_failures=AUTOSCALER_MAX_NUM_FAILURES,
                  process_runner=subprocess,
                  update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S,
-                 prefix_cluster_info=False):
+                 prefix_cluster_info=False,
+                 event_summarizer=None):
         self.config_path = config_path
         # Prefix each line of info string with cluster name if True
         self.prefix_cluster_info = prefix_cluster_info
@@ -89,6 +92,7 @@ class StandardAutoscaler:
         self.max_launch_batch = max_launch_batch
         self.max_concurrent_launches = max_concurrent_launches
         self.process_runner = process_runner
+        self.event_summarizer = event_summarizer or EventSummarizer()
 
         # Map from node_id to NodeUpdater processes
         self.updaters = {}
@@ -193,10 +197,20 @@ class StandardAutoscaler:
             if node_ip in last_used and last_used[node_ip] < horizon:
                 logger.info("StandardAutoscaler: "
                             "{}: Terminating idle node.".format(node_id))
+                self.event_summarizer.add(
+                    "Removing {} nodes of type " + self._get_node_type(node_id)
+                    + " (idle).",
+                    quantity=1,
+                    aggregate=operator.add)
                 nodes_to_terminate.append(node_id)
             elif not self.launch_config_ok(node_id):
                 logger.info("StandardAutoscaler: "
                             "{}: Terminating outdated node.".format(node_id))
+                self.event_summarizer.add(
+                    "Removing {} nodes of type " + self._get_node_type(node_id)
+                    + " (outdated).",
+                    quantity=1,
+                    aggregate=operator.add)
                 nodes_to_terminate.append(node_id)
 
         if nodes_to_terminate:
@@ -210,6 +224,11 @@ class StandardAutoscaler:
             to_terminate = nodes.pop()
             logger.info("StandardAutoscaler: "
                         "{}: Terminating unneeded node.".format(to_terminate))
+            self.event_summarizer.add(
+                "Removing {} nodes of type " +
+                self._get_node_type(to_terminate) + " (max workers).",
+                quantity=1,
+                aggregate=operator.add)
             nodes_to_terminate.append(to_terminate)
 
         if nodes_to_terminate:
@@ -246,6 +265,11 @@ class StandardAutoscaler:
                 else:
                     logger.error(f"StandardAutoscaler: {node_id}: Terminating "
                                  "failed to setup/initialize node.")
+                    self.event_summarizer.add(
+                        "Removing {} nodes of type " +
+                        self._get_node_type(node_id) + " (launch failed).",
+                        quantity=1,
+                        aggregate=operator.add)
                     nodes_to_terminate.append(node_id)
                     self.num_failed_updates[node_id] += 1
                 del self.updaters[node_id]
@@ -544,6 +568,11 @@ class StandardAutoscaler:
         logger.warning("StandardAutoscaler: "
                        "{}: No recent heartbeat, "
                        "restarting Ray to recover...".format(node_id))
+        self.event_summarizer.add(
+            "Restarting {} nodes of type " + self._get_node_type(node_id) +
+            " (lost contact with raylet).",
+            quantity=1,
+            aggregate=operator.add)
         updater = NodeUpdaterThread(
             node_id=node_id,
             provider_config=self.config["provider"],
@@ -565,6 +594,13 @@ class StandardAutoscaler:
         updater.start()
         self.updaters[node_id] = updater
 
+    def _get_node_type(self, node_id: str) -> str:
+        node_tags = self.provider.node_tags(node_id)
+        if TAG_RAY_USER_NODE_TYPE in node_tags:
+            return node_tags[TAG_RAY_USER_NODE_TYPE]
+        else:
+            return "unknown"
+
     def _get_node_type_specific_fields(self, node_id: str,
                                        fields_key: str) -> Any:
         fields = self.config[fields_key]
@@ -661,6 +697,10 @@ class StandardAutoscaler:
     def launch_new_node(self, count: int, node_type: Optional[str]) -> None:
         logger.info(
             "StandardAutoscaler: Queue {} new nodes for launch".format(count))
+        self.event_summarizer.add(
+            "Adding {} nodes of type " + str(node_type) + ".",
+            quantity=count,
+            aggregate=operator.add)
         self.pending_launches.inc(node_type, count)
         config = copy.deepcopy(self.config)
         # Split into individual launch requests of the max batch size.

python/ray/autoscaler/_private/commands.py

@@ -131,8 +131,10 @@ def request_resources(num_cpus: Optional[int] = None,
         to_request += [{"CPU": 1}] * num_cpus
     if bundles:
         to_request += bundles
-    _internal_kv_put(AUTOSCALER_RESOURCE_REQUEST_CHANNEL,
-                     json.dumps(to_request))
+    _internal_kv_put(
+        AUTOSCALER_RESOURCE_REQUEST_CHANNEL,
+        json.dumps(to_request),
+        overwrite=True)
 
 
 def create_or_update_cluster(config_file: str,

python/ray/autoscaler/_private/constants.py

@@ -12,6 +12,9 @@ def env_integer(key, default):
     return default
 
 
+# Whether event logging to driver is enabled. Set to 0 to disable.
+AUTOSCALER_EVENTS = env_integer("AUTOSCALER_EVENTS", 1)
+
 # How long to wait for a node to start, in seconds
 NODE_START_WAIT_S = env_integer("AUTOSCALER_NODE_START_WAIT_S", 900)
 

python/ray/autoscaler/_private/event_summarizer.py

@@ -0,0 +1,39 @@
+from typing import Any, Callable, Dict, List
+
+
+class EventSummarizer:
+    """Utility that aggregates related log messages to reduce log spam."""
+
+    def __init__(self):
+        self.events_by_key: Dict[str, int] = {}
+
+    def add(self, template: str, *, quantity: Any,
+            aggregate: Callable[[Any, Any], Any]) -> None:
+        """Add a log message, which will be combined by template.
+
+        Args:
+            template (str): Format string with one placeholder for quantity.
+            quantity (Any): Quantity to aggregate.
+            aggregate (func): Aggregation function used to combine the
+                quantities. The result is inserted into the template to
+                produce the final log message.
+        """
+        # Enforce proper sentence structure.
+        if not template.endswith("."):
+            template += "."
+        if template in self.events_by_key:
+            self.events_by_key[template] = aggregate(
+                self.events_by_key[template], quantity)
+        else:
+            self.events_by_key[template] = quantity
+
+    def summary(self) -> List[str]:
+        """Generate the aggregated log summary of all added events."""
+        out = []
+        for template, quantity in self.events_by_key.items():
+            out.append(template.format(quantity))
+        return out
+
+    def clear(self) -> None:
+        """Clear the events added."""
+        self.events_by_key.clear()

python/ray/autoscaler/_private/load_metrics.py

@@ -190,6 +190,19 @@ class LoadMetrics:
     def get_pending_placement_groups(self):
         return self.pending_placement_groups
 
+    def resources_avail_summary(self) -> str:
+        """Return a concise string of cluster size to report to event logs.
+
+        For example, "3 CPUs, 4 GPUs".
+        """
+        total_resources = reduce(add_resources,
+                                 self.static_resources_by_ip.values()
+                                 ) if self.static_resources_by_ip else {}
+        out = "{} CPUs".format(int(total_resources.get("CPU", 0)))
+        if "GPU" in total_resources:
+            out += ", {} GPUs".format(int(total_resources["GPU"]))
+        return out
+
     def summary(self):
         available_resources = reduce(add_resources,
                                      self.dynamic_resources_by_ip.values()

python/ray/autoscaler/_private/util.py

@@ -311,9 +311,11 @@ def format_pg(pg):
     return f"{bundles_str} ({strategy})"
 
 
-def get_usage_report(lm_summary):
+def get_usage_report(lm_summary) -> str:
     usage_lines = []
     for resource, (used, total) in lm_summary.usage.items():
+        if "node:" in resource:
+            continue  # Skip the auto-added per-node "node:<ip>" resource.
         line = f" {used}/{total} {resource}"
         if resource in ["memory", "object_store_memory"]:
             to_GiB = ray.ray_constants.MEMORY_RESOURCE_UNIT_BYTES / 2**30