[Core] Error info pubsub (Remove ray.errors API) (#9665)

python/ray/test_utils.py

@@ -208,17 +208,6 @@ def run_string_as_driver_nonblocking(driver_script):
     return proc
 
 
-def flat_errors():
-    errors = []
-    for job_errors in ray.errors(all_jobs=True).values():
-        errors.extend(job_errors)
-    return errors
-
-
-def relevant_errors(error_type):
-    return [error for error in flat_errors() if error["type"] == error_type]
-
-
 def wait_for_num_actors(num_actors, timeout=10):
     start_time = time.time()
     while time.time() - start_time < timeout:
@@ -228,16 +217,6 @@ def wait_for_num_actors(num_actors, timeout=10):
     raise RayTestTimeoutException("Timed out while waiting for global state.")
 
 
-def wait_for_errors(error_type, num_errors, timeout=20):
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        if len(relevant_errors(error_type)) >= num_errors:
-            return
-        time.sleep(0.1)
-    raise RayTestTimeoutException("Timed out waiting for {} {} errors.".format(
-        num_errors, error_type))
-
-
 def wait_for_condition(condition_predictor, timeout=30, retry_interval_ms=100):
     """Wait until a condition is met or time out with an exception.
 
@@ -404,3 +383,31 @@ def get_other_nodes(cluster, exclude_head=False):
 def get_non_head_nodes(cluster):
     """Get all non-head nodes."""
     return list(filter(lambda x: x.head is False, cluster.list_all_nodes()))
+
+
+def init_error_pubsub():
+    """Initialize redis error info pub/sub"""
+    p = ray.worker.global_worker.redis_client.pubsub(
+        ignore_subscribe_messages=True)
+    error_pubsub_channel = ray.gcs_utils.RAY_ERROR_PUBSUB_PATTERN
+    p.psubscribe(error_pubsub_channel)
+    return p
+
+
+def get_error_message(pub_sub, num, error_type=None, timeout=10):
+    """Get errors through pub/sub."""
+    start_time = time.time()
+    msgs = []
+    while time.time() - start_time < timeout and len(msgs) < num:
+        msg = pub_sub.get_message()
+        if msg is None:
+            time.sleep(0.01)
+            continue
+        pubsub_msg = ray.gcs_utils.PubSubMessage.FromString(msg["data"])
+        error_data = ray.gcs_utils.ErrorTableData.FromString(pubsub_msg.data)
+        if error_type is None or error_type == error_data.type:
+            msgs.append(error_data)
+        else:
+            time.sleep(0.01)
+
+    return msgs