Fix bug in ray.errors and update its default behavior (#5576)

python/ray/tests/utils.py

@@ -75,8 +75,15 @@ def run_string_as_driver_nonblocking(driver_script):
             [sys.executable, f.name], stdout=subprocess.PIPE)
 
 
+def flat_errors():
+    errors = []
+    for job_errors in ray.errors(all_jobs=True).values():
+        errors.extend(job_errors)
+    return errors
+
+
 def relevant_errors(error_type):
-    return [info for info in ray.errors() if info["type"] == error_type]
+    return [error for error in flat_errors() if error["type"] == error_type]
 
 
 def wait_for_errors(error_type, num_errors, timeout=10):