[Cancellation] Make Test Cancel Easier to Debug (#13243)

* first commit

* lint-fix

python/ray/tests/test_cancel.py

@@ -185,16 +185,16 @@ def test_stress(shutdown_only, use_force):
 
     for done in cancelled:
         with pytest.raises(valid_exceptions(use_force)):
-            ray.get(done)
+            ray.get(done, timeout=120)
     for indx, t in enumerate(tasks):
         if sleep_or_no[indx]:
             ray.cancel(t, force=use_force)
             cancelled.add(t)
         if t in cancelled:
             with pytest.raises(valid_exceptions(use_force)):
-                ray.get(t)
+                ray.get(t, timeout=120)
         else:
-            ray.get(t)
+            ray.get(t, timeout=120)
 
 
 @pytest.mark.parametrize("use_force", [True, False])
@@ -209,6 +209,12 @@ def test_fast(shutdown_only, use_force):
     ids = list()
     for _ in range(100):
         x = fast.remote("a")
+        # NOTE If a non-force Cancellation is attempted in the time
+        # between a worker receiving a task and the worker executing
+        # that task (specifically the python execution), Cancellation
+        # can fail.
+        if not use_force:
+            time.sleep(0.1)
         ray.cancel(x, force=use_force)
         ids.append(x)
 
@@ -225,11 +231,12 @@ def test_fast(shutdown_only, use_force):
         if random.random() > 0.95:
             ray.cancel(ids[idx], force=use_force)
     signaler.send.remote()
-    for obj_ref in ids:
+    for i, obj_ref in enumerate(ids):
         try:
-            ray.get(obj_ref)
+            ray.get(obj_ref, timeout=120)
         except Exception as e:
-            assert isinstance(e, valid_exceptions(use_force))
+            assert isinstance(
+                e, valid_exceptions(use_force)), f"Failure on iteration: {i}"
 
 
 @pytest.mark.parametrize("use_force", [True, False])