diff --git a/python/ray/tests/test_cancel.py b/python/ray/tests/test_cancel.py index 99f5227b6..11b4dfbd4 100644 --- a/python/ray/tests/test_cancel.py +++ b/python/ray/tests/test_cancel.py @@ -185,16 +185,16 @@ def test_stress(shutdown_only, use_force): for done in cancelled: with pytest.raises(valid_exceptions(use_force)): - ray.get(done) + ray.get(done, timeout=120) for indx, t in enumerate(tasks): if sleep_or_no[indx]: ray.cancel(t, force=use_force) cancelled.add(t) if t in cancelled: with pytest.raises(valid_exceptions(use_force)): - ray.get(t) + ray.get(t, timeout=120) else: - ray.get(t) + ray.get(t, timeout=120) @pytest.mark.parametrize("use_force", [True, False]) @@ -209,6 +209,12 @@ def test_fast(shutdown_only, use_force): ids = list() for _ in range(100): x = fast.remote("a") + # NOTE If a non-force Cancellation is attempted in the time + # between a worker receiving a task and the worker executing + # that task (specifically the python execution), Cancellation + # can fail. + if not use_force: + time.sleep(0.1) ray.cancel(x, force=use_force) ids.append(x) @@ -225,11 +231,12 @@ def test_fast(shutdown_only, use_force): if random.random() > 0.95: ray.cancel(ids[idx], force=use_force) signaler.send.remote() - for obj_ref in ids: + for i, obj_ref in enumerate(ids): try: - ray.get(obj_ref) + ray.get(obj_ref, timeout=120) except Exception as e: - assert isinstance(e, valid_exceptions(use_force)) + assert isinstance( + e, valid_exceptions(use_force)), f"Failure on iteration: {i}" @pytest.mark.parametrize("use_force", [True, False])