Fix failure handling for actor death (#3359)

* Broadcast actor death, clean up dummy objects

* Reduce logging and clean up state when failing a task

* lint

* Make actor failure test nicer, reduce node timeout
This commit is contained in:
Stephanie Wang
2018-11-21 12:26:22 -08:00
committed by GitHub
parent 1a926c9b7c
commit 3e33f6f71b
9 changed files with 231 additions and 119 deletions
+6 -1
View File
@@ -3,6 +3,7 @@ from __future__ import division
from __future__ import print_function
import os
import json
import signal
import time
@@ -262,7 +263,11 @@ def _test_component_failed(component_type):
num_local_schedulers=num_local_schedulers,
start_ray_local=True,
num_cpus=[num_workers_per_scheduler] * num_local_schedulers,
redirect_output=True)
redirect_output=True,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10,
}))
# Submit many tasks with many dependencies.
@ray.remote