Push an error to the driver when the workload hangs on ray.put reconstruction (#382)

* Fix worker blocked bug

* tmp

* Push an error to the driver on ray.put for non-driver tasks

* Fix result table tests

* Fix test, logging

* Address comments

* Fix suppression bug

* Fix redis module test

* Edit error message

* Get values in chunks during reconstruction

* Test case for driver ray.put errors

* Error for evicting ray.put objects from the driver

* Fix tests

* Reduce verbosity

* Documentation
This commit is contained in:
Stephanie Wang
2017-03-21 00:16:48 -07:00
committed by Robert Nishihara
parent 4618fd45b1
commit 083e7a28ad
21 changed files with 528 additions and 132 deletions
+3 -1
View File
@@ -34,6 +34,7 @@ void new_object_fail_callback(UniqueID id,
void new_object_done_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
CHECK(ObjectID_equal(object_id, new_object_id));
@@ -60,7 +61,7 @@ void new_object_task_callback(TaskID task_id, void *user_context) {
.fail_callback = new_object_fail_callback,
};
DBHandle *db = (DBHandle *) user_context;
result_table_add(db, new_object_id, new_object_task_id, &retry,
result_table_add(db, new_object_id, new_object_task_id, false, &retry,
new_object_lookup_callback, (void *) db);
}
@@ -95,6 +96,7 @@ TEST new_object_test(void) {
void new_object_no_task_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
CHECK(IS_NIL_ID(task_id));