wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-10 12:38:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Check for GRPC status codes (#366)

Browse Source 
* check for GRPC status codes

* more checkss
This commit is contained in:
Philipp Moritz
2016-08-11 11:10:00 -07:00
committed by Robert Nishihara
parent 362ffa1f3c
commit b0ecff69ad
4 changed files with 38 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/worker.cc
+19 -19
View File
@@ -107,11 +107,10 @@ Worker::Worker(const std::string& node_ip_address, const std::string& scheduler_
SubmitTaskReply Worker::submit_task(SubmitTaskRequest* request, int max_retries, int retry_wait_milliseconds) {
RAY_CHECK(connected_, "Attempted to perform submit_task but failed.");
SubmitTaskReply reply;
Status status;
request->set_workerid(workerid_);
for (int i = 0; i < 1 + max_retries; ++i) {
ClientContext context;
status = scheduler_stub_->SubmitTask(&context, *request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->SubmitTask(&context, *request, &reply));
if (reply.function_registered()) {
break;
}
@@ -124,7 +123,7 @@ SubmitTaskReply Worker::submit_task(SubmitTaskRequest* request, int max_retries,
bool Worker::kill_workers(ClientContext &context) {
KillWorkersRequest request;
KillWorkersReply reply;
Status status = scheduler_stub_->KillWorkers(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->KillWorkers(&context, request, &reply));
return reply.success();
}
@@ -138,18 +137,19 @@ void Worker::register_worker(const std::string& node_ip_address, const std::stri
request.set_objstore_address(objstore_address);
request.set_is_driver(is_driver);
RegisterWorkerReply reply;
grpc::StatusCode status_code = grpc::UNAVAILABLE;
Status status;
// TODO: HACK: retrying is a hack
for (int i = 0; i < 5; ++i) {
ClientContext context;
status_code = scheduler_stub_->RegisterWorker(&context, request, &reply).error_code();
if (status_code != grpc::UNAVAILABLE) {
status = scheduler_stub_->RegisterWorker(&context, request, &reply);
if (status.error_code() != grpc::UNAVAILABLE) {
break;
}
// Note that each pass through the loop may take substantially longer than
// retry_wait_milliseconds because grpc may do its own retrying.
std::this_thread::sleep_for(std::chrono::milliseconds(retry_wait_milliseconds));
}
RAY_CHECK_GRPC(status);
workerid_ = reply.workerid();
objstoreid_ = reply.objstoreid();
objstore_address_ = reply.objstore_address();
@@ -173,7 +173,7 @@ void Worker::request_object(ObjectID objectid) {
request.set_objectid(objectid);
AckReply reply;
ClientContext context;
Status status = scheduler_stub_->RequestObj(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->RequestObj(&context, request, &reply));
return;
}
@@ -184,7 +184,7 @@ ObjectID Worker::get_objectid() {
request.set_workerid(workerid_);
PutObjReply reply;
ClientContext context;
Status status = scheduler_stub_->PutObj(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->PutObj(&context, request, &reply));
return reply.objectid();
}
@@ -240,7 +240,7 @@ void Worker::put_object(ObjectID objectid, const Obj* obj, std::vector<ObjectID>
}
AckReply reply;
ClientContext context;
scheduler_stub_->AddContainedObjectIDs(&context, contained_objectids_request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->AddContainedObjectIDs(&context, contained_objectids_request, &reply));
}
#define CHECK_ARROW_STATUS(s, msg) \
@@ -322,7 +322,7 @@ void Worker::alias_objectids(ObjectID alias_objectid, ObjectID target_objectid)
request.set_alias_objectid(alias_objectid);
request.set_target_objectid(target_objectid);
AckReply reply;
scheduler_stub_->AliasObjectIDs(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->AliasObjectIDs(&context, request, &reply));
}
void Worker::increment_reference_count(std::vector<ObjectID> &objectids) {
@@ -338,7 +338,7 @@ void Worker::increment_reference_count(std::vector<ObjectID> &objectids) {
request.add_objectid(objectids[i]);
}
AckReply reply;
scheduler_stub_->IncrementRefCount(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->IncrementRefCount(&context, request, &reply));
}
}
@@ -355,7 +355,7 @@ void Worker::decrement_reference_count(std::vector<ObjectID> &objectids) {
request.add_objectid(objectids[i]);
}
AckReply reply;
scheduler_stub_->DecrementRefCount(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->DecrementRefCount(&context, request, &reply));
}
}
@@ -367,7 +367,7 @@ void Worker::register_remote_function(const std::string& name, size_t num_return
request.set_function_name(name);
request.set_num_return_vals(num_return_vals);
AckReply reply;
scheduler_stub_->RegisterRemoteFunction(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->RegisterRemoteFunction(&context, request, &reply));
}
void Worker::notify_failure(FailedType type, const std::string& name, const std::string& error_message) {
@@ -380,7 +380,7 @@ void Worker::notify_failure(FailedType type, const std::string& name, const std:
request.mutable_failure()->set_name(name);
request.mutable_failure()->set_error_message(error_message);
AckReply reply;
scheduler_stub_->NotifyFailure(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->NotifyFailure(&context, request, &reply));
}
std::unique_ptr<WorkerMessage> Worker::receive_next_message() {
@@ -395,7 +395,7 @@ void Worker::ready_for_new_task() {
ReadyForNewTaskRequest request;
request.set_workerid(workerid_);
AckReply reply;
scheduler_stub_->ReadyForNewTask(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->ReadyForNewTask(&context, request, &reply));
}
void Worker::disconnect() {
@@ -410,12 +410,12 @@ void Worker::disconnect() {
// TODO(rkn): Should we be using pointers or references? And should they be const?
void Worker::scheduler_info(ClientContext &context, SchedulerInfoRequest &request, SchedulerInfoReply &reply) {
RAY_CHECK(connected_, "Attempted to get scheduler info but failed.");
scheduler_stub_->SchedulerInfo(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->SchedulerInfo(&context, request, &reply));
}
void Worker::task_info(ClientContext &context, TaskInfoRequest &request, TaskInfoReply &reply) {
RAY_CHECK(connected_, "Attempted to get worker info but failed.");
scheduler_stub_->TaskInfo(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->TaskInfo(&context, request, &reply));
}
bool Worker::export_remote_function(const std::string& function_name, const std::string& function) {
@@ -425,7 +425,7 @@ bool Worker::export_remote_function(const std::string& function_name, const std:
request.mutable_function()->set_name(function_name);
request.mutable_function()->set_implementation(function);
AckReply reply;
Status status = scheduler_stub_->ExportRemoteFunction(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->ExportRemoteFunction(&context, request, &reply));
return true;
}
@@ -437,7 +437,7 @@ void Worker::export_reusable_variable(const std::string& name, const std::string
request.mutable_reusable_variable()->mutable_initializer()->set_implementation(initializer);
request.mutable_reusable_variable()->mutable_reinitializer()->set_implementation(reinitializer);
AckReply reply;
Status status = scheduler_stub_->ExportReusableVariable(&context, request, &reply);
RAY_CHECK_GRPC(scheduler_stub_->ExportReusableVariable(&context, request, &reply));
}
// Communication between the WorkerServer and the Worker happens via a message