catch exceptions on workers and pass them to the scheduler (#93)

src/raylib.cc

@@ -662,10 +662,14 @@ PyObject* submit_task(PyObject* self, PyObject* args) {
 
 PyObject* notify_task_completed(PyObject* self, PyObject* args) {
   Worker* worker;
-  if (!PyArg_ParseTuple(args, "O&", &PyObjectToWorker, &worker)) {
+  PyObject* task_succeeded_obj;
+  const char* error_message_ptr;
+  if (!PyArg_ParseTuple(args, "O&Os", &PyObjectToWorker, &worker, &task_succeeded_obj, &error_message_ptr)) {
     return NULL;
   }
-  worker->notify_task_completed();
+  std::string error_message(error_message_ptr);
+  bool task_succeeded = PyObject_IsTrue(task_succeeded_obj);
+  worker->notify_task_completed(task_succeeded, error_message);
   Py_RETURN_NONE;
 }
 

src/scheduler.cc

@@ -144,12 +144,15 @@ Status SchedulerService::ObjReady(ServerContext* context, const ObjReadyRequest*
   return Status::OK;
 }
 
-Status SchedulerService::WorkerReady(ServerContext* context, const WorkerReadyRequest* request, AckReply* reply) {
+Status SchedulerService::NotifyTaskCompleted(ServerContext* context, const NotifyTaskCompletedRequest* request, AckReply* reply) {
   RAY_LOG(RAY_INFO, "worker " << request->workerid() << " reported back");
   {
     std::lock_guard<std::mutex> lock(avail_workers_lock_);
     avail_workers_.push_back(request->workerid());
   }
+  if (!request->task_succeeded()) {
+    RAY_LOG(RAY_FATAL, "The task on worker " << request->workerid() << " threw an exception with the following error message: " << request->error_message());
+  }
   schedule();
   return Status::OK;
 }

src/scheduler.h

@@ -60,7 +60,7 @@ public:
   Status RegisterWorker(ServerContext* context, const RegisterWorkerRequest* request, RegisterWorkerReply* reply) override;
   Status RegisterFunction(ServerContext* context, const RegisterFunctionRequest* request, AckReply* reply) override;
   Status ObjReady(ServerContext* context, const ObjReadyRequest* request, AckReply* reply) override;
-  Status WorkerReady(ServerContext* context, const WorkerReadyRequest* request, AckReply* reply) override;
+  Status NotifyTaskCompleted(ServerContext* context, const NotifyTaskCompletedRequest* request, AckReply* reply) override;
   Status IncrementRefCount(ServerContext* context, const IncrementRefCountRequest* request, AckReply* reply) override;
   Status DecrementRefCount(ServerContext* context, const DecrementRefCountRequest* request, AckReply* reply) override;
   Status AddContainedObjRefs(ServerContext* context, const AddContainedObjRefsRequest* request, AckReply* reply) override;

src/worker.cc

@@ -263,15 +263,17 @@ Task* Worker::receive_next_task() {
   return task;
 }
 
-void Worker::notify_task_completed() {
+void Worker::notify_task_completed(bool task_succeeded, std::string error_message) {
   if (!connected_) {
     RAY_LOG(RAY_FATAL, "Attempting to perform notify_task_completed, but connected_ = " << connected_ << ".");
   }
   ClientContext context;
-  WorkerReadyRequest request;
+  NotifyTaskCompletedRequest request;
   request.set_workerid(workerid_);
+  request.set_task_succeeded(task_succeeded);
+  request.set_error_message(error_message);
   AckReply reply;
-  scheduler_stub_->WorkerReady(&context, request, &reply);
+  scheduler_stub_->NotifyTaskCompleted(&context, request, &reply);
 }
 
 void Worker::disconnect() {

src/worker.h

@@ -71,8 +71,10 @@ class Worker {
   void start_worker_service();
   // wait for next task from the RPC system
   Task* receive_next_task();
-  // tell the scheduler that we are done with the current task and request the next one
-  void notify_task_completed();
+  // tell the scheduler that we are done with the current task and request the
+  // next one, if task_succeeded is false, this tells the scheduler that the
+  // task threw an exception
+  void notify_task_completed(bool task_succeeded, std::string error_message);
   // disconnect the worker
   void disconnect();
   // return connected_