#ifndef RAY_WORKER_H #define RAY_WORKER_H #include #include #include #include #include #include using grpc::Server; using grpc::ServerBuilder; using grpc::ServerContext; using grpc::Status; #include "ray.grpc.pb.h" #include "ray/ray.h" #include "ipc.h" using grpc::Channel; using grpc::ClientContext; using grpc::ClientWriter; // These three constants are used to define the mode that a worker is running // in. Right now, this is mostly used for determining how to print information // about task failures. enum Mode {SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, SILENT_MODE}; class WorkerServiceImpl final : public WorkerService::Service { public: WorkerServiceImpl(const std::string& worker_address, Mode mode); Status ExecuteTask(ServerContext* context, const ExecuteTaskRequest* request, AckReply* reply) override; Status ImportRemoteFunction(ServerContext* context, const ImportRemoteFunctionRequest* request, AckReply* reply) override; Status Die(ServerContext* context, const DieRequest* request, AckReply* reply) override; Status ImportReusableVariable(ServerContext* context, const ImportReusableVariableRequest* request, AckReply* reply) override; Status PrintErrorMessage(ServerContext* context, const PrintErrorMessageRequest* request, AckReply* reply) override; private: // The queue used to send commands from the worker service to the worker. This // corresponds to the receive_queue_ in the worker. MessageQueue send_queue_; // This is true if the worker service is part of a driver process and false // if it is part of a worker process. Mode mode_; }; class Worker { public: Worker(const std::string& node_ip_address, const std::string& scheduler_address, Mode mode); // Submit a remote task to the scheduler. If the function in the task is not // registered with the scheduler, we will sleep for retry_wait_milliseconds // and try to resubmit the task to the scheduler up to max_retries more times. SubmitTaskReply submit_task(SubmitTaskRequest* request, int max_retries = 120, int retry_wait_milliseconds = 500); // Requests the scheduler to kill workers bool kill_workers(ClientContext &context); // send request to the scheduler to register this worker void register_worker(const std::string& ip_address, const std::string& objstore_address, bool is_driver); // get a new object ID that is registered with the scheduler ObjectID get_objectid(); // request an object to be delivered to the local object store void request_object(ObjectID objectid); // stores an object to the local object store void put_object(ObjectID objectid, const Obj* obj, std::vector &contained_objectids); // retrieve serialized object from local object store slice get_object(ObjectID objectid); // Allocates buffer for objectid with size of size const char* allocate_buffer(ObjectID objectid, int64_t size, SegmentId& segmentid); // Finishes buffer with segmentid and an offset of metadata_ofset PyObject* finish_buffer(ObjectID objectid, SegmentId segmentid, int64_t metadata_offset); // Gets the buffer for objectid const char* get_buffer(ObjectID objectid, int64_t& size, SegmentId& segmentid, int64_t& metadata_offset); // determine if the object stored in objectid is an arrow object // TODO(pcm): more general mechanism for this? bool is_arrow(ObjectID objectid); // unmap the segment containing an object from the local address space void unmap_object(ObjectID objectid); // make `alias_objectid` refer to the same object that `target_objectid` refers to void alias_objectids(ObjectID alias_objectid, ObjectID target_objectid); // increment the reference count for objectid void increment_reference_count(std::vector &objectid); // decrement the reference count for objectid void decrement_reference_count(std::vector &objectid); // Notify the scheduler that a remote function has been imported successfully. void register_remote_function(const std::string& name, size_t num_return_vals); // Notify the scheduler that a failure has occurred. void notify_failure(FailedType type, const std::string& name, const std::string& error_message); // Start the worker server which accepts commands from the scheduler. For // workers, these commands are stored in the message queue, which is read by // the Python interpreter. For drivers, these commands are only for printing // error messages. void start_worker_service(Mode mode); // wait for next task from the RPC system. If null, it means there are no more tasks and the worker should shut down. std::unique_ptr receive_next_message(); // Tell the scheduler that the worker is ready for a new task. void ready_for_new_task(); // disconnect the worker void disconnect(); // return connected_ bool connected() { return connected_; } // get info about scheduler state void scheduler_info(ClientContext &context, SchedulerInfoRequest &request, SchedulerInfoReply &reply); // get task statuses from scheduler void task_info(ClientContext &context, TaskInfoRequest &request, TaskInfoReply &reply); // gets indices of available objects std::vector select(std::vector& objectids); // export function to workers bool export_remote_function(const std::string& function_name, const std::string& function); // export reusable variable to workers void export_reusable_variable(const std::string& name, const std::string& initializer, const std::string& reinitializer); // return the worker address const char* get_worker_address() { return worker_address_.c_str(); } private: Mode mode_; bool connected_; const size_t CHUNK_SIZE = 8 * 1024; std::unique_ptr scheduler_stub_; Server* server_ptr_; std::thread worker_server_thread_; bip::managed_shared_memory segment_; WorkerId workerid_; ObjStoreId objstoreid_; std::string scheduler_address_; std::string objstore_address_; std::string worker_address_; std::string node_ip_address_; // The queue used to send commands from the worker service to the worker. // This queue is created by the worker. This corresponds to the send_queue_ in // the worker service. MessageQueue receive_queue_; // The name of the receive queue. std::string receive_queue_name_; // The queue used to send requests to the object store. There is a single // queue shared by all workers sending requests to the object store, and this // queue is created by the object store. MessageQueue request_obj_queue_; // The queue used to receive object addresses from the object store. This // queue is created by this worker. MessageQueue receive_obj_queue_; std::shared_ptr segmentpool_; }; #endif