wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use gRPC to handle communication and data transmission between object manager (#4996)

Browse Source
This commit is contained in:
Joey Jiang
2019-06-28 10:56:34 +08:00
committed by Hao Chen
parent 62e4b591e3
commit d6bbbdef35
21 changed files with 714 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/object_manager/object_manager.h
+113 -65
View File
@@ -6,6 +6,7 @@
#include <deque>
#include <map>
#include <memory>
#include <mutex>
#include <random>
#include <thread>
@@ -25,6 +26,8 @@
#include "ray/object_manager/object_directory.h"
#include "ray/object_manager/object_manager_client_connection.h"
#include "ray/object_manager/object_store_notification_manager.h"
#include "ray/rpc/object_manager/object_manager_client.h"
#include "ray/rpc/object_manager/object_manager_server.h"
namespace ray {
@@ -36,10 +39,6 @@ struct ObjectManagerConfig {
/// The time in milliseconds to wait before retrying a pull
/// that fails due to client id lookup.
uint pull_timeout_ms;
/// Maximum number of sends allowed.
int max_sends;
/// Maximum number of receives allowed.
int max_receives;
/// Object chunk size, in bytes
uint64_t object_chunk_size;
/// The store socket name.
@@ -49,6 +48,9 @@ struct ObjectManagerConfig {
/// Negative: waiting infinitely.
/// 0: giving up retrying immediately.
int push_timeout_ms;
/// Number of threads of rpc service
/// Send and receive request in these threads
int rpc_service_threads_number;
};
struct LocalObjectInfo {
@@ -67,7 +69,81 @@ class ObjectManagerInterface {
};
// TODO(hme): Add success/failure callbacks for push and pull.
class ObjectManager : public ObjectManagerInterface {
class ObjectManager : public ObjectManagerInterface,
public rpc::ObjectManagerServiceHandler {
public:
/// Implementation of object manager service
/// Handle push request from remote object manager
///
/// Push request will contain the object which is specified by pull request
/// the object will be transfered by a sequence of chunks.
///
/// \param request Push request including the object chunk data
/// \param reply Reply to the sender
/// \param done_callback Callback of the request
void HandlePushRequest(const rpc::PushRequest &request, rpc::PushReply *reply,
rpc::RequestDoneCallback done_callback) override;
/// Handle pull request from remote object manager
///
/// \param request Pull request
/// \param reply Reply
/// \param done_callback Callback of request
void HandlePullRequest(const rpc::PullRequest &request, rpc::PullReply *reply,
rpc::RequestDoneCallback done_callback) override;
/// Handle free objects request
///
/// \param request Free objects request
/// \param reply Reply
/// \param done_callback
void HandleFreeObjectsRequest(const rpc::FreeObjectsRequest &request,
rpc::FreeObjectsReply *reply,
rpc::RequestDoneCallback done_callback) override;
/// Send object to remote object manager
///
/// Object will be transfered as a sequence of chunks, small object(defined in config)
/// contains only one chunk
/// \param push_id Unique push id to indicate this push request
/// \param object_id Object id
/// \param data_size Data size
/// \param metadata_size Metadata size
/// \param chunk_index Chunk index of this object chunk, start with 0
/// \param rpc_client Rpc client used to send message to remote object manager
ray::Status SendObjectChunk(const UniqueID &push_id, const ObjectID &object_id,
const ClientID &client_id, uint64_t data_size,
uint64_t metadata_size, uint64_t chunk_index,
std::shared_ptr<rpc::ObjectManagerClient> rpc_client);
/// Receive object chunk from remote object manager, small object may contain one chunk
///
/// \param client_id Client id of remote object manager which sends this chunk
/// \param object_id Object id
/// \param data_size Data size
/// \param metadata_size Metadata size
/// \param chunk_index Chunk index
/// \param data Chunk data
ray::Status ReceiveObjectChunk(const ClientID &client_id, const ObjectID &object_id,
uint64_t data_size, uint64_t metadata_size,
uint64_t chunk_index, const std::string &data);
/// Send pull request
///
/// \param object_id Object id
/// \param client_id Remote server client id
void SendPullRequest(const ObjectID &object_id, const ClientID &client_id,
std::shared_ptr<rpc::ObjectManagerClient> rpc_client);
/// Get the rpc client according to the client ID
///
/// \param client_id Remote client id, will send rpc request to it
std::shared_ptr<rpc::ObjectManagerClient> GetRpcClient(const ClientID &client_id);
/// Get the port of the object manager rpc server.
int GetServerPort() const { return object_manager_server_.GetPort(); }
public:
/// Takes user-defined ObjectDirectoryInterface implementation.
/// When this constructor is used, the ObjectManager assumes ownership of
@@ -244,13 +320,14 @@ class ObjectManager : public ObjectManagerInterface {
/// Spread the Free request to all objects managers.
///
/// \param object_ids the The list of ObjectIDs to be deleted.
void SpreadFreeObjectRequest(const std::vector<ObjectID> &object_ids);
void SpreadFreeObjectsRequest(
const std::vector<ObjectID> &object_ids,
const std::vector<std::shared_ptr<rpc::ObjectManagerClient>> &rpc_clients);
/// Handle starting, running, and stopping asio io_service.
void StartIOService();
void RunSendService();
void RunReceiveService();
void StopIOService();
/// Handle starting, running, and stopping asio rpc_service.
void StartRpcService();
void RunRpcService();
void StopRpcService();
/// Handle an object being added to this node. This adds the object to the
/// directory, pushes the object to other nodes if necessary, and cancels any
@@ -274,9 +351,6 @@ class ObjectManager : public ObjectManagerInterface {
void PullSendRequest(const ObjectID &object_id,
std::shared_ptr<SenderConnection> &conn);
std::shared_ptr<SenderConnection> CreateSenderConnection(
ConnectionPool::ConnectionType type, RemoteConnectionInfo info);
/// This is used to notify the main thread that the sending of a chunk has
/// completed.
///
@@ -309,49 +383,15 @@ class ObjectManager : public ObjectManagerInterface {
uint64_t chunk_index, double start_time_us,
double end_time_us, ray::Status status);
/// Begin executing a send.
/// Executes on send_service_ thread pool.
ray::Status ExecuteSendObject(const UniqueID &push_id, const ClientID &client_id,
const ObjectID &object_id, uint64_t data_size,
uint64_t metadata_size, uint64_t chunk_index,
const RemoteConnectionInfo &connection_info);
/// This method synchronously sends the object id and object size
/// to the remote object manager.
/// Executes on send_service_ thread pool.
ray::Status SendObjectHeaders(const UniqueID &push_id, const ObjectID &object_id,
uint64_t data_size, uint64_t metadata_size,
uint64_t chunk_index,
std::shared_ptr<SenderConnection> &conn);
/// This method initiates the actual object transfer.
/// Executes on send_service_ thread pool.
ray::Status SendObjectData(const ObjectID &object_id,
const ObjectBufferPool::ChunkInfo &chunk_info,
std::shared_ptr<SenderConnection> &conn);
/// Invoked when a remote object manager pushes an object to this object manager.
/// This will invoke the object receive on the receive_service_ thread pool.
void ReceivePushRequest(std::shared_ptr<TcpClientConnection> &conn,
const uint8_t *message);
/// Execute a receive on the receive_service_ thread pool.
ray::Status ExecuteReceiveObject(const ClientID &client_id, const ObjectID &object_id,
uint64_t data_size, uint64_t metadata_size,
uint64_t chunk_index, TcpClientConnection &conn);
/// Handles receiving a pull request message.
void ReceivePullRequest(std::shared_ptr<TcpClientConnection> &conn,
const uint8_t *message);
/// Handles freeing objects request.
void ReceiveFreeRequest(std::shared_ptr<TcpClientConnection> &conn,
const uint8_t *message);
/// Handles connect message of a new client connection.
void ConnectClient(std::shared_ptr<TcpClientConnection> &conn, const uint8_t *message);
/// Handles disconnect message of an existing client connection.
void DisconnectClient(std::shared_ptr<TcpClientConnection> &conn,
const uint8_t *message);
/// Handle Push task timeout.
void HandlePushTaskTimeout(const ObjectID &object_id, const ClientID &client_id);
@@ -361,28 +401,19 @@ class ObjectManager : public ObjectManagerInterface {
ObjectStoreNotificationManager store_notification_;
ObjectBufferPool buffer_pool_;
/// This runs on a thread pool dedicated to sending objects.
boost::asio::io_service send_service_;
/// This runs on a thread pool dedicated to receiving objects.
boost::asio::io_service receive_service_;
/// Weak reference to main service. We ensure this object is destroyed before
/// main_service_ is stopped.
boost::asio::io_service *main_service_;
/// Used to create "work" for send_service_.
/// Without this, if send_service_ has no more sends to process, it will stop.
boost::asio::io_service::work send_work_;
/// Used to create "work" for receive_service_.
/// Without this, if receive_service_ has no more receives to process, it will stop.
boost::asio::io_service::work receive_work_;
/// Multi-thread asio service, deal with all outgoing and incoming RPC request.
boost::asio::io_service rpc_service_;
/// Runs the send service, which handle
/// all outgoing object transfers.
std::vector<std::thread> send_threads_;
/// Runs the receive service, which handle
/// all incoming object transfers.
std::vector<std::thread> receive_threads_;
/// Keep rpc service running when no task in rpc service.
boost::asio::io_service::work rpc_work_;
/// The thread pool used for running `rpc_service`.
/// Data copy operations during request are done in this thread pool.
std::vector<std::thread> rpc_threads_;
/// Connection pool for reusing outgoing connections to remote object managers.
ConnectionPool connection_pool_;
@@ -414,8 +445,25 @@ class ObjectManager : public ObjectManagerInterface {
/// table in the GCS.
std::vector<rpc::ProfileTableData::ProfileEvent> profile_events_;
/// mutex lock used to protect profile_events_, profile_events_ is used in main thread
/// and rpc thread.
std::mutex profile_mutex_;
/// Internally maintained random number generator.
std::mt19937_64 gen_;
/// The gPRC server.
rpc::GrpcServer object_manager_server_;
/// The gRPC service.
rpc::ObjectManagerGrpcService object_manager_service_;
/// The client call manager used to deal with reply.
rpc::ClientCallManager client_call_manager_;
/// clientID - object manager gRPC client.
std::unordered_map<ClientID, std::shared_ptr<rpc::ObjectManagerClient>>
remote_object_manager_clients_;
};
} // namespace ray