wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-15 11:25:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming]Streaming queue support failover (#8161)

Browse Source
This commit is contained in:
wanxing
2020-08-25 14:19:45 +08:00
committed by GitHub
parent 5a787a8253
commit e816e3aefb
16 changed files with 1049 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/queue/queue_handler.cc
+137 -2
View File
@@ -39,6 +39,15 @@ std::shared_ptr<Message> QueueMessageHandler::ParseMessage(
case queue::protobuf::StreamingQueueMessageType::StreamingQueueCheckRspMsgType:
message = CheckRspMessage::FromBytes(bytes);
break;
case queue::protobuf::StreamingQueuePullRequestMsgType:
message = PullRequestMessage::FromBytes(bytes);
break;
case queue::protobuf::StreamingQueuePullResponseMsgType:
message = PullResponseMessage::FromBytes(bytes);
break;
case queue::protobuf::StreamingQueueResendDataMsgType:
message = ResendDataMessage::FromBytes(bytes);
break;
default:
STREAMING_CHECK(false) << "nonsupport message type: "
<< queue::protobuf::StreamingQueueMessageType_Name(*type);
@@ -109,6 +118,21 @@ void QueueMessageHandler::Stop() {
}
}
void UpstreamQueueMessageHandler::Start() {
STREAMING_LOG(INFO) << "UpstreamQueueMessageHandler::Start";
QueueMessageHandler::Start();
handle_service_thread_ = std::thread([this] { handler_service_.run(); });
}
void UpstreamQueueMessageHandler::Stop() {
STREAMING_LOG(INFO) << "UpstreamQueueMessageHandler::Stop";
handler_service_.stop();
if (handle_service_thread_.joinable()) {
handle_service_thread_.join();
}
QueueMessageHandler::Stop();
}
std::shared_ptr<UpstreamQueueMessageHandler> UpstreamQueueMessageHandler::CreateService(
const ActorID &actor_id) {
if (nullptr == upstream_handler_) {
@@ -203,7 +227,7 @@ void UpstreamQueueMessageHandler::DispatchMessageInternal(
std::shared_ptr<LocalMemoryBuffer> buffer,
std::function<void(std::shared_ptr<LocalMemoryBuffer>)> callback) {
std::shared_ptr<Message> msg = ParseMessage(buffer);
STREAMING_LOG(DEBUG) << "QueueMessageHandler::DispatchMessageInternal: "
STREAMING_LOG(DEBUG) << "UpstreamQueueMessageHandler::DispatchMessageInternal: "
<< " qid: " << msg->QueueId() << " actorid " << msg->ActorId()
<< " peer actorid: " << msg->PeerActorId() << " type: "
<< queue::protobuf::StreamingQueueMessageType_Name(msg->Type());
@@ -214,6 +238,13 @@ void UpstreamQueueMessageHandler::DispatchMessageInternal(
} else if (msg->Type() ==
queue::protobuf::StreamingQueueMessageType::StreamingQueueCheckRspMsgType) {
STREAMING_CHECK(false) << "Should not receive StreamingQueueCheckRspMsg";
} else if (msg->Type() ==
queue::protobuf::StreamingQueueMessageType::StreamingQueuePullRequestMsgType) {
STREAMING_CHECK(callback) << "StreamingQueuePullRequestMsg "
<< " qid: " << msg->QueueId() << " actorid "
<< msg->ActorId()
<< " peer actorid: " << msg->PeerActorId();
OnPullRequest(std::dynamic_pointer_cast<PullRequestMessage>(msg), callback);
} else {
STREAMING_CHECK(false) << "message type should be added: "
<< queue::protobuf::StreamingQueueMessageType_Name(
@@ -235,6 +266,25 @@ void UpstreamQueueMessageHandler::OnNotify(
queue->OnNotify(notify_msg);
}
void UpstreamQueueMessageHandler::OnPullRequest(
std::shared_ptr<PullRequestMessage> pull_msg,
std::function<void(std::shared_ptr<LocalMemoryBuffer>)> callback) {
STREAMING_LOG(INFO) << "OnPullRequest";
auto queue = upstream_queues_.find(pull_msg->QueueId());
if (queue == upstream_queues_.end()) {
STREAMING_LOG(INFO) << "Can not find queue " << pull_msg->QueueId();
PullResponseMessage msg(pull_msg->PeerActorId(), pull_msg->ActorId(),
pull_msg->QueueId(), QUEUE_INVALID_SEQ_ID,
QUEUE_INVALID_SEQ_ID,
queue::protobuf::StreamingQueueError::QUEUE_NOT_EXIST, false);
std::unique_ptr<LocalMemoryBuffer> buffer = msg.ToBytes();
callback(std::move(buffer));
return;
}
queue->second->OnPull(pull_msg, handler_service_, callback);
}
void UpstreamQueueMessageHandler::ReleaseAllUpQueues() {
STREAMING_LOG(INFO) << "ReleaseAllUpQueues";
upstream_queues_.clear();
@@ -244,6 +294,8 @@ void UpstreamQueueMessageHandler::ReleaseAllUpQueues() {
std::shared_ptr<DownstreamQueueMessageHandler>
DownstreamQueueMessageHandler::CreateService(const ActorID &actor_id) {
if (nullptr == downstream_handler_) {
STREAMING_LOG(INFO) << "DownstreamQueueMessageHandler::CreateService "
<< " actorid: " << actor_id;
downstream_handler_ = std::make_shared<DownstreamQueueMessageHandler>(actor_id);
}
return downstream_handler_;
@@ -275,6 +327,25 @@ std::shared_ptr<ReaderQueue> DownstreamQueueMessageHandler::CreateDownstreamQueu
return queue;
}
StreamingQueueStatus DownstreamQueueMessageHandler::PullQueue(
const ObjectID &queue_id, uint64_t start_msg_id, bool &is_upstream_first_pull,
uint64_t timeout_ms) {
STREAMING_LOG(INFO) << "PullQueue queue_id: "
<< queue_id
<< " start_msg_id: " << start_msg_id
<< " is_upstream_first_pull: " << is_upstream_first_pull;
uint64_t start_time = current_time_ms();
uint64_t current_time = start_time;
StreamingQueueStatus st = StreamingQueueStatus::OK;
while (current_time < start_time + timeout_ms &&
(st = PullPeerAsync(queue_id, start_msg_id, is_upstream_first_pull,
timeout_ms)) == StreamingQueueStatus::Timeout) {
std::this_thread::sleep_for(std::chrono::milliseconds(200));
current_time = current_time_ms();
}
return st;
}
std::shared_ptr<streaming::ReaderQueue> DownstreamQueueMessageHandler::GetDownQueue(
const ObjectID &queue_id) {
auto it = downstream_queues_.find(queue_id);
@@ -311,7 +382,7 @@ void DownstreamQueueMessageHandler::DispatchMessageInternal(
std::shared_ptr<LocalMemoryBuffer> buffer,
std::function<void(std::shared_ptr<LocalMemoryBuffer>)> callback) {
std::shared_ptr<Message> msg = ParseMessage(buffer);
STREAMING_LOG(DEBUG) << "QueueMessageHandler::DispatchMessageInternal: "
STREAMING_LOG(DEBUG) << "DownstreamQueueMessageHandler::DispatchMessageInternal: "
<< " qid: " << msg->QueueId() << " actorid " << msg->ActorId()
<< " peer actorid: " << msg->PeerActorId() << " type: "
<< queue::protobuf::StreamingQueueMessageType_Name(msg->Type());
@@ -326,6 +397,22 @@ void DownstreamQueueMessageHandler::DispatchMessageInternal(
if (callback != nullptr) {
callback(check_result);
}
} else if (msg->Type() ==
queue::protobuf::StreamingQueueMessageType::StreamingQueueResendDataMsgType) {
auto queue = downstream_queues_.find(msg->QueueId());
if (queue == downstream_queues_.end()) {
std::shared_ptr<ResendDataMessage> data_msg =
std::dynamic_pointer_cast<ResendDataMessage>(msg);
STREAMING_LOG(DEBUG) << "Can not find queue for "
<< queue::protobuf::StreamingQueueMessageType_Name(msg->Type())
<< ", maybe queue has been destroyed, ignore it."
<< " seq id: " << data_msg->SeqId();
return;
}
std::shared_ptr<ResendDataMessage> resend_data_msg =
std::dynamic_pointer_cast<ResendDataMessage>(msg);
queue->second->OnResendData(resend_data_msg);
} else {
STREAMING_CHECK(false) << "message type should be added: "
<< queue::protobuf::StreamingQueueMessageType_Name(
@@ -347,5 +434,53 @@ void DownstreamQueueMessageHandler::OnData(std::shared_ptr<DataMessage> msg) {
queue->OnData(item);
}
StreamingQueueStatus DownstreamQueueMessageHandler::PullPeerAsync(
const ObjectID &queue_id, uint64_t start_msg_id, bool &is_upstream_first_pull,
uint64_t timeout_ms) {
STREAMING_LOG(INFO) << "PullPeerAsync queue_id: " << queue_id
<< " start_msg_id: " << start_msg_id;
auto queue = GetDownQueue(queue_id);
STREAMING_CHECK(queue != nullptr);
STREAMING_LOG(INFO) << "PullPeerAsync "
<< " actorid: " << queue->GetActorID();
PullRequestMessage msg(queue->GetActorID(), queue->GetPeerActorID(), queue_id,
start_msg_id);
std::unique_ptr<LocalMemoryBuffer> buffer = msg.ToBytes();
auto transport_it = GetOutTransport(queue_id);
STREAMING_CHECK(transport_it != nullptr);
std::shared_ptr<LocalMemoryBuffer> result_buffer =
transport_it->SendForResultWithRetry(std::move(buffer), 1, timeout_ms);
if (result_buffer == nullptr) {
return StreamingQueueStatus::Timeout;
}
std::shared_ptr<Message> result_msg = ParseMessage(result_buffer);
STREAMING_CHECK(result_msg->Type() ==
queue::protobuf::StreamingQueueMessageType::StreamingQueuePullResponseMsgType);
std::shared_ptr<PullResponseMessage> response_msg =
std::dynamic_pointer_cast<PullResponseMessage>(result_msg);
STREAMING_LOG(INFO) << "PullPeerAsync error: "
<< queue::protobuf::StreamingQueueError_Name(
response_msg->Error())
<< " start_msg_id: " << start_msg_id;
is_upstream_first_pull = response_msg->IsUpstreamFirstPull();
if (response_msg->Error() == queue::protobuf::StreamingQueueError::OK) {
STREAMING_LOG(INFO) << "Set queue " << queue_id << " expect_seq_id to "
<< response_msg->SeqId();
return StreamingQueueStatus::OK;
} else if (response_msg->Error() ==
queue::protobuf::StreamingQueueError::DATA_LOST) {
return StreamingQueueStatus::DataLost;
} else if (response_msg->Error() ==
queue::protobuf::StreamingQueueError::NO_VALID_DATA) {
return StreamingQueueStatus::NoValidData;
} else { // QUEUE_NOT_EXIST
return StreamingQueueStatus::Timeout;
}
}
} // namespace streaming
} // namespace ray