[Streaming]Streaming queue support failover (#8161)

2026-07-27 11:26:41 +08:00 · 2020-08-25 14:19:45 +08:00
parent 5a787a8253
commit e816e3aefb
16 changed files with 1049 additions and 212 deletions
@@ -13,7 +13,7 @@ std::unique_ptr<LocalMemoryBuffer> Message::ToBytes() {

  queue::protobuf::StreamingQueueMessageType type = Type();
  size_t total_len =
-      sizeof(Message::MagicNum) + sizeof(type) + sizeof(fbs_length) + fbs_length;
+      kItemHeaderSize + fbs_length;
  if (buffer_ != nullptr) {
    total_len += buffer_->Size();
  }
@@ -45,28 +45,35 @@ std::unique_ptr<LocalMemoryBuffer> Message::ToBytes() {
  return buffer;
 }

+
+void Message::FillMessageCommon(queue::protobuf::MessageCommon *common) {
+  common->set_src_actor_id(actor_id_.Binary());
+  common->set_dst_actor_id(peer_actor_id_.Binary());
+  common->set_queue_id(queue_id_.Binary());
+}
+
 void DataMessage::ToProtobuf(std::string *output) {
  queue::protobuf::StreamingQueueDataMsg msg;
-  msg.set_src_actor_id(actor_id_.Binary());
-  msg.set_dst_actor_id(peer_actor_id_.Binary());
-  msg.set_queue_id(queue_id_.Binary());
+  FillMessageCommon(msg.mutable_common());
  msg.set_seq_id(seq_id_);
+  msg.set_msg_id_start(msg_id_start_);
+  msg.set_msg_id_end(msg_id_end_);
  msg.set_length(buffer_->Size());
  msg.set_raw(raw_);
  msg.SerializeToString(output);
 }

 std::shared_ptr<DataMessage> DataMessage::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *fbs_length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *fbs_length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *fbs_length);
  queue::protobuf::StreamingQueueDataMsg message;
  message.ParseFromString(inputpb);
-  ActorID src_actor_id = ActorID::FromBinary(message.src_actor_id());
-  ActorID dst_actor_id = ActorID::FromBinary(message.dst_actor_id());
-  ObjectID queue_id = ObjectID::FromBinary(message.queue_id());
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
+  uint64_t msg_id_start = message.msg_id_start();
+  uint64_t msg_id_end = message.msg_id_end();
  uint64_t seq_id = message.seq_id();
  uint64_t length = message.length();
  bool raw = message.raw();
@@ -76,32 +83,27 @@ std::shared_ptr<DataMessage> DataMessage::FromBytes(uint8_t *bytes) {
  std::shared_ptr<LocalMemoryBuffer> buffer =
      std::make_shared<LocalMemoryBuffer>(bytes, (size_t)length, true);
  std::shared_ptr<DataMessage> data_msg = std::make_shared<DataMessage>(
-      src_actor_id, dst_actor_id, queue_id, seq_id, buffer, raw);
+      src_actor_id, dst_actor_id, queue_id, seq_id, msg_id_start, msg_id_end, buffer, raw);

  return data_msg;
 }

 void NotificationMessage::ToProtobuf(std::string *output) {
  queue::protobuf::StreamingQueueNotificationMsg msg;
-  msg.set_src_actor_id(actor_id_.Binary());
-  msg.set_dst_actor_id(peer_actor_id_.Binary());
-  msg.set_queue_id(queue_id_.Binary());
+  FillMessageCommon(msg.mutable_common());
  msg.set_seq_id(seq_id_);
  msg.SerializeToString(output);
 }

 std::shared_ptr<NotificationMessage> NotificationMessage::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
  queue::protobuf::StreamingQueueNotificationMsg message;
  message.ParseFromString(inputpb);
-  STREAMING_LOG(INFO) << "message.src_actor_id: " << message.src_actor_id();
-  ActorID src_actor_id = ActorID::FromBinary(message.src_actor_id());
-  ActorID dst_actor_id = ActorID::FromBinary(message.dst_actor_id());
-  ObjectID queue_id = ObjectID::FromBinary(message.queue_id());
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
  uint64_t seq_id = message.seq_id();

  std::shared_ptr<NotificationMessage> notify_msg =
@@ -112,23 +114,19 @@ std::shared_ptr<NotificationMessage> NotificationMessage::FromBytes(uint8_t *byt

 void CheckMessage::ToProtobuf(std::string *output) {
  queue::protobuf::StreamingQueueCheckMsg msg;
-  msg.set_src_actor_id(actor_id_.Binary());
-  msg.set_dst_actor_id(peer_actor_id_.Binary());
-  msg.set_queue_id(queue_id_.Binary());
+  FillMessageCommon(msg.mutable_common());
  msg.SerializeToString(output);
 }

 std::shared_ptr<CheckMessage> CheckMessage::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
  queue::protobuf::StreamingQueueCheckMsg message;
  message.ParseFromString(inputpb);
-  ActorID src_actor_id = ActorID::FromBinary(message.src_actor_id());
-  ActorID dst_actor_id = ActorID::FromBinary(message.dst_actor_id());
-  ObjectID queue_id = ObjectID::FromBinary(message.queue_id());
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());

  std::shared_ptr<CheckMessage> check_msg =
      std::make_shared<CheckMessage>(src_actor_id, dst_actor_id, queue_id);
@@ -138,24 +136,20 @@ std::shared_ptr<CheckMessage> CheckMessage::FromBytes(uint8_t *bytes) {

 void CheckRspMessage::ToProtobuf(std::string *output) {
  queue::protobuf::StreamingQueueCheckRspMsg msg;
-  msg.set_src_actor_id(actor_id_.Binary());
-  msg.set_dst_actor_id(peer_actor_id_.Binary());
-  msg.set_queue_id(queue_id_.Binary());
+  FillMessageCommon(msg.mutable_common());
  msg.set_err_code(err_code_);
  msg.SerializeToString(output);
 }

 std::shared_ptr<CheckRspMessage> CheckRspMessage::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
  queue::protobuf::StreamingQueueCheckRspMsg message;
  message.ParseFromString(inputpb);
-  ActorID src_actor_id = ActorID::FromBinary(message.src_actor_id());
-  ActorID dst_actor_id = ActorID::FromBinary(message.dst_actor_id());
-  ObjectID queue_id = ObjectID::FromBinary(message.queue_id());
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
  queue::protobuf::StreamingQueueError err_code = message.err_code();

  std::shared_ptr<CheckRspMessage> check_rsp_msg =
@@ -164,6 +158,117 @@ std::shared_ptr<CheckRspMessage> CheckRspMessage::FromBytes(uint8_t *bytes) {
  return check_rsp_msg;
 }

+void PullRequestMessage::ToProtobuf(std::string *output) {
+  queue::protobuf::StreamingQueuePullRequestMsg msg;
+  FillMessageCommon(msg.mutable_common());
+  msg.set_msg_id(msg_id_);
+  msg.SerializeToString(output);
+}
+
+std::shared_ptr<PullRequestMessage> PullRequestMessage::FromBytes(uint8_t *bytes) {
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
+  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
+  queue::protobuf::StreamingQueuePullRequestMsg message;
+  message.ParseFromString(inputpb);
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
+  uint64_t msg_id = message.msg_id();
+  STREAMING_LOG(DEBUG) << "src_actor_id:" << src_actor_id
+                       << " dst_actor_id:" << dst_actor_id << " queue_id:" << queue_id
+                       << " msg_id:" << msg_id;
+
+  std::shared_ptr<PullRequestMessage> pull_msg =
+      std::make_shared<PullRequestMessage>(src_actor_id, dst_actor_id, queue_id, msg_id);
+  return pull_msg;
+}
+
+void PullResponseMessage::ToProtobuf(std::string *output) {
+  queue::protobuf::StreamingQueuePullResponseMsg msg;
+  FillMessageCommon(msg.mutable_common());
+  msg.set_seq_id(seq_id_);
+  msg.set_msg_id(msg_id_);
+  msg.set_err_code(err_code_);
+  msg.set_is_upstream_first_pull(is_upstream_first_pull_);
+  msg.SerializeToString(output);
+}
+
+std::shared_ptr<PullResponseMessage> PullResponseMessage::FromBytes(uint8_t *bytes) {
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
+  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
+  queue::protobuf::StreamingQueuePullResponseMsg message;
+  message.ParseFromString(inputpb);
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
+  uint64_t seq_id = message.seq_id();
+  uint64_t msg_id = message.msg_id();
+  queue::protobuf::StreamingQueueError err_code = message.err_code();
+  bool is_upstream_first_pull = message.is_upstream_first_pull();
+
+  STREAMING_LOG(INFO) << "src_actor_id:" << src_actor_id
+                       << " dst_actor_id:" << dst_actor_id << " queue_id:" << queue_id
+                       << " seq_id: " << seq_id << " msg_id: " << msg_id << " err_code:"
+                       << queue::protobuf::StreamingQueueError_Name(err_code)
+                       << " is_upstream_first_pull: " << is_upstream_first_pull;
+
+  std::shared_ptr<PullResponseMessage> pull_rsp_msg =
+      std::make_shared<PullResponseMessage>(src_actor_id, dst_actor_id, queue_id, seq_id,
+                                            msg_id, err_code, is_upstream_first_pull);
+
+  return pull_rsp_msg;
+}
+
+void ResendDataMessage::ToProtobuf(std::string *output) {
+  queue::protobuf::StreamingQueueResendDataMsg msg;
+  FillMessageCommon(msg.mutable_common());
+  msg.set_first_seq_id(first_seq_id_);
+  msg.set_last_seq_id(last_seq_id_);
+  msg.set_seq_id(seq_id_);
+  msg.set_msg_id_start(msg_id_start_);
+  msg.set_msg_id_end(msg_id_end_);
+  msg.set_length(buffer_->Size());
+  msg.set_raw(raw_);
+  msg.SerializeToString(output);
+}
+
+std::shared_ptr<ResendDataMessage> ResendDataMessage::FromBytes(uint8_t *bytes) {
+  uint64_t *fbs_length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
+  std::string inputpb(reinterpret_cast<char const *>(bytes), *fbs_length);
+  queue::protobuf::StreamingQueueResendDataMsg message;
+  message.ParseFromString(inputpb);
+  ActorID src_actor_id = ActorID::FromBinary(message.common().src_actor_id());
+  ActorID dst_actor_id = ActorID::FromBinary(message.common().dst_actor_id());
+  ObjectID queue_id = ObjectID::FromBinary(message.common().queue_id());
+  uint64_t first_seq_id = message.first_seq_id();
+  uint64_t last_seq_id = message.last_seq_id();
+  uint64_t seq_id = message.seq_id();
+  uint64_t msg_id_start = message.msg_id_start();
+  uint64_t msg_id_end = message.msg_id_end();
+  uint64_t length = message.length();
+  bool raw = message.raw();
+
+  STREAMING_LOG(DEBUG) << "src_actor_id:" << src_actor_id
+                       << " dst_actor_id:" << dst_actor_id
+                       << " first_seq_id:" << first_seq_id << " seq_id:" << seq_id
+                       << " msg_id_start: " << msg_id_start
+                       << " msg_id_end: " << msg_id_end << " last_seq_id:" << last_seq_id
+                       << " queue_id:" << queue_id << " length:" << length;
+
+  bytes += *fbs_length;
+  /// COPY
+  std::shared_ptr<LocalMemoryBuffer> buffer =
+      std::make_shared<LocalMemoryBuffer>(bytes, (size_t)length, true);
+  std::shared_ptr<ResendDataMessage> pull_data_msg = std::make_shared<ResendDataMessage>(
+      src_actor_id, dst_actor_id, queue_id, first_seq_id, seq_id, msg_id_start,
+      msg_id_end, last_seq_id, buffer, raw);
+
+  return pull_data_msg;
+}
+
 void TestInitMessage::ToProtobuf(std::string *output) {
  queue::protobuf::StreamingQueueTestInitMsg msg;
  msg.set_role(role_);
@@ -183,10 +288,8 @@ void TestInitMessage::ToProtobuf(std::string *output) {
 }

 std::shared_ptr<TestInitMessage> TestInitMessage::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
  queue::protobuf::StreamingQueueTestInitMsg message;
  message.ParseFromString(inputpb);
@@ -221,10 +324,8 @@ void TestCheckStatusRspMsg::ToProtobuf(std::string *output) {
 }

 std::shared_ptr<TestCheckStatusRspMsg> TestCheckStatusRspMsg::FromBytes(uint8_t *bytes) {
-  bytes += sizeof(uint32_t) + sizeof(queue::protobuf::StreamingQueueMessageType);
-  uint64_t *length = (uint64_t *)bytes;
-  bytes += sizeof(uint64_t);
-
+  uint64_t *length = (uint64_t *)(bytes + kItemMetaHeaderSize);
+  bytes += kItemHeaderSize;
  std::string inputpb(reinterpret_cast<char const *>(bytes), *length);
  queue::protobuf::StreamingQueueTestCheckStatusRspMsg message;
  message.ParseFromString(inputpb);
@@ -237,4 +338,4 @@ std::shared_ptr<TestCheckStatusRspMsg> TestCheckStatusRspMsg::FromBytes(uint8_t
  return test_check_msg;
 }
 }  // namespace streaming
-}  // namespace ray
+}  // namespace ray