wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-21 12:50:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1226 additions and 7039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/queue/message.cc
+1 -1
View File
@@ -90,7 +90,7 @@ std::shared_ptr<DataMessage> DataMessage::FromBytes(uint8_t *bytes) {
void NotificationMessage::ToProtobuf(std::string *output) {
queue::protobuf::StreamingQueueNotificationMsg msg;
FillMessageCommon(msg.mutable_common());
msg.set_seq_id(msg_id_);
msg.set_seq_id(seq_id_);
msg.SerializeToString(output);
}
streaming/src/queue/message.h
+4 -4
View File
@@ -102,19 +102,19 @@ class DataMessage : public Message {
class NotificationMessage : public Message {
public:
NotificationMessage(const ActorID &actor_id, const ActorID &peer_actor_id,
const ObjectID &queue_id, uint64_t msg_id)
: Message(actor_id, peer_actor_id, queue_id), msg_id_(msg_id) {}
const ObjectID &queue_id, uint64_t seq_id)
: Message(actor_id, peer_actor_id, queue_id), seq_id_(seq_id) {}
virtual ~NotificationMessage() {}
static std::shared_ptr<NotificationMessage> FromBytes(uint8_t *bytes);
virtual void ToProtobuf(std::string *output);
inline uint64_t MsgId() { return msg_id_; }
inline uint64_t SeqId() { return seq_id_; }
inline queue::protobuf::StreamingQueueMessageType Type() { return type_; }
private:
uint64_t msg_id_;
uint64_t seq_id_;
const queue::protobuf::StreamingQueueMessageType type_ =
queue::protobuf::StreamingQueueMessageType::StreamingQueueNotificationMsgType;
};
streaming/src/queue/queue.cc
+25 -35
View File
@@ -101,8 +101,9 @@ size_t Queue::PendingCount() {
return begin->SeqId() - end->SeqId() + 1;
}
Status WriterQueue::Push(uint8_t *buffer, uint32_t buffer_size, uint64_t timestamp,
uint64_t msg_id_start, uint64_t msg_id_end, bool raw) {
Status WriterQueue::Push(uint64_t seq_id, uint8_t *buffer, uint32_t buffer_size,
uint64_t timestamp, uint64_t msg_id_start, uint64_t msg_id_end,
bool raw) {
if (IsPendingFull(buffer_size)) {
return Status::OutOfMemory("Queue Push OutOfMemory");
}
@@ -112,9 +113,9 @@ Status WriterQueue::Push(uint8_t *buffer, uint32_t buffer_size, uint64_t timesta
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
QueueItem item(seq_id_, buffer, buffer_size, timestamp, msg_id_start, msg_id_end, raw);
QueueItem item(seq_id, buffer, buffer_size, timestamp, msg_id_start, msg_id_end, raw);
Queue::Push(item);
STREAMING_LOG(DEBUG) << "WriterQueue::Push seq_id: " << seq_id_;
STREAMING_LOG(DEBUG) << "WriterQueue::Push seq_id_: " << seq_id_;
seq_id_++;
return Status::OK();
}
@@ -131,41 +132,33 @@ void WriterQueue::Send() {
}
Status WriterQueue::TryEvictItems() {
STREAMING_LOG(INFO) << "TryEvictItems";
QueueItem item = FrontProcessed();
STREAMING_LOG(DEBUG) << "TryEvictItems queue_id: " << queue_id_ << " first_item: ("
<< item.MsgIdStart() << "," << item.MsgIdEnd() << ")"
<< " min_consumed_msg_id_: " << min_consumed_msg_id_
<< " eviction_limit_: " << eviction_limit_
<< " max_data_size_: " << max_data_size_
<< " data_size_sent_: " << data_size_sent_
<< " data_size_: " << data_size_;
if (min_consumed_msg_id_ == QUEUE_INVALID_SEQ_ID ||
min_consumed_msg_id_ < item.MsgIdEnd()) {
uint64_t first_seq_id = item.SeqId();
STREAMING_LOG(INFO) << "TryEvictItems first_seq_id: " << first_seq_id
<< " min_consumed_id_: " << min_consumed_id_
<< " eviction_limit_: " << eviction_limit_;
if (min_consumed_id_ == QUEUE_INVALID_SEQ_ID || first_seq_id > min_consumed_id_) {
return Status::OutOfMemory("The queue is full and some reader doesn't consume");
}
if (eviction_limit_ == QUEUE_INVALID_SEQ_ID || eviction_limit_ < item.MsgIdEnd()) {
if (eviction_limit_ == QUEUE_INVALID_SEQ_ID || first_seq_id > eviction_limit_) {
return Status::OutOfMemory("The queue is full and eviction limit block evict");
}
uint64_t evict_target_msg_id = std::min(min_consumed_msg_id_, eviction_limit_);
uint64_t evict_target_seq_id = std::min(min_consumed_id_, eviction_limit_);
int count = 0;
while (item.MsgIdEnd() <= evict_target_msg_id) {
while (item.SeqId() <= evict_target_seq_id) {
PopProcessed();
STREAMING_LOG(INFO) << "TryEvictItems directly " << item.MsgIdEnd();
STREAMING_LOG(INFO) << "TryEvictItems directly " << item.SeqId();
item = FrontProcessed();
count++;
}
STREAMING_LOG(DEBUG) << count << " items evicted, current item: (" << item.MsgIdStart()
<< "," << item.MsgIdEnd() << ")";
return Status::OK();
}
void WriterQueue::OnNotify(std::shared_ptr<NotificationMessage> notify_msg) {
STREAMING_LOG(INFO) << "OnNotify target msg_id: " << notify_msg->MsgId();
min_consumed_msg_id_ = notify_msg->MsgId();
STREAMING_LOG(INFO) << "OnNotify target seq_id: " << notify_msg->SeqId();
min_consumed_id_ = notify_msg->SeqId();
}
void WriterQueue::ResendItem(QueueItem &item, uint64_t first_seq_id,
@@ -280,22 +273,22 @@ void WriterQueue::OnPull(
});
}
void ReaderQueue::OnConsumed(uint64_t msg_id) {
STREAMING_LOG(INFO) << "OnConsumed: " << msg_id;
void ReaderQueue::OnConsumed(uint64_t seq_id) {
STREAMING_LOG(INFO) << "OnConsumed: " << seq_id;
QueueItem item = FrontProcessed();
while (item.MsgIdEnd() <= msg_id) {
while (item.SeqId() <= seq_id) {
PopProcessed();
item = FrontProcessed();
}
Notify(msg_id);
Notify(seq_id);
}
void ReaderQueue::Notify(uint64_t msg_id) {
void ReaderQueue::Notify(uint64_t seq_id) {
std::vector<TaskArg> task_args;
CreateNotifyTask(msg_id, task_args);
CreateNotifyTask(seq_id, task_args);
// SubmitActorTask
NotificationMessage msg(actor_id_, peer_actor_id_, queue_id_, msg_id);
NotificationMessage msg(actor_id_, peer_actor_id_, queue_id_, seq_id);
std::unique_ptr<LocalMemoryBuffer> buffer = msg.ToBytes();
transport_->Send(std::move(buffer));
@@ -305,10 +298,7 @@ void ReaderQueue::CreateNotifyTask(uint64_t seq_id, std::vector<TaskArg> &task_a
void ReaderQueue::OnData(QueueItem &item) {
last_recv_seq_id_ = item.SeqId();
last_recv_msg_id_ = item.MsgIdEnd();
STREAMING_LOG(DEBUG) << "ReaderQueue::OnData queue_id: " << queue_id_
<< " seq_id: " << last_recv_seq_id_ << " msg_id: ("
<< item.MsgIdStart() << "," << item.MsgIdEnd() << ")";
STREAMING_LOG(DEBUG) << "ReaderQueue::OnData seq_id: " << last_recv_seq_id_;
Push(item);
}
streaming/src/queue/queue.h
+20 -19
View File
@@ -94,10 +94,10 @@ class Queue {
inline size_t Count() { return buffer_queue_.size(); }
/// Return item count in pending state.
inline size_t PendingCount();
size_t PendingCount();
/// Return item count in processed state.
inline size_t ProcessedCount();
size_t ProcessedCount();
inline ActorID GetActorID() { return actor_id_; }
inline ActorID GetPeerActorID() { return peer_actor_id_; }
@@ -135,7 +135,7 @@ class WriterQueue : public Queue {
peer_actor_id_(peer_actor_id),
seq_id_(QUEUE_INITIAL_SEQ_ID),
eviction_limit_(QUEUE_INVALID_SEQ_ID),
min_consumed_msg_id_(QUEUE_INVALID_SEQ_ID),
min_consumed_id_(QUEUE_INVALID_SEQ_ID),
peer_last_msg_id_(0),
peer_last_seq_id_(QUEUE_INVALID_SEQ_ID),
transport_(transport),
@@ -143,14 +143,12 @@ class WriterQueue : public Queue {
is_upstream_first_pull_(true) {}
/// Push a continuous buffer into queue, the buffer consists of some messages packed by
/// DataWriter.
/// \param data, the buffer address
/// \param data_size, buffer size
/// \param timestamp, the timestamp when the buffer pushed in
/// \param msg_id_start, the message id of the first message in the buffer
/// \param msg_id_end, the message id of the last message in the buffer
/// \param raw, whether this buffer is raw data, be True only in test
Status Push(uint8_t *buffer, uint32_t buffer_size, uint64_t timestamp,
/// DataWriter. \param data, the buffer address \param data_size, buffer size \param
/// timestamp, the timestamp when the buffer pushed in \param msg_id_start, the message
/// id of the first message in the buffer \param msg_id_end, the message id of the last
/// message in the buffer \param raw, whether this buffer is raw data, be True only in
/// test
Status Push(uint64_t seq_id, uint8_t *buffer, uint32_t buffer_size, uint64_t timestamp,
uint64_t msg_id_start, uint64_t msg_id_end, bool raw = false);
/// Callback function, will be called when downstream queue notifies
@@ -169,14 +167,16 @@ class WriterQueue : public Queue {
void Send();
/// Called when user pushs item into queue. The count of items
/// can be evicted, determined by eviction_limit_ and min_consumed_msg_id_.
/// can be evicted, determined by eviction_limit_ and min_consumed_id_.
Status TryEvictItems();
void SetQueueEvictionLimit(uint64_t msg_id) { eviction_limit_ = msg_id; }
void SetQueueEvictionLimit(uint64_t eviction_limit) {
eviction_limit_ = eviction_limit;
}
uint64_t EvictionLimit() { return eviction_limit_; }
uint64_t GetMinConsumedMsgID() { return min_consumed_msg_id_; }
uint64_t GetMinConsumedSeqID() { return min_consumed_id_; }
void SetPeerLastIds(uint64_t msg_id, uint64_t seq_id) {
peer_last_msg_id_ = msg_id;
@@ -215,7 +215,7 @@ class WriterQueue : public Queue {
ActorID peer_actor_id_;
uint64_t seq_id_;
uint64_t eviction_limit_;
uint64_t min_consumed_msg_id_;
uint64_t min_consumed_id_;
uint64_t peer_last_msg_id_;
uint64_t peer_last_seq_id_;
std::shared_ptr<Transport> transport_;
@@ -238,8 +238,8 @@ class ReaderQueue : public Queue {
transport),
actor_id_(actor_id),
peer_actor_id_(peer_actor_id),
min_consumed_id_(QUEUE_INVALID_SEQ_ID),
last_recv_seq_id_(QUEUE_INVALID_SEQ_ID),
last_recv_msg_id_(QUEUE_INVALID_SEQ_ID),
transport_(transport) {}
/// Delete processed items whose seq id <= seq_id,
@@ -252,8 +252,9 @@ class ReaderQueue : public Queue {
/// NOTE: this callback function is called in queue thread.
void OnResendData(std::shared_ptr<ResendDataMessage> msg);
inline uint64_t GetLastRecvSeqId() { return last_recv_seq_id_; }
inline uint64_t GetLastRecvMsgId() { return last_recv_msg_id_; }
uint64_t GetMinConsumedSeqID() { return min_consumed_id_; }
uint64_t GetLastRecvSeqId() { return last_recv_seq_id_; }
private:
void Notify(uint64_t seq_id);
@@ -262,8 +263,8 @@ class ReaderQueue : public Queue {
private:
ActorID actor_id_;
ActorID peer_actor_id_;
uint64_t min_consumed_id_;
uint64_t last_recv_seq_id_;
uint64_t last_recv_msg_id_;
std::shared_ptr<PromiseWrapper> promise_for_pull_;
std::shared_ptr<Transport> transport_;
};
streaming/src/queue/queue_handler.cc
+1 -1
View File
@@ -260,7 +260,7 @@ void UpstreamQueueMessageHandler::OnNotify(
<< queue::protobuf::StreamingQueueMessageType_Name(
notify_msg->Type())
<< ", maybe queue has been destroyed, ignore it."
<< " msg id: " << notify_msg->MsgId();
<< " seq id: " << notify_msg->SeqId();
return;
}
queue->OnNotify(notify_msg);
streaming/src/queue/queue_item.h
-1
View File
@@ -24,7 +24,6 @@ const uint64_t QUEUE_INITIAL_SEQ_ID = 1;
/// LocalMemoryBuffer shared_ptr, which will be sent out by Transport.
class QueueItem {
public:
QueueItem() = default;
/// Construct a QueueItem object.
/// \param[in] seq_id the sequential id assigned by DataWriter for a message bundle and
/// QueueItem.
streaming/src/queue/transport.cc
+1 -1
View File
@@ -36,7 +36,7 @@ void Transport::SendInternal(std::shared_ptr<LocalMemoryBuffer> buffer,
}
void Transport::Send(std::shared_ptr<LocalMemoryBuffer> buffer) {
STREAMING_LOG(DEBUG) << "Transport::Send buffer size: " << buffer->Size();
STREAMING_LOG(INFO) << "Transport::Send buffer size: " << buffer->Size();
std::vector<ObjectID> return_ids;
SendInternal(std::move(buffer), async_func_, TASK_OPTION_RETURN_NUM_0, return_ids);
}