wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-11 20:43:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1227 additions and 7040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/queue/queue.h
+20 -19
View File
@@ -94,10 +94,10 @@ class Queue {
inline size_t Count() { return buffer_queue_.size(); }
/// Return item count in pending state.
inline size_t PendingCount();
size_t PendingCount();
/// Return item count in processed state.
inline size_t ProcessedCount();
size_t ProcessedCount();
inline ActorID GetActorID() { return actor_id_; }
inline ActorID GetPeerActorID() { return peer_actor_id_; }
@@ -135,7 +135,7 @@ class WriterQueue : public Queue {
peer_actor_id_(peer_actor_id),
seq_id_(QUEUE_INITIAL_SEQ_ID),
eviction_limit_(QUEUE_INVALID_SEQ_ID),
min_consumed_msg_id_(QUEUE_INVALID_SEQ_ID),
min_consumed_id_(QUEUE_INVALID_SEQ_ID),
peer_last_msg_id_(0),
peer_last_seq_id_(QUEUE_INVALID_SEQ_ID),
transport_(transport),
@@ -143,14 +143,12 @@ class WriterQueue : public Queue {
is_upstream_first_pull_(true) {}
/// Push a continuous buffer into queue, the buffer consists of some messages packed by
/// DataWriter.
/// \param data, the buffer address
/// \param data_size, buffer size
/// \param timestamp, the timestamp when the buffer pushed in
/// \param msg_id_start, the message id of the first message in the buffer
/// \param msg_id_end, the message id of the last message in the buffer
/// \param raw, whether this buffer is raw data, be True only in test
Status Push(uint8_t *buffer, uint32_t buffer_size, uint64_t timestamp,
/// DataWriter. \param data, the buffer address \param data_size, buffer size \param
/// timestamp, the timestamp when the buffer pushed in \param msg_id_start, the message
/// id of the first message in the buffer \param msg_id_end, the message id of the last
/// message in the buffer \param raw, whether this buffer is raw data, be True only in
/// test
Status Push(uint64_t seq_id, uint8_t *buffer, uint32_t buffer_size, uint64_t timestamp,
uint64_t msg_id_start, uint64_t msg_id_end, bool raw = false);
/// Callback function, will be called when downstream queue notifies
@@ -169,14 +167,16 @@ class WriterQueue : public Queue {
void Send();
/// Called when user pushs item into queue. The count of items
/// can be evicted, determined by eviction_limit_ and min_consumed_msg_id_.
/// can be evicted, determined by eviction_limit_ and min_consumed_id_.
Status TryEvictItems();
void SetQueueEvictionLimit(uint64_t msg_id) { eviction_limit_ = msg_id; }
void SetQueueEvictionLimit(uint64_t eviction_limit) {
eviction_limit_ = eviction_limit;
}
uint64_t EvictionLimit() { return eviction_limit_; }
uint64_t GetMinConsumedMsgID() { return min_consumed_msg_id_; }
uint64_t GetMinConsumedSeqID() { return min_consumed_id_; }
void SetPeerLastIds(uint64_t msg_id, uint64_t seq_id) {
peer_last_msg_id_ = msg_id;
@@ -215,7 +215,7 @@ class WriterQueue : public Queue {
ActorID peer_actor_id_;
uint64_t seq_id_;
uint64_t eviction_limit_;
uint64_t min_consumed_msg_id_;
uint64_t min_consumed_id_;
uint64_t peer_last_msg_id_;
uint64_t peer_last_seq_id_;
std::shared_ptr<Transport> transport_;
@@ -238,8 +238,8 @@ class ReaderQueue : public Queue {
transport),
actor_id_(actor_id),
peer_actor_id_(peer_actor_id),
min_consumed_id_(QUEUE_INVALID_SEQ_ID),
last_recv_seq_id_(QUEUE_INVALID_SEQ_ID),
last_recv_msg_id_(QUEUE_INVALID_SEQ_ID),
transport_(transport) {}
/// Delete processed items whose seq id <= seq_id,
@@ -252,8 +252,9 @@ class ReaderQueue : public Queue {
/// NOTE: this callback function is called in queue thread.
void OnResendData(std::shared_ptr<ResendDataMessage> msg);
inline uint64_t GetLastRecvSeqId() { return last_recv_seq_id_; }
inline uint64_t GetLastRecvMsgId() { return last_recv_msg_id_; }
uint64_t GetMinConsumedSeqID() { return min_consumed_id_; }
uint64_t GetLastRecvSeqId() { return last_recv_seq_id_; }
private:
void Notify(uint64_t seq_id);
@@ -262,8 +263,8 @@ class ReaderQueue : public Queue {
private:
ActorID actor_id_;
ActorID peer_actor_id_;
uint64_t min_consumed_id_;
uint64_t last_recv_seq_id_;
uint64_t last_recv_msg_id_;
std::shared_ptr<PromiseWrapper> promise_for_pull_;
std::shared_ptr<Transport> transport_;
};