wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-14 11:17:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming]Streaming queue support failover (#8161)

Browse Source
This commit is contained in:
wanxing
2020-08-25 14:19:45 +08:00
committed by GitHub
parent 5a787a8253
commit e816e3aefb
16 changed files with 1049 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/queue/queue_item.h
+27 -10
View File
@@ -15,6 +15,7 @@ namespace streaming {
using ray::ObjectID;
const uint64_t QUEUE_INVALID_SEQ_ID = std::numeric_limits<uint64_t>::max();
const uint64_t QUEUE_INITIAL_SEQ_ID = 1;
/// QueueItem is the element stored in `Queue`. Actually, when DataWriter pushes a message
/// bundle into a queue, the bundle is packed into one QueueItem, so a one-to-one
@@ -31,24 +32,28 @@ class QueueItem {
/// \param[in] timestamp the time when this QueueItem created.
/// \param[in] raw whether the data content is raw bytes, only used in some tests.
QueueItem(uint64_t seq_id, uint8_t *data, uint32_t data_size, uint64_t timestamp,
bool raw = false)
: seq_id_(seq_id),
uint64_t msg_id_start, uint64_t msg_id_end, bool raw = false)
: seq_id_(seq_id), msg_id_start_(msg_id_start), msg_id_end_(msg_id_end),
timestamp_(timestamp),
raw_(raw),
/*COPY*/ buffer_(std::make_shared<LocalMemoryBuffer>(data, data_size, true)) {}
QueueItem(uint64_t seq_id, std::shared_ptr<LocalMemoryBuffer> buffer,
uint64_t timestamp, bool raw = false)
: seq_id_(seq_id), timestamp_(timestamp), raw_(raw), buffer_(buffer) {}
uint64_t timestamp, uint64_t msg_id_start, uint64_t msg_id_end, bool raw = false)
: seq_id_(seq_id), msg_id_start_(msg_id_start), msg_id_end_(msg_id_end), timestamp_(timestamp), raw_(raw), buffer_(buffer) {}
QueueItem(std::shared_ptr<DataMessage> data_msg)
: seq_id_(data_msg->SeqId()),
msg_id_start_(data_msg->MsgIdStart()),
msg_id_end_(data_msg->MsgIdEnd()),
raw_(data_msg->IsRaw()),
buffer_(data_msg->Buffer()) {}
QueueItem(const QueueItem &&item) {
buffer_ = item.buffer_;
seq_id_ = item.seq_id_;
msg_id_start_ = item.msg_id_start_;
msg_id_end_ = item.msg_id_end_;
timestamp_ = item.timestamp_;
raw_ = item.raw_;
}
@@ -56,6 +61,8 @@ class QueueItem {
QueueItem(const QueueItem &item) {
buffer_ = item.buffer_;
seq_id_ = item.seq_id_;
msg_id_start_ = item.msg_id_start_;
msg_id_end_ = item.msg_id_end_;
timestamp_ = item.timestamp_;
raw_ = item.raw_;
}
@@ -63,6 +70,8 @@ class QueueItem {
QueueItem &operator=(const QueueItem &item) {
buffer_ = item.buffer_;
seq_id_ = item.seq_id_;
msg_id_start_ = item.msg_id_start_;
msg_id_end_ = item.msg_id_end_;
timestamp_ = item.timestamp_;
raw_ = item.raw_;
return *this;
@@ -70,11 +79,16 @@ class QueueItem {
virtual ~QueueItem() = default;
uint64_t SeqId() { return seq_id_; }
bool IsRaw() { return raw_; }
uint64_t TimeStamp() { return timestamp_; }
size_t DataSize() { return buffer_->Size(); }
std::shared_ptr<LocalMemoryBuffer> Buffer() { return buffer_; }
inline uint64_t SeqId() { return seq_id_; }
inline uint64_t MsgIdStart() { return msg_id_start_; }
inline uint64_t MsgIdEnd() { return msg_id_end_; }
inline bool InItem(uint64_t msg_id) {
return msg_id >= msg_id_start_ && msg_id <= msg_id_end_;
}
inline bool IsRaw() { return raw_; }
inline uint64_t TimeStamp() { return timestamp_; }
inline size_t DataSize() { return buffer_->Size(); }
inline std::shared_ptr<LocalMemoryBuffer> Buffer() { return buffer_; }
/// Get max message id in this item.
/// \return max message id.
@@ -88,6 +102,8 @@ class QueueItem {
protected:
uint64_t seq_id_;
uint64_t msg_id_start_;
uint64_t msg_id_end_;
uint64_t timestamp_;
bool raw_;
@@ -96,7 +112,8 @@ class QueueItem {
class InvalidQueueItem : public QueueItem {
public:
InvalidQueueItem() : QueueItem(QUEUE_INVALID_SEQ_ID, data_, 1, 0) {}
InvalidQueueItem() : QueueItem(QUEUE_INVALID_SEQ_ID, data_, 1, 0, QUEUE_INVALID_SEQ_ID,
QUEUE_INVALID_SEQ_ID) {}
private:
uint8_t data_[1];