wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-27 11:26:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1226 additions and 7039 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/data_reader.cc
+89 -244
View File
@@ -18,16 +18,15 @@ const uint32_t DataReader::kReadItemTimeout = 1000;
void DataReader::Init(const std::vector<ObjectID> &input_ids,
const std::vector<ChannelCreationParameter> &init_params,
const std::vector<uint64_t> &queue_seq_ids,
const std::vector<uint64_t> &streaming_msg_ids,
std::vector<TransferCreationStatus> &creation_status,
int64_t timer_interval) {
Init(input_ids, init_params, timer_interval);
for (size_t i = 0; i < input_ids.size(); ++i) {
auto &q_id = input_ids[i];
last_message_id_[q_id] = streaming_msg_ids[i];
channel_info_map_[q_id].current_seq_id = queue_seq_ids[i];
channel_info_map_[q_id].current_message_id = streaming_msg_ids[i];
}
InitChannel(creation_status);
}
void DataReader::Init(const std::vector<ObjectID> &input_ids,
@@ -54,23 +53,19 @@ void DataReader::Init(const std::vector<ObjectID> &input_ids,
channel_info.last_queue_item_latency = 0;
channel_info.last_queue_target_diff = 0;
channel_info.get_queue_item_times = 0;
channel_info.resend_notify_timer = 0;
}
reliability_helper_ = ReliabilityHelperFactory::CreateReliabilityHelper(
runtime_context_->GetConfig(), barrier_helper_, nullptr, this);
/// Make the input id location stable.
sort(input_queue_ids_.begin(), input_queue_ids_.end(),
[](const ObjectID &a, const ObjectID &b) { return a.Hash() < b.Hash(); });
std::copy(input_ids.begin(), input_ids.end(), std::back_inserter(unready_queue_ids_));
InitChannel();
}
StreamingStatus DataReader::InitChannel(
std::vector<TransferCreationStatus> &creation_status) {
StreamingStatus DataReader::InitChannel() {
STREAMING_LOG(INFO) << "[Reader] Getting queues. total queue num "
<< input_queue_ids_.size()
<< ", unready queue num=" << unready_queue_ids_.size();
<< input_queue_ids_.size() << ", unready queue num => "
<< unready_queue_ids_.size();
for (const auto &input_channel : unready_queue_ids_) {
auto &channel_info = channel_info_map_[input_channel];
@@ -83,10 +78,8 @@ StreamingStatus DataReader::InitChannel(
channel_map_.emplace(input_channel, channel);
TransferCreationStatus status = channel->CreateTransferChannel();
creation_status.push_back(status);
if (TransferCreationStatus::PullOk != status) {
STREAMING_LOG(ERROR) << "Initialize queue failed, id=" << input_channel
<< ", status=" << static_cast<uint32_t>(status);
STREAMING_LOG(ERROR) << "Initialize queue failed, id => " << input_channel;
}
}
runtime_context_->SetRuntimeStatus(RuntimeStatus::Running);
@@ -94,11 +87,10 @@ StreamingStatus DataReader::InitChannel(
return StreamingStatus::OK;
}
StreamingStatus DataReader::InitChannelMerger(uint32_t timeout_ms) {
StreamingStatus DataReader::InitChannelMerger() {
STREAMING_LOG(INFO) << "[Reader] Initializing queue merger.";
// Init reader merger by given comparator when it's first created.
StreamingReaderMsgPtrComparator comparator(
runtime_context_->GetConfig().GetReliabilityLevel());
StreamingReaderMsgPtrComparator comparator;
if (!reader_merger_) {
reader_merger_.reset(
new PriorityQueue<std::shared_ptr<DataBundle>, StreamingReaderMsgPtrComparator>(
@@ -108,255 +100,106 @@ StreamingStatus DataReader::InitChannelMerger(uint32_t timeout_ms) {
// An old item in merger vector must be evicted before new queue item has been
// pushed.
if (!unready_queue_ids_.empty() && last_fetched_queue_item_) {
STREAMING_LOG(INFO) << "pop old item from=" << last_fetched_queue_item_->from;
RETURN_IF_NOT_OK(StashNextMessageAndPop(last_fetched_queue_item_, timeout_ms))
STREAMING_LOG(INFO) << "pop old item from => " << last_fetched_queue_item_->from;
RETURN_IF_NOT_OK(StashNextMessage(last_fetched_queue_item_))
last_fetched_queue_item_.reset();
}
// Create initial heap for priority queue.
std::vector<ObjectID> unready_queue_ids_stashed;
for (auto &input_queue : unready_queue_ids_) {
std::shared_ptr<DataBundle> msg = std::make_shared<DataBundle>();
auto status = GetMessageFromChannel(channel_info_map_[input_queue], msg, timeout_ms,
timeout_ms);
if (StreamingStatus::OK != status) {
STREAMING_LOG(INFO)
<< "[Reader] initializing merger, get message from channel timeout, "
<< input_queue << ", status => " << static_cast<uint32_t>(status);
unready_queue_ids_stashed.push_back(input_queue);
continue;
}
RETURN_IF_NOT_OK(GetMessageFromChannel(channel_info_map_[input_queue], msg))
channel_info_map_[msg->from].current_seq_id = msg->seq_id;
channel_info_map_[msg->from].current_message_id = msg->meta->GetLastMessageId();
reader_merger_->push(msg);
}
if (unready_queue_ids_stashed.empty()) {
STREAMING_LOG(INFO) << "[Reader] Initializing merger done.";
return StreamingStatus::OK;
} else {
STREAMING_LOG(INFO) << "[Reader] Initializing merger unfinished.";
unready_queue_ids_ = unready_queue_ids_stashed;
return StreamingStatus::GetBundleTimeOut;
}
STREAMING_LOG(INFO) << "[Reader] Initializing merger done.";
return StreamingStatus::OK;
}
StreamingStatus DataReader::GetMessageFromChannel(ConsumerChannelInfo &channel_info,
std::shared_ptr<DataBundle> &message,
uint32_t timeout_ms,
uint32_t wait_time_ms) {
std::shared_ptr<DataBundle> &message) {
auto &qid = channel_info.channel_id;
message->from = qid;
last_read_q_id_ = qid;
bool is_valid_bundle = false;
int64_t start_time = current_sys_time_ms();
STREAMING_LOG(DEBUG) << "GetMessageFromChannel, timeout_ms=" << timeout_ms
<< ", wait_time_ms=" << wait_time_ms;
while (runtime_context_->GetRuntimeStatus() == RuntimeStatus::Running &&
!is_valid_bundle && current_sys_time_ms() - start_time < timeout_ms) {
STREAMING_LOG(DEBUG) << "[Reader] send get request queue seq id=" << qid;
/// In AT_LEAST_ONCE, wait_time_ms is set to 0, means `ConsumeItemFromChannel`
/// will return immediately if no items in queue. At the same time, `timeout_ms` is
/// ignored.
channel_map_[channel_info.channel_id]->ConsumeItemFromChannel(
message->data, message->data_size, wait_time_ms);
STREAMING_LOG(DEBUG) << "[Reader] send get request queue seq id => " << qid;
while (RuntimeStatus::Running == runtime_context_->GetRuntimeStatus() &&
!message->data) {
auto status = channel_map_[channel_info.channel_id]->ConsumeItemFromChannel(
message->seq_id, message->data, message->data_size, kReadItemTimeout);
channel_info.get_queue_item_times++;
if (!message->data) {
RETURN_IF_NOT_OK(reliability_helper_->HandleNoValidItem(channel_info));
} else {
uint64_t current_time = current_sys_time_ms();
channel_info.resend_notify_timer = current_time;
// Note(lingxuan.zlx): To find which channel get an invalid data and
// print channel id for debugging.
STREAMING_CHECK(StreamingMessageBundleMeta::CheckBundleMagicNum(message->data))
<< "Magic number invalid, from channel " << channel_info.channel_id;
message->meta = StreamingMessageBundleMeta::FromBytes(message->data);
is_valid_bundle = true;
if (!runtime_context_->GetConfig().IsAtLeastOnce()) {
// filter message when msg_id doesn't match.
// reader will filter message only when using streaming queue and
// non-at-least-once mode
BundleCheckStatus status = CheckBundle(message);
STREAMING_LOG(DEBUG) << "CheckBundle, result=" << status
<< ", last_msg_id=" << last_message_id_[message->from];
if (status == BundleCheckStatus::BundleToBeSplit) {
SplitBundle(message, last_message_id_[qid]);
}
if (status == BundleCheckStatus::BundleToBeThrown && message->meta->IsBarrier()) {
STREAMING_LOG(WARNING)
<< "Throw barrier, msg_id=" << message->meta->GetLastMessageId();
}
is_valid_bundle = status != BundleCheckStatus::BundleToBeThrown;
}
STREAMING_LOG(DEBUG) << "[Reader] Queue " << qid << " status " << status
<< " get item timeout, resend notify "
<< channel_info.current_seq_id;
// TODO(lingxuan.zlx): notify consumed when it's timeout.
}
}
if (RuntimeStatus::Interrupted == runtime_context_->GetRuntimeStatus()) {
return StreamingStatus::Interrupted;
}
STREAMING_LOG(DEBUG) << "[Reader] recevied queue seq id => " << message->seq_id
<< ", queue id => " << qid;
if (!is_valid_bundle) {
STREAMING_LOG(DEBUG) << "GetMessageFromChannel timeout, qid="
<< channel_info.channel_id;
return StreamingStatus::GetBundleTimeOut;
}
STREAMING_LOG(DEBUG) << "[Reader] received message id="
<< message->meta->GetLastMessageId() << ", queue id=" << qid;
last_message_id_[message->from] = message->meta->GetLastMessageId();
message->from = qid;
message->meta = StreamingMessageBundleMeta::FromBytes(message->data);
return StreamingStatus::OK;
}
BundleCheckStatus DataReader::CheckBundle(const std::shared_ptr<DataBundle> &message) {
uint64_t end_msg_id = message->meta->GetLastMessageId();
uint64_t start_msg_id = message->meta->IsEmptyMsg()
? end_msg_id
: end_msg_id - message->meta->GetMessageListSize() + 1;
uint64_t last_msg_id = last_message_id_[message->from];
// Writer will keep sending bundles when downstream reader failover. After reader
// recovered, it will receive these bundles whoes msg_id is larger than expected.
if (start_msg_id > last_msg_id + 1) {
return BundleCheckStatus::BundleToBeThrown;
}
if (end_msg_id < last_msg_id + 1) {
// Empty message and barrier's msg_id equals to last message, so we shouldn't throw
// them.
return end_msg_id == last_msg_id && !message->meta->IsBundle()
? BundleCheckStatus::OkBundle
: BundleCheckStatus::BundleToBeThrown;
}
// Normal bundles.
if (start_msg_id == last_msg_id + 1) {
return BundleCheckStatus::OkBundle;
}
return BundleCheckStatus::BundleToBeSplit;
}
void DataReader::SplitBundle(std::shared_ptr<DataBundle> &message, uint64_t last_msg_id) {
std::list<StreamingMessagePtr> msg_list;
StreamingMessageBundle::GetMessageListFromRawData(
message->data + kMessageBundleHeaderSize,
message->data_size - kMessageBundleHeaderSize, message->meta->GetMessageListSize(),
msg_list);
uint64_t bundle_size = 0;
for (auto it = msg_list.begin(); it != msg_list.end();) {
if ((*it)->GetMessageId() > last_msg_id) {
bundle_size += (*it)->ClassBytesSize();
it++;
} else {
it = msg_list.erase(it);
}
}
STREAMING_LOG(DEBUG) << "Split message, from_queue_id=" << message->from
<< ", start_msg_id=" << msg_list.front()->GetMessageId()
<< ", end_msg_id=" << msg_list.back()->GetMessageId();
// recreate bundle
auto cut_msg_bundle = std::make_shared<StreamingMessageBundle>(
msg_list, message->meta->GetMessageBundleTs(), msg_list.back()->GetMessageId(),
StreamingMessageBundleType::Bundle, bundle_size);
message->Realloc(cut_msg_bundle->ClassBytesSize());
cut_msg_bundle->ToBytes(message->data);
message->meta = StreamingMessageBundleMeta::FromBytes(message->data);
}
StreamingStatus DataReader::StashNextMessageAndPop(std::shared_ptr<DataBundle> &message,
uint32_t timeout_ms) {
STREAMING_LOG(DEBUG) << "StashNextMessageAndPop, timeout_ms=" << timeout_ms;
// Get the first message.
message = reader_merger_->top();
STREAMING_LOG(DEBUG) << "Messages to be poped=" << *message
<< ", merger size=" << reader_merger_->size();
// Then stash next message from its from queue.
StreamingStatus DataReader::StashNextMessage(std::shared_ptr<DataBundle> &message) {
// Push new message into priority queue and record the channel metrics in
// channel info.
std::shared_ptr<DataBundle> new_msg = std::make_shared<DataBundle>();
auto &channel_info = channel_info_map_[message->from];
RETURN_IF_NOT_OK(GetMessageFromChannel(channel_info, new_msg, timeout_ms, timeout_ms))
new_msg->last_barrier_id = channel_info.barrier_id;
reader_merger_->push(new_msg);
STREAMING_LOG(DEBUG) << "New message pushed=" << *new_msg
<< ", merger size=" << reader_merger_->size();
// Pop message.
reader_merger_->pop();
STREAMING_LOG(DEBUG) << "Message poped, msg=" << *message;
// Record some metrics.
int64_t cur_time = current_time_ms();
RETURN_IF_NOT_OK(GetMessageFromChannel(channel_info, new_msg))
reader_merger_->push(new_msg);
channel_info.last_queue_item_delay =
new_msg->meta->GetMessageBundleTs() - message->meta->GetMessageBundleTs();
channel_info.last_queue_item_latency = current_time_ms() - current_time_ms();
channel_info.last_queue_item_latency = current_time_ms() - cur_time;
return StreamingStatus::OK;
}
StreamingStatus DataReader::GetMergedMessageBundle(std::shared_ptr<DataBundle> &message,
bool &is_valid_break,
uint32_t timeout_ms) {
RETURN_IF_NOT_OK(StashNextMessageAndPop(message, timeout_ms))
auto &offset_info = channel_info_map_[message->from];
uint64_t cur_queue_previous_msg_id = offset_info.current_message_id;
STREAMING_LOG(DEBUG) << "[Reader] [Bundle]" << *message
<< ", cur_queue_previous_msg_id=" << cur_queue_previous_msg_id;
bool &is_valid_break) {
int64_t cur_time = current_time_ms();
if (last_fetched_queue_item_) {
RETURN_IF_NOT_OK(StashNextMessage(last_fetched_queue_item_))
}
message = reader_merger_->top();
last_fetched_queue_item_ = message;
auto &offset_info = channel_info_map_[message->from];
uint64_t cur_queue_previous_msg_id = offset_info.current_message_id;
STREAMING_LOG(DEBUG) << "[Reader] [Bundle] from q_id =>" << message->from << "cur => "
<< cur_queue_previous_msg_id << ", message list size"
<< message->meta->GetMessageListSize() << ", lst message id =>"
<< message->meta->GetLastMessageId() << ", q seq id => "
<< message->seq_id << ", last barrier id => " << message->data_size
<< ", " << message->meta->GetMessageBundleTs();
if (message->meta->IsBundle()) {
last_message_ts_ = cur_time;
is_valid_break = true;
} else if (message->meta->IsBarrier() && BarrierAlign(message)) {
last_message_ts_ = cur_time;
is_valid_break = true;
} else if (timer_interval_ != -1 && cur_time - last_message_ts_ >= timer_interval_ &&
message->meta->IsEmptyMsg()) {
// Sent empty message when reaching timer_interval
} else if (timer_interval_ != -1 && cur_time - last_message_ts_ > timer_interval_) {
// Throw empty message when reaching timer_interval.
last_message_ts_ = cur_time;
is_valid_break = true;
}
offset_info.current_message_id = message->meta->GetLastMessageId();
offset_info.current_seq_id = message->seq_id;
last_bundle_ts_ = message->meta->GetMessageBundleTs();
STREAMING_LOG(DEBUG) << "[Reader] [Bundle] Get merged message bundle=" << *message
<< ", is_valid_break=" << is_valid_break;
last_fetched_queue_item_ = message;
STREAMING_LOG(DEBUG) << "[Reader] [Bundle] message type =>"
<< static_cast<int>(message->meta->GetBundleType())
<< " from id => " << message->from << ", queue seq id =>"
<< message->seq_id << ", message id => "
<< message->meta->GetLastMessageId();
return StreamingStatus::OK;
}
bool DataReader::BarrierAlign(std::shared_ptr<DataBundle> &message) {
// Arrange barrier action when barrier is arriving.
StreamingBarrierHeader barrier_header;
StreamingMessage::GetBarrierIdFromRawData(message->data + kMessageHeaderSize,
&barrier_header);
uint64_t barrier_id = barrier_header.barrier_id;
auto *barrier_align_cnt = &global_barrier_cnt_;
auto &channel_info = channel_info_map_[message->from];
// Target count is input vector size (global barrier).
uint32_t target_count = 0;
channel_info.barrier_id = barrier_header.barrier_id;
target_count = input_queue_ids_.size();
(*barrier_align_cnt)[barrier_id]++;
// The next message checkpoint is changed if this's barrier message.
STREAMING_LOG(INFO) << "[Reader] [Barrier] get barrier, barrier_id=" << barrier_id
<< ", barrier_cnt=" << (*barrier_align_cnt)[barrier_id]
<< ", global barrier id=" << barrier_header.barrier_id
<< ", from q_id=" << message->from << ", barrier type="
<< static_cast<uint32_t>(barrier_header.barrier_type)
<< ", target count=" << target_count;
// Notify invoker the last barrier, so that checkpoint or something related can be
// taken right now.
if ((*barrier_align_cnt)[barrier_id] == target_count) {
// map can't be used in multithread (crash in report timer)
barrier_align_cnt->erase(barrier_id);
STREAMING_LOG(INFO)
<< "[Reader] [Barrier] last barrier received, return barrier. barrier_id = "
<< barrier_id << ", from q_id=" << message->from;
return true;
}
return false;
}
StreamingStatus DataReader::GetBundle(const uint32_t timeout_ms,
std::shared_ptr<DataBundle> &message) {
STREAMING_LOG(DEBUG) << "GetBundle, timeout_ms=" << timeout_ms;
// Notify upstream that last fetched item has been consumed.
if (last_fetched_queue_item_) {
NotifyConsumed(last_fetched_queue_item_);
@@ -379,25 +222,28 @@ StreamingStatus DataReader::GetBundle(const uint32_t timeout_ms,
return StreamingStatus::GetBundleTimeOut;
}
if (!unready_queue_ids_.empty()) {
std::vector<TransferCreationStatus> creation_status;
StreamingStatus status = InitChannel(creation_status);
StreamingStatus status = InitChannel();
switch (status) {
case StreamingStatus::InitQueueFailed:
break;
case StreamingStatus::WaitQueueTimeOut:
STREAMING_LOG(ERROR)
<< "Wait upstream queue timeout, maybe some actors in deadlock";
break;
default:
STREAMING_LOG(INFO) << "Init reader queue in GetBundle";
}
if (StreamingStatus::OK != status) {
return status;
}
RETURN_IF_NOT_OK(InitChannelMerger(timeout_ms))
RETURN_IF_NOT_OK(InitChannelMerger())
unready_queue_ids_.clear();
auto &merge_vec = reader_merger_->getRawVector();
for (auto &bundle : merge_vec) {
STREAMING_LOG(INFO) << "merger vector item=" << bundle->from;
STREAMING_LOG(INFO) << "merger vector item => " << bundle->from;
}
}
RETURN_IF_NOT_OK(GetMergedMessageBundle(message, is_valid_break, timeout_ms));
RETURN_IF_NOT_OK(GetMergedMessageBundle(message, is_valid_break));
if (!is_valid_break) {
empty_bundle_cnt++;
NotifyConsumed(message);
@@ -415,12 +261,16 @@ void DataReader::GetOffsetInfo(
offset_map = &channel_info_map_;
for (auto &offset_info : channel_info_map_) {
STREAMING_LOG(INFO) << "[Reader] [GetOffsetInfo], q id " << offset_info.first
<< ", message id=" << offset_info.second.current_message_id;
<< ", seq id => " << offset_info.second.current_seq_id
<< ", message id => " << offset_info.second.current_message_id;
}
}
void DataReader::NotifyConsumedItem(ConsumerChannelInfo &channel_info, uint64_t offset) {
channel_map_[channel_info.channel_id]->NotifyChannelConsumed(offset);
if (offset == channel_info.queue_info.last_seq_id) {
STREAMING_LOG(DEBUG) << "notify seq id equal to last seq id => " << offset;
}
}
DataReader::DataReader(std::shared_ptr<RuntimeContext> &runtime_context)
@@ -436,42 +286,37 @@ void DataReader::NotifyConsumed(std::shared_ptr<DataBundle> &message) {
auto &channel_info = channel_info_map_[message->from];
auto &queue_info = channel_info.queue_info;
channel_info.notify_cnt++;
if (queue_info.target_message_id <= message->meta->GetLastMessageId()) {
NotifyConsumedItem(channel_info, message->meta->GetLastMessageId());
if (queue_info.target_seq_id <= message->seq_id) {
NotifyConsumedItem(channel_info, message->seq_id);
channel_map_[channel_info.channel_id]->RefreshChannelInfo();
if (queue_info.last_message_id != QUEUE_INVALID_SEQ_ID) {
uint64_t original_target_message_id = queue_info.target_message_id;
queue_info.target_message_id =
std::min(queue_info.last_message_id,
message->meta->GetLastMessageId() +
runtime_context_->GetConfig().GetReaderConsumedStep());
if (queue_info.last_seq_id != QUEUE_INVALID_SEQ_ID) {
uint64_t original_target_seq_id = queue_info.target_seq_id;
queue_info.target_seq_id = std::min(
queue_info.last_seq_id,
message->seq_id + runtime_context_->GetConfig().GetReaderConsumedStep());
channel_info.last_queue_target_diff =
queue_info.target_message_id - original_target_message_id;
queue_info.target_seq_id - original_target_seq_id;
} else {
STREAMING_LOG(WARNING) << "[Reader] [QueueInfo] channel id " << message->from
<< ", last message id " << queue_info.last_message_id;
<< ", last seq id " << queue_info.last_seq_id;
}
STREAMING_LOG(DEBUG) << "[Reader] [Consumed] Trigger notify consumed"
<< ", channel id=" << message->from
<< ", last message id=" << queue_info.last_message_id
<< ", target message id=" << queue_info.target_message_id
<< ", consumed message id=" << message->meta->GetLastMessageId()
<< ", bundle type="
<< ", channel id => " << message->from << ", last seq id => "
<< queue_info.last_seq_id << ", target seq id => "
<< queue_info.target_seq_id << ", consumed seq id => "
<< message->seq_id << ", last message id => "
<< message->meta->GetLastMessageId() << ", bundle type => "
<< static_cast<uint32_t>(message->meta->GetBundleType())
<< ", last message bundle ts="
<< ", last message bundle ts => "
<< message->meta->GetMessageBundleTs();
}
}
bool StreamingReaderMsgPtrComparator::operator()(const std::shared_ptr<DataBundle> &a,
const std::shared_ptr<DataBundle> &b) {
if (comp_strategy == ReliabilityLevel::EXACTLY_ONCE) {
if (a->last_barrier_id != b->last_barrier_id)
return a->last_barrier_id > b->last_barrier_id;
}
STREAMING_CHECK(a->meta);
// We proposed fixed id sequnce for stability of message in sorting.
// We use hash value of id for stability of message in sorting.
if (a->meta->GetMessageBundleTs() == b->meta->GetMessageBundleTs()) {
return a->from.Hash() > b->from.Hash();
}