wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-30 13:47:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Support streaming flow control (#7152)

Browse Source 
* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/data_reader.cc
+34 -4
View File
@@ -199,10 +199,9 @@ StreamingStatus DataReader::GetMergedMessageBundle(std::shared_ptr<DataBundle> &
StreamingStatus DataReader::GetBundle(const uint32_t timeout_ms,
std::shared_ptr<DataBundle> &message) {
// Notify consumed every item in this mode.
// Notify upstream that last fetched item has been consumed.
if (last_fetched_queue_item_) {
NotifyConsumedItem(channel_info_map_[last_fetched_queue_item_->from],
last_fetched_queue_item_->seq_id);
NotifyConsumed(last_fetched_queue_item_);
}
/// DataBundle will be returned to the upper layer in the following cases:
@@ -246,7 +245,7 @@ StreamingStatus DataReader::GetBundle(const uint32_t timeout_ms,
RETURN_IF_NOT_OK(GetMergedMessageBundle(message, is_valid_break));
if (!is_valid_break) {
empty_bundle_cnt++;
NotifyConsumedItem(channel_info_map_[message->from], message->seq_id);
NotifyConsumed(message);
}
}
last_message_latency_ += current_time_ms() - start_time;
@@ -282,6 +281,37 @@ void DataReader::Stop() {
runtime_context_->SetRuntimeStatus(RuntimeStatus::Interrupted);
}
void DataReader::NotifyConsumed(std::shared_ptr<DataBundle> &message) {
auto &channel_info = channel_info_map_[message->from];
auto &queue_info = channel_info.queue_info;
channel_info.notify_cnt++;
if (queue_info.target_seq_id <= message->seq_id) {
NotifyConsumedItem(channel_info, message->seq_id);
channel_map_[channel_info.channel_id]->RefreshChannelInfo();
if (queue_info.last_seq_id != QUEUE_INVALID_SEQ_ID) {
uint64_t original_target_seq_id = queue_info.target_seq_id;
queue_info.target_seq_id = std::min(
queue_info.last_seq_id,
message->seq_id + runtime_context_->GetConfig().GetReaderConsumedStep());
channel_info.last_queue_target_diff =
queue_info.target_seq_id - original_target_seq_id;
} else {
STREAMING_LOG(WARNING) << "[Reader] [QueueInfo] channel id " << message->from
<< ", last seq id " << queue_info.last_seq_id;
}
STREAMING_LOG(DEBUG) << "[Reader] [Consumed] Trigger notify consumed"
<< ", channel id => " << message->from << ", last seq id => "
<< queue_info.last_seq_id << ", target seq id => "
<< queue_info.target_seq_id << ", consumed seq id => "
<< message->seq_id << ", last message id => "
<< message->meta->GetLastMessageId() << ", bundle type => "
<< static_cast<uint32_t>(message->meta->GetBundleType())
<< ", last message bundle ts => "
<< message->meta->GetMessageBundleTs();
}
}
bool StreamingReaderMsgPtrComparator::operator()(const std::shared_ptr<DataBundle> &a,
const std::shared_ptr<DataBundle> &b) {
STREAMING_CHECK(a->meta);