wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-11 12:43:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Support streaming flow control (#7152)

Browse Source 
* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
co-authored by wanxing
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/channel.h
+8 -6
View File
@@ -51,11 +51,13 @@ struct ConsumerChannelInfo {
StreamingQueueInfo queue_info;
uint64_t last_queue_item_delay;
uint64_t last_queue_item_latency;
uint64_t last_queue_target_diff;
uint64_t get_queue_item_times;
uint64_t last_queue_item_delay = 0;
uint64_t last_queue_item_latency = 0;
uint64_t last_queue_target_diff = 0;
uint64_t get_queue_item_times = 0;
ActorID actor_id;
// Total count of notify request.
uint64_t notify_cnt = 0;
};
/// Two types of channel are presented:
@@ -162,7 +164,7 @@ class MockProducer : public ProducerChannel {
return StreamingStatus::OK;
}
StreamingStatus RefreshChannelInfo() override { return StreamingStatus::OK; }
StreamingStatus RefreshChannelInfo() override;
StreamingStatus ProduceItemToChannel(uint8_t *data, uint32_t data_size) override;
@@ -182,7 +184,7 @@ class MockConsumer : public ConsumerChannel {
uint64_t checkpoint_offset) override {
return StreamingStatus::OK;
}
StreamingStatus RefreshChannelInfo() override { return StreamingStatus::OK; }
StreamingStatus RefreshChannelInfo() override;
StreamingStatus ConsumeItemFromChannel(uint64_t &offset_id, uint8_t *&data,
uint32_t &data_size, uint32_t timeout) override;
StreamingStatus NotifyChannelConsumed(uint64_t offset_id) override;