[Streaming] Support streaming flow control (#7152)

* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
co-authored by wanxing
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
+10
View File
@@ -11,6 +11,12 @@ enum OperatorType {
SINK = 3;
}
enum FlowControlType {
UNKNOWN_FLOW_CONTROL_TYPE = 0;
UnconsumedSeqFlowControl = 1;
NoFlowControl = 2;
}
// all string in this message is ASCII string
message StreamingConfig {
string job_name = 1;
@@ -20,4 +26,8 @@ message StreamingConfig {
OperatorType role = 5;
uint32 ring_buffer_capacity = 6;
uint32 empty_message_interval = 7;
FlowControlType flow_control_type = 8;
uint32 writer_consumed_step = 9;
uint32 reader_consumed_step = 10;
uint32 event_driven_flow_control_interval = 11;
}