wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-25 13:30:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Support streaming flow control (#7152)

Browse Source 
* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
co-authored by wanxing
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/config/streaming_config.cc
+15
View File
@@ -37,6 +37,21 @@ void StreamingConfig::FromProto(const uint8_t *data, uint32_t size) {
if (config.empty_message_interval() != 0) {
SetEmptyMessageTimeInterval(config.empty_message_interval());
}
if (config.flow_control_type() != proto::FlowControlType::UNKNOWN_FLOW_CONTROL_TYPE) {
SetFlowControlType(config.flow_control_type());
}
if (config.writer_consumed_step() != 0) {
SetWriterConsumedStep(config.writer_consumed_step());
}
if (config.reader_consumed_step() != 0) {
SetReaderConsumedStep(config.reader_consumed_step());
}
if (config.event_driven_flow_control_interval()) {
SetReaderConsumedStep(config.event_driven_flow_control_interval());
}
STREAMING_CHECK(writer_consumed_step_ >= reader_consumed_step_)
<< "Writer consuemd step " << writer_consumed_step_
<< "can not be smaller then reader consumed step " << reader_consumed_step_;
}
uint32_t StreamingConfig::GetRingBufferCapacity() const { return ring_buffer_capacity_; }
streaming/src/config/streaming_config.h
+17
View File
@@ -33,6 +33,17 @@ class StreamingConfig {
std::string task_job_id_ = JobID::Nil().Hex();
// Default flow control type is unconsumed sequence flow control. More detail
// introducation and implemention in ray/streaming/src/flow_control.h.
streaming::proto::FlowControlType flow_control_type_ =
streaming::proto::FlowControlType::UnconsumedSeqFlowControl;
// Default writer and reader consumed step.
uint32_t writer_consumed_step_ = 1000;
uint32_t reader_consumed_step_ = 100;
uint32_t event_driven_flow_control_interval_ = 1;
public:
void FromProto(const uint8_t *, uint32_t size);
@@ -46,6 +57,12 @@ class StreamingConfig {
DECL_GET_SET_PROPERTY(uint32_t, EmptyMessageTimeInterval, empty_message_time_interval_)
DECL_GET_SET_PROPERTY(streaming::proto::OperatorType, OperatorType, operator_type_)
DECL_GET_SET_PROPERTY(const std::string &, JobName, job_name_)
DECL_GET_SET_PROPERTY(uint32_t, WriterConsumedStep, writer_consumed_step_)
DECL_GET_SET_PROPERTY(uint32_t, ReaderConsumedStep, reader_consumed_step_)
DECL_GET_SET_PROPERTY(streaming::proto::FlowControlType, FlowControlType,
flow_control_type_)
DECL_GET_SET_PROPERTY(uint32_t, EventDrivenFlowControlInterval,
event_driven_flow_control_interval_)
uint32_t GetRingBufferCapacity() const;
/// Note(lingxuan.zlx), RingBufferCapacity's valid range is from 1 to