wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 00:29:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Support streaming flow control (#7152)

Browse Source 
* streaming writer use event driven model.

* add RefreshChannelInfo

* fix name

* minor changes according reviewer comments

* Fix according to reviewer's comments

* fix bazel lint

* code polished

* Add more comments

* rename Stop & Start of EventQueue to Freeze and Unfreeze.

* add override

* fix

* fix return value

* support flow control

* add flow control ut in mock transfer

* minor changes according to comments

* add java and python worker adaption

Co-authored-by: wanxing <wanxing.wwx@alibaba-inc.com>
This commit is contained in:
Lingxuan Zuo
2020-02-24 23:48:04 +08:00
committed by GitHub
parent 7e115490d9
commit f995099e00
16 changed files with 410 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/java/streaming-api/src/main/java/org/ray/streaming/util/Config.java
+5
View File
@@ -40,5 +40,10 @@ public class Config {
// operator type
public static final String OPERATOR_TYPE = "operator_type";
// flow control
public static final String FLOW_CONTROL_TYPE = "streaming.flow_control_type";
public static final String WRITER_CONSUMED_STEP = "streaming.writer.consumed_step";
public static final String READER_CONSUMED_STEP = "streaming.reader.consumed_step";
}
streaming/java/streaming-runtime/src/main/java/org/ray/streaming/runtime/transfer/ChannelUtils.java
+13
View File
@@ -31,6 +31,19 @@ public class ChannelUtils {
builder.setEmptyMessageInterval(
Integer.parseInt(conf.get(Config.STREAMING_EMPTY_MESSAGE_INTERVAL)));
}
if (conf.containsKey(Config.FLOW_CONTROL_TYPE)) {
builder.setFlowControlType(
Streaming.FlowControlType.forNumber(
Integer.parseInt(conf.get(Config.FLOW_CONTROL_TYPE))));
}
if (conf.containsKey(Config.WRITER_CONSUMED_STEP)) {
builder.setWriterConsumedStep(
Integer.parseInt(conf.get(Config.WRITER_CONSUMED_STEP)));
}
if (conf.containsKey(Config.READER_CONSUMED_STEP)) {
builder.setReaderConsumedStep(
Integer.parseInt(conf.get(Config.READER_CONSUMED_STEP)));
}
Streaming.StreamingConfig streamingConf = builder.build();
LOGGER.info("Streaming native conf {}", streamingConf.toString());
return streamingConf.toByteArray();