[Streaming] Streaming Python API (#6755)

streaming/src/config/streaming_config.cc

@@ -28,8 +28,8 @@ void StreamingConfig::FromProto(const uint8_t *data, uint32_t size) {
   if (!config.op_name().empty()) {
     SetOpName(config.op_name());
   }
-  if (config.role() != proto::OperatorType::UNKNOWN) {
-    SetOperatorType(config.role());
+  if (config.role() != proto::NodeType::UNKNOWN) {
+    SetNodeType(config.role());
   }
   if (config.ring_buffer_capacity() != 0) {
     SetRingBufferCapacity(config.ring_buffer_capacity());

streaming/src/config/streaming_config.h

@@ -22,8 +22,7 @@ class StreamingConfig {
 
   uint32_t empty_message_time_interval_ = DEFAULT_EMPTY_MESSAGE_TIME_INTERVAL;
 
-  streaming::proto::OperatorType operator_type_ =
-      streaming::proto::OperatorType::TRANSFORM;
+  streaming::proto::NodeType node_type_ = streaming::proto::NodeType::TRANSFORM;
 
   std::string job_name_ = "DEFAULT_JOB_NAME";
 
@@ -55,7 +54,7 @@ class StreamingConfig {
   DECL_GET_SET_PROPERTY(const std::string &, WorkerName, worker_name_)
   DECL_GET_SET_PROPERTY(const std::string &, OpName, op_name_)
   DECL_GET_SET_PROPERTY(uint32_t, EmptyMessageTimeInterval, empty_message_time_interval_)
-  DECL_GET_SET_PROPERTY(streaming::proto::OperatorType, OperatorType, operator_type_)
+  DECL_GET_SET_PROPERTY(streaming::proto::NodeType, NodeType, node_type_)
   DECL_GET_SET_PROPERTY(const std::string &, JobName, job_name_)
   DECL_GET_SET_PROPERTY(uint32_t, WriterConsumedStep, writer_consumed_step_)
   DECL_GET_SET_PROPERTY(uint32_t, ReaderConsumedStep, reader_consumed_step_)

streaming/src/protobuf/remote_call.proto

@@ -0,0 +1,59 @@
+syntax = "proto3";
+
+package ray.streaming.proto;
+
+import "streaming/src/protobuf/streaming.proto";
+
+option java_package = "org.ray.streaming.runtime.generated";
+
+// Streaming execution graph
+message ExecutionGraph {
+  // A parallel operation consisting of multiple execution tasks
+  message ExecutionNode {
+    int32 node_id = 1;
+    int32 parallelism = 2;
+    NodeType node_type = 3;
+    Language language = 4;
+    // serialized user function
+    bytes function = 5;
+    repeated ExecutionTask execution_tasks = 6;
+    repeated ExecutionEdge input_edges = 7;
+    repeated ExecutionEdge output_edges = 8;
+  }
+
+  // execution edge
+  message ExecutionEdge {
+    // upstream execution node id
+    int32 src_node_id = 1;
+    // downstream execution node id
+    int32 target_node_id = 2;
+    // serialized partition between src/target node
+    bytes partition = 3;
+  }
+
+  // a parallel subtask of the execution
+  message ExecutionTask {
+    // unique execution task id
+    int32 task_id = 1;
+    // an ordered task index range from 0 to parallelism - 1
+    int32 task_index = 2;
+    // serialized actor handle
+    bytes worker_actor = 3;
+  }
+
+  // graph build time
+  uint64 build_time = 1;
+  repeated ExecutionNode execution_nodes = 2;
+}
+
+// Streaming worker context
+message WorkerContext {
+  // job name
+  string job_name = 1;
+  // unique execution task id
+  int32 task_id = 2;
+  // job config
+  map<string, string> conf = 3;
+  // execution graph
+  ExecutionGraph graph = 4;
+}

streaming/src/protobuf/streaming.proto

@@ -4,10 +4,19 @@ package ray.streaming.proto;
 
 option java_package = "org.ray.streaming.runtime.generated";
 
-enum OperatorType {
+enum Language {
+  JAVA = 0;
+  PYTHON = 1;
+}
+
+enum NodeType {
   UNKNOWN = 0;
-  TRANSFORM = 1;
-  SOURCE = 2;
+  // Sources are where your program reads its input from
+  SOURCE = 1;
+  // Transform one or more DataStreams into a new DataStream.
+  TRANSFORM = 2;
+  // Sinks consume DataStreams and forward them to files, sockets, external
+  // systems, or print them.
   SINK = 3;
 }
 
@@ -23,7 +32,7 @@ message StreamingConfig {
   string task_job_id = 2;
   string worker_name = 3;
   string op_name = 4;
-  OperatorType role = 5;
+  NodeType role = 5;
   uint32 ring_buffer_capacity = 6;
   uint32 empty_message_interval = 7;
   FlowControlType flow_control_type = 8;