[Streaming] Streaming scheduler - part1-1: job graph (#6712)

2026-07-07 20:14:09 +08:00 · 2020-01-15 13:12:03 +08:00
parent 4227fd1b60
commit 9a4da1951e
27 changed files with 322 additions and 250 deletions
@@ -2,7 +2,6 @@ package org.ray.streaming.runtime.cluster;

 import java.util.ArrayList;
 import java.util.List;
-
 import org.ray.api.Ray;
 import org.ray.api.RayActor;
 import org.ray.streaming.runtime.worker.JobWorker;
@@ -1,7 +1,6 @@
 package org.ray.streaming.runtime.core.graph;

 import java.io.Serializable;
-
 import org.ray.streaming.api.partition.Partition;

 /**
@@ -6,7 +6,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-
 import org.ray.api.RayActor;
 import org.ray.streaming.runtime.worker.JobWorker;

@@ -3,13 +3,14 @@ package org.ray.streaming.runtime.core.graph;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import org.ray.streaming.jobgraph.VertexType;
 import org.ray.streaming.operator.StreamOperator;
-import org.ray.streaming.plan.VertexType;

 /**
 * A node in the physical execution graph.
 */
 public class ExecutionNode implements Serializable {
+
  private int nodeId;
  private int parallelism;
  private NodeType nodeType;
@@ -1,7 +1,6 @@
 package org.ray.streaming.runtime.core.graph;

 import java.io.Serializable;
-
 import org.ray.api.RayActor;
 import org.ray.streaming.runtime.worker.JobWorker;

@@ -6,8 +6,8 @@ import java.util.Map;
 import org.ray.api.Ray;
 import org.ray.api.RayActor;
 import org.ray.api.RayObject;
-import org.ray.streaming.plan.Plan;
-import org.ray.streaming.plan.PlanVertex;
+import org.ray.streaming.jobgraph.JobGraph;
+import org.ray.streaming.jobgraph.JobVertex;
 import org.ray.streaming.runtime.cluster.ResourceManager;
 import org.ray.streaming.runtime.core.graph.ExecutionGraph;
 import org.ray.streaming.runtime.core.graph.ExecutionNode;
@@ -21,8 +21,8 @@ import org.ray.streaming.schedule.JobScheduler;
 * from ResourceManager.
 */
 public class JobSchedulerImpl implements JobScheduler {
-  private Plan plan;
-  private Map<String, Object> jobConfig;
+  private JobGraph jobGraph;
+  private Map<String, String> jobConfig;
  private ResourceManager resourceManager;
  private TaskAssigner taskAssigner;

@@ -35,14 +35,14 @@ public class JobSchedulerImpl implements JobScheduler {
   * Schedule physical plan to execution graph, and call streaming worker to init and run.
   */
  @Override
-  public void schedule(Plan plan, Map<String, Object> jobConfig) {
+  public void schedule(JobGraph jobGraph, Map<String, String> jobConfig) {
    this.jobConfig = jobConfig;
-    this.plan = plan;
+    this.jobGraph = jobGraph;
    System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
    Ray.init();

    List<RayActor<JobWorker>> workers = this.resourceManager.createWorkers(getPlanWorker());
-    ExecutionGraph executionGraph = this.taskAssigner.assign(this.plan, workers);
+    ExecutionGraph executionGraph = this.taskAssigner.assign(this.jobGraph, workers);

    List<ExecutionNode> executionNodes = executionGraph.getExecutionNodeList();
    List<RayObject<Boolean>> waits = new ArrayList<>();
@@ -59,7 +59,7 @@ public class JobSchedulerImpl implements JobScheduler {
  }

  private int getPlanWorker() {
-    List<PlanVertex> planVertexList = plan.getPlanVertexList();
-    return planVertexList.stream().map(PlanVertex::getParallelism).reduce(0, Integer::sum);
+    List<JobVertex> jobVertexList = jobGraph.getJobVertexList();
+    return jobVertexList.stream().map(JobVertex::getParallelism).reduce(0, Integer::sum);
  }
 }
@@ -3,7 +3,7 @@ package org.ray.streaming.runtime.schedule;
 import java.io.Serializable;
 import java.util.List;
 import org.ray.api.RayActor;
-import org.ray.streaming.plan.Plan;
+import org.ray.streaming.jobgraph.JobGraph;
 import org.ray.streaming.runtime.core.graph.ExecutionGraph;
 import org.ray.streaming.runtime.worker.JobWorker;

@@ -15,6 +15,6 @@ public interface TaskAssigner extends Serializable {
  /**
   * Assign logical plan to physical execution graph.
   */
-  ExecutionGraph assign(Plan plan, List<RayActor<JobWorker>> workers);
+  ExecutionGraph assign(JobGraph jobGraph, List<RayActor<JobWorker>> workers);

 }
@@ -6,9 +6,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.ray.api.RayActor;
-import org.ray.streaming.plan.Plan;
-import org.ray.streaming.plan.PlanEdge;
-import org.ray.streaming.plan.PlanVertex;
+import org.ray.streaming.jobgraph.JobEdge;
+import org.ray.streaming.jobgraph.JobGraph;
+import org.ray.streaming.jobgraph.JobVertex;
 import org.ray.streaming.runtime.core.graph.ExecutionEdge;
 import org.ray.streaming.runtime.core.graph.ExecutionGraph;
 import org.ray.streaming.runtime.core.graph.ExecutionNode;
@@ -20,37 +20,37 @@ public class TaskAssignerImpl implements TaskAssigner {
  /**
   * Assign an optimized logical plan to execution graph.
   *
-   * @param plan    The logical plan.
+   * @param jobGraph    The logical plan.
   * @param workers The worker actors.
   * @return The physical execution graph.
   */
  @Override
-  public ExecutionGraph assign(Plan plan, List<RayActor<JobWorker>> workers) {
-    List<PlanVertex> planVertices = plan.getPlanVertexList();
-    List<PlanEdge> planEdges = plan.getPlanEdgeList();
+  public ExecutionGraph assign(JobGraph jobGraph, List<RayActor<JobWorker>> workers) {
+    List<JobVertex> jobVertices = jobGraph.getJobVertexList();
+    List<JobEdge> jobEdges = jobGraph.getJobEdgeList();

    int taskId = 0;
    Map<Integer, ExecutionNode> idToExecutionNode = new HashMap<>();
-    for (PlanVertex planVertex : planVertices) {
-      ExecutionNode executionNode = new ExecutionNode(planVertex.getVertexId(),
-          planVertex.getParallelism());
-      executionNode.setNodeType(planVertex.getVertexType());
+    for (JobVertex jobVertex : jobVertices) {
+      ExecutionNode executionNode = new ExecutionNode(jobVertex.getVertexId(),
+          jobVertex.getParallelism());
+      executionNode.setNodeType(jobVertex.getVertexType());
      List<ExecutionTask> vertexTasks = new ArrayList<>();
-      for (int taskIndex = 0; taskIndex < planVertex.getParallelism(); taskIndex++) {
+      for (int taskIndex = 0; taskIndex < jobVertex.getParallelism(); taskIndex++) {
        vertexTasks.add(new ExecutionTask(taskId, taskIndex, workers.get(taskId)));
        taskId++;
      }
      executionNode.setExecutionTasks(vertexTasks);
-      executionNode.setStreamOperator(planVertex.getStreamOperator());
+      executionNode.setStreamOperator(jobVertex.getStreamOperator());
      idToExecutionNode.put(executionNode.getNodeId(), executionNode);
    }

-    for (PlanEdge planEdge : planEdges) {
-      int srcNodeId = planEdge.getSrcVertexId();
-      int targetNodeId = planEdge.getTargetVertexId();
+    for (JobEdge jobEdge : jobEdges) {
+      int srcNodeId = jobEdge.getSrcVertexId();
+      int targetNodeId = jobEdge.getTargetVertexId();

      ExecutionEdge executionEdge = new ExecutionEdge(srcNodeId, targetNodeId,
-          planEdge.getPartition());
+          jobEdge.getPartition());
      idToExecutionNode.get(srcNodeId).addExecutionEdge(executionEdge);
      idToExecutionNode.get(targetNodeId).addInputEdge(executionEdge);
    }
@@ -36,7 +36,7 @@ public class JobWorker implements Serializable {
  }

  private int taskId;
-  private Map<String, Object> config;
+  private Map<String, String> config;
  private WorkerContext workerContext;
  private ExecutionNode executionNode;
  private ExecutionTask executionTask;
@@ -88,7 +88,7 @@ public class JobWorker implements Serializable {
    return taskId;
  }

-  public Map<String, Object> getConfig() {
+  public Map<String, String> getConfig() {
    return config;
  }

@@ -3,7 +3,6 @@ package org.ray.streaming.runtime.worker.context;
 import static org.ray.streaming.util.Config.STREAMING_BATCH_MAX_COUNT;

 import java.util.Map;
-
 import org.ray.streaming.api.context.RuntimeContext;
 import org.ray.streaming.runtime.core.graph.ExecutionTask;

@@ -16,16 +15,16 @@ public class RayRuntimeContext implements RuntimeContext {
  private int parallelism;
  private Long batchId;
  private final Long maxBatch;
-  private Map<String, Object> config;
+  private Map<String, String> config;

-  public RayRuntimeContext(ExecutionTask executionTask, Map<String, Object> config,
+  public RayRuntimeContext(ExecutionTask executionTask, Map<String, String> config,
      int parallelism) {
    this.taskId = executionTask.getTaskId();
    this.config = config;
    this.taskIndex = executionTask.getTaskIndex();
    this.parallelism = parallelism;
    if (config.containsKey(STREAMING_BATCH_MAX_COUNT)) {
-      this.maxBatch = Long.valueOf(String.valueOf(config.get(STREAMING_BATCH_MAX_COUNT)));
+      this.maxBatch = Long.valueOf(config.get(STREAMING_BATCH_MAX_COUNT));
    } else {
      this.maxBatch = Long.MAX_VALUE;
    }
@@ -11,9 +11,9 @@ public class WorkerContext implements Serializable {

  private int taskId;
  private ExecutionGraph executionGraph;
-  private Map<String, Object> config;
+  private Map<String, String> config;

-  public WorkerContext(int taskId, ExecutionGraph executionGraph, Map<String, Object> jobConfig) {
+  public WorkerContext(int taskId, ExecutionGraph executionGraph, Map<String, String> jobConfig) {
    this.taskId = taskId;
    this.executionGraph = executionGraph;
    this.config = jobConfig;
@@ -35,7 +35,7 @@ public class WorkerContext implements Serializable {
    this.executionGraph = executionGraph;
  }

-  public Map<String, Object> getConfig() {
+  public Map<String, String> getConfig() {
    return config;
  }
 }