[Streaming] Implement streaming job-worker. (#8780)

This commit is contained in:
Tianyi Chen
2020-06-10 14:13:55 +08:00
committed by GitHub
parent 04cffb7e65
commit ec5ecb661f
56 changed files with 1078 additions and 1213 deletions
@@ -3,9 +3,9 @@ package io.ray.streaming.api.context;
import com.google.common.base.Preconditions;
import io.ray.api.Ray;
import io.ray.streaming.api.stream.StreamSink;
import io.ray.streaming.client.JobClient;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobGraphBuilder;
import io.ray.streaming.schedule.JobScheduler;
import io.ray.streaming.util.Config;
import java.io.Serializable;
import java.util.ArrayList;
@@ -74,12 +74,12 @@ public class StreamingContext implements Serializable {
LOG.info("Reuse existing cluster.");
}
ServiceLoader<JobScheduler> serviceLoader = ServiceLoader.load(JobScheduler.class);
Iterator<JobScheduler> iterator = serviceLoader.iterator();
ServiceLoader<JobClient> serviceLoader = ServiceLoader.load(JobClient.class);
Iterator<JobClient> iterator = serviceLoader.iterator();
Preconditions.checkArgument(iterator.hasNext(),
"No JobScheduler implementation has been provided.");
JobScheduler jobSchedule = iterator.next();
jobSchedule.schedule(jobGraph, jobConfig);
"No JobClient implementation has been provided.");
JobClient jobClient = iterator.next();
jobClient.submit(jobGraph, jobConfig);
}
public int generateId() {
@@ -0,0 +1,17 @@
package io.ray.streaming.client;
import io.ray.streaming.jobgraph.JobGraph;
import java.util.Map;
/**
* Interface of the job client.
*/
public interface JobClient {
/**
* Submit job with logical plan to run.
*
* @param jobGraph The logical plan.
*/
void submit(JobGraph jobGraph, Map<String, String> conf);
}
@@ -1,18 +0,0 @@
package io.ray.streaming.schedule;
import io.ray.streaming.jobgraph.JobGraph;
import java.util.Map;
/**
* Interface of the job scheduler.
*/
public interface JobScheduler {
/**
* Assign logical plan to physical execution graph, and schedule job to run.
*
* @param jobGraph The logical plan.
*/
void schedule(JobGraph jobGraph, Map<String, String> conf);
}