wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-13 17:45:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Implement streaming job-worker. (#8780)

Browse Source
This commit is contained in:
Tianyi Chen
2020-06-10 14:13:55 +08:00
committed by GitHub
parent 04cffb7e65
commit ec5ecb661f
56 changed files with 1078 additions and 1213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java
+8 -6
View File
@@ -35,6 +35,8 @@ public class ExecutionGraphTest extends BaseUnitTest {
StreamingConfig streamingConfig = new StreamingConfig(jobConf);
GraphManager graphManager = new GraphManagerImpl(new JobRuntimeContext(streamingConfig));
JobGraph jobGraph = buildJobGraph();
jobGraph.getJobConfig().put("streaming.task.resource.cpu.limitation.enable", "true");
ExecutionGraph executionGraph = buildExecutionGraph(graphManager, jobGraph);
List<ExecutionJobVertex> executionJobVertices = executionGraph.getExecutionJobVertexList();
@@ -48,24 +50,24 @@ public class ExecutionGraphTest extends BaseUnitTest {
executionGraph.getAllExecutionVertices().forEach(vertex -> {
Assert.assertNotNull(vertex.getStreamOperator());
Assert.assertNotNull(vertex.getJobVertexName());
Assert.assertNotNull(vertex.getExecutionJobVertexName());
Assert.assertNotNull(vertex.getVertexType());
Assert.assertNotNull(vertex.getLanguage());
Assert.assertEquals(vertex.getVertexName(),
vertex.getJobVertexId() + "-" + vertex.getJobVertexName() + "-" + vertex.getVertexIndex());
Assert.assertEquals(vertex.getExecutionVertexName(),
vertex.getExecutionJobVertexName() + "-" + vertex.getExecutionVertexIndex());
});
int startIndex = 0;
ExecutionJobVertex upStream = executionJobVertices.get(startIndex);
ExecutionJobVertex downStream = executionJobVertices.get(startIndex + 1);
Assert.assertEquals(upStream.getOutputEdges().get(0).getTargetVertex(), downStream);
Assert.assertEquals(upStream.getOutputEdges().get(0).getTargetExecutionJobVertex(), downStream);
List<ExecutionVertex> upStreamVertices = upStream.getExecutionVertices();
List<ExecutionVertex> downStreamVertices = downStream.getExecutionVertices();
upStreamVertices.forEach(vertex -> {
Assert.assertEquals(vertex.getResources().get(ResourceType.CPU.name()), 2.0);
Assert.assertEquals(vertex.getResource().get(ResourceType.CPU.name()), 2.0);
vertex.getOutputEdges().stream().forEach(upStreamOutPutEdge -> {
Assert.assertTrue(downStreamVertices.contains(upStreamOutPutEdge.getTargetVertex()));
Assert.assertTrue(downStreamVertices.contains(upStreamOutPutEdge.getTargetExecutionVertex()));
});
});
}
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java
+4 -5
View File
@@ -21,18 +21,17 @@ import org.testng.annotations.Test;
public class WordCountTest extends BaseUnitTest implements Serializable {
private static final Logger LOGGER = LoggerFactory.getLogger(WordCountTest.class);
private static final Logger LOG = LoggerFactory.getLogger(WordCountTest.class);
// TODO(zhenxuanpan): this test only works in single-process mode, because we put
// results in this in-memory map.
static Map<String, Integer> wordCount = new ConcurrentHashMap<>();
@Test(timeOut = 60000)
public void testWordCount() {
Ray.shutdown();
StreamingContext streamingContext = StreamingContext.buildContext();
Map<String, String> config = new HashMap<>();
config.put(Config.CHANNEL_TYPE, Config.MEMORY_CHANNEL);
config.put(Config.CHANNEL_TYPE, "MEMORY_CHANNEL");
streamingContext.withConfig(config);
List<String> text = new ArrayList<>();
text.add("hello world eagle eagle eagle");
@@ -58,7 +57,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOGGER.warn("Got an exception while sleeping.", e);
LOG.warn("Got an exception while sleeping.", e);
}
}
streamingContext.stop();
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobSchedulerTest.java → streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobClientTest.java
+1 -1
View File
@@ -2,7 +2,7 @@ package io.ray.streaming.runtime.master.jobscheduler;
import org.testng.annotations.Test;
public class JobSchedulerTest {
public class JobClientTest {
@Test
public void testSchedule() {
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/schedule/TaskAssignerImplTest.java
-67
View File
@@ -1,67 +0,0 @@
package io.ray.streaming.runtime.schedule;
import com.google.common.collect.Lists;
import io.ray.api.Ray;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.partition.impl.RoundRobinPartition;
import io.ray.streaming.api.stream.DataStream;
import io.ray.streaming.api.stream.DataStreamSink;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobGraphBuilder;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.core.graph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
public class TaskAssignerImplTest extends BaseUnitTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TaskAssignerImplTest.class);
@Test
public void testTaskAssignImpl() {
Ray.init();
JobGraph jobGraph = buildDataSyncPlan();
TaskAssigner taskAssigner = new TaskAssignerImpl();
ExecutionGraph executionGraph = taskAssigner.assign(jobGraph);
List<ExecutionNode> executionNodeList = executionGraph.getExecutionNodeList();
Assert.assertEquals(executionNodeList.size(), 2);
ExecutionNode sourceNode = executionNodeList.get(0);
Assert.assertEquals(sourceNode.getNodeType(), NodeType.SOURCE);
Assert.assertEquals(sourceNode.getExecutionTasks().size(), 1);
Assert.assertEquals(sourceNode.getOutputEdges().size(), 1);
List<ExecutionEdge> sourceExecutionEdges = sourceNode.getOutputEdges();
Assert.assertEquals(sourceExecutionEdges.size(), 1);
ExecutionEdge source2Sink = sourceExecutionEdges.get(0);
Assert.assertEquals(source2Sink.getPartition().getClass(), RoundRobinPartition.class);
ExecutionNode sinkNode = executionNodeList.get(1);
Assert.assertEquals(sinkNode.getNodeType(), NodeType.SINK);
Assert.assertEquals(sinkNode.getExecutionTasks().size(), 1);
Assert.assertEquals(sinkNode.getOutputEdges().size(), 0);
Ray.shutdown();
}
public JobGraph buildDataSyncPlan() {
StreamingContext streamingContext = StreamingContext.buildContext();
DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
Lists.newArrayList("a", "b", "c"));
DataStreamSink streamSink = dataStream.sink(LOGGER::info);
JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink));
return jobGraphBuilder.build();
}
}
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java
+4 -7
View File
@@ -37,6 +37,7 @@ import org.testng.annotations.Test;
public class StreamingQueueTest extends BaseUnitTest implements Serializable {
private static Logger LOGGER = LoggerFactory.getLogger(StreamingQueueTest.class);
static {
EnvUtil.loadNativeLibraries();
}
@@ -56,10 +57,6 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
LOGGER.warn("Do tear down");
}
@BeforeClass
public void setUp() {
}
@BeforeMethod
void beforeMethod() {
LOGGER.info("beforeTest");
@@ -79,7 +76,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
System.clearProperty("ray.run-mode");
}
@Test(timeOut = 3000000)
@Test(timeOut = 300000)
public void testReaderWriter() {
LOGGER.info("StreamingQueueTest.testReaderWriter run-mode: {}",
System.getProperty("ray.run-mode"));
@@ -120,7 +117,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
readerActor.call(ReaderWorker::init, inputQueueList, writerActor, msgCount);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
} catch (Exception e) {
e.printStackTrace();
}
writerActor.call(WriterWorker::init, outputQueueList, readerActor, msgCount);
@@ -160,7 +157,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
Map<String, Integer> wordCount = new ConcurrentHashMap<>();
StreamingContext streamingContext = StreamingContext.buildContext();
Map<String, String> config = new HashMap<>();
config.put(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL);
config.put(Config.CHANNEL_TYPE, "NATIVE_CHANNEL");
config.put(Config.CHANNEL_SIZE, "100000");
streamingContext.withConfig(config);
List<String> text = new ArrayList<>();
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java
+11 -8
View File
@@ -4,6 +4,7 @@ import io.ray.api.BaseActorHandle;
import io.ray.api.Ray;
import io.ray.api.ActorHandle;
import io.ray.runtime.functionmanager.JavaFunctionDescriptor;
import io.ray.streaming.runtime.config.StreamingWorkerConfig;
import io.ray.streaming.runtime.transfer.ChannelID;
import io.ray.streaming.runtime.transfer.ChannelCreationParametersBuilder;
import io.ray.streaming.runtime.transfer.DataMessage;
@@ -52,7 +53,7 @@ class ReaderWorker extends Worker {
private String name = null;
private List<String> inputQueueList = null;
Map<String, BaseActorHandle> fromActors = new HashMap<>();
Map<String, BaseActorHandle> inputActors = new HashMap<>();
private DataReader dataReader = null;
private long handler = 0;
private ActorHandle<WriterWorker> peerActor = null;
@@ -87,19 +88,20 @@ class ReaderWorker extends Worker {
LOGGER.info("java.library.path = {}", System.getProperty("java.library.path"));
for (String queue : this.inputQueueList) {
fromActors.put(queue, this.peerActor);
inputActors.put(queue, this.peerActor);
LOGGER.info("ReaderWorker actorId: {}", this.peerActor.getId());
}
Map<String, String> conf = new HashMap<>();
conf.put(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL);
conf.put(Config.CHANNEL_TYPE, "NATIVE_CHANNEL");
conf.put(Config.CHANNEL_SIZE, "100000");
conf.put(Config.STREAMING_JOB_NAME, "integrationTest1");
ChannelCreationParametersBuilder.setJavaWriterFunctionDesc(
new JavaFunctionDescriptor(Worker.class.getName(), "onWriterMessage", "([B)V"),
new JavaFunctionDescriptor(Worker.class.getName(), "onWriterMessageSync", "([B)[B"));
dataReader = new DataReader(inputQueueList, fromActors, conf);
StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf);
dataReader = new DataReader(inputQueueList, inputActors, workerConfig);
// Should not GetBundle in RayCall thread
Thread readThread = new Thread(Ray.wrapRunnable(new Runnable() {
@@ -171,7 +173,7 @@ class WriterWorker extends Worker {
private String name = null;
private List<String> outputQueueList = null;
Map<String, BaseActorHandle> toActors = new HashMap<>();
Map<String, BaseActorHandle> outputActors = new HashMap<>();
DataWriter dataWriter = null;
ActorHandle<ReaderWorker> peerActor = null;
int msgCount = 0;
@@ -203,7 +205,7 @@ class WriterWorker extends Worker {
LOGGER.info("WriterWorker init:");
for (String queue : this.outputQueueList) {
toActors.put(queue, this.peerActor);
outputActors.put(queue, this.peerActor);
LOGGER.info("WriterWorker actorId: {}", this.peerActor.getId());
}
@@ -219,13 +221,14 @@ class WriterWorker extends Worker {
}
Map<String, String> conf = new HashMap<>();
conf.put(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL);
conf.put(Config.CHANNEL_TYPE, "NATIVE_CHANNEL");
conf.put(Config.CHANNEL_SIZE, "100000");
conf.put(Config.STREAMING_JOB_NAME, "integrationTest1");
ChannelCreationParametersBuilder.setJavaReaderFunctionDesc(
new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessage", "([B)V"),
new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessageSync", "([B)[B"));
dataWriter = new DataWriter(this.outputQueueList, this.toActors, conf);
StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf);
dataWriter = new DataWriter(outputQueueList, outputActors, workerConfig);
Thread writerThread = new Thread(Ray.wrapRunnable(new Runnable() {
@Override
public void run() {