wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-06 05:16:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

Browse Source 
This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1227 additions and 7040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java
+2 -2
View File
@@ -15,7 +15,7 @@ import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.master.context.JobMasterRuntimeContext;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.graphmanager.GraphManager;
import io.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import java.util.HashMap;
@@ -34,7 +34,7 @@ public class ExecutionGraphTest extends BaseUnitTest {
public void testBuildExecutionGraph() {
Map<String, String> jobConf = new HashMap<>();
StreamingConfig streamingConfig = new StreamingConfig(jobConf);
GraphManager graphManager = new GraphManagerImpl(new JobMasterRuntimeContext(streamingConfig));
GraphManager graphManager = new GraphManagerImpl(new JobRuntimeContext(streamingConfig));
JobGraph jobGraph = buildJobGraph();
jobGraph.getJobConfig().put("streaming.task.resource.cpu.limitation.enable", "true");
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java
+1 -1
View File
@@ -36,7 +36,7 @@ public class UnionStreamTest {
streamSource1
.union(streamSource2, streamSource3)
.sink((SinkFunction<Integer>) value -> {
LOG.info("UnionStreamTest, sink: {}", value);
LOG.info("UnionStreamTest: {}", value);
try {
if (!Files.exists(Paths.get(sinkFileName))) {
Files.createFile(Paths.get(sinkFileName));
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/JobMasterTest.java
+1 -1
View File
@@ -14,7 +14,7 @@ public class JobMasterTest {
Assert.assertNull(jobMaster.getGraphManager());
Assert.assertNull(jobMaster.getResourceManager());
Assert.assertNull(jobMaster.getJobMasterActor());
Assert.assertFalse(jobMaster.init(false));
Assert.assertFalse(jobMaster.init());
}
}
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManagerTest.java
+3 -3
View File
@@ -7,7 +7,7 @@ import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.config.StreamingConfig;
import io.ray.streaming.runtime.config.global.CommonConfig;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.master.context.JobMasterRuntimeContext;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.util.RayUtils;
import java.util.HashMap;
import java.util.List;
@@ -44,8 +44,8 @@ public class ResourceManagerTest extends BaseUnitTest {
Map<String, String> conf = new HashMap<String, String>();
conf.put(CommonConfig.JOB_NAME, "testApi");
StreamingConfig config = new StreamingConfig(conf);
JobMasterRuntimeContext jobMasterRuntimeContext = new JobMasterRuntimeContext(config);
ResourceManager resourceManager = new ResourceManagerImpl(jobMasterRuntimeContext);
JobRuntimeContext jobRuntimeContext = new JobRuntimeContext(config);
ResourceManager resourceManager = new ResourceManagerImpl(jobRuntimeContext);
// test register container
List<Container> containers = resourceManager.getRegisteredContainers();
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/strategy/PipelineFirstStrategyTest.java
+2 -2
View File
@@ -9,7 +9,7 @@ import io.ray.streaming.runtime.core.graph.ExecutionGraphTest;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.master.context.JobMasterRuntimeContext;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.graphmanager.GraphManager;
import io.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import io.ray.streaming.runtime.master.resourcemanager.ResourceAssignmentView;
@@ -64,7 +64,7 @@ public class PipelineFirstStrategyTest extends BaseUnitTest {
Map<String, String> jobConf = new HashMap<>();
StreamingConfig streamingConfig = new StreamingConfig(jobConf);
GraphManager graphManager = new GraphManagerImpl(new JobMasterRuntimeContext(streamingConfig));
GraphManager graphManager = new GraphManagerImpl(new JobRuntimeContext(streamingConfig));
JobGraph jobGraph = ExecutionGraphTest.buildJobGraph();
ExecutionGraph executionGraph = ExecutionGraphTest.buildExecutionGraph(graphManager, jobGraph);
ResourceAssignmentView assignmentView = strategy.assignResource(containers, executionGraph);
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java
+1 -1
View File
@@ -9,7 +9,7 @@ import io.ray.streaming.api.function.impl.FlatMapFunction;
import io.ray.streaming.api.function.impl.ReduceFunction;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.transfer.channel.ChannelId;
import io.ray.streaming.runtime.transfer.ChannelId;
import io.ray.streaming.runtime.util.EnvUtil;
import io.ray.streaming.util.Config;
import java.io.File;
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java
+5 -5
View File
@@ -6,11 +6,11 @@ import io.ray.api.Ray;
import io.ray.runtime.functionmanager.JavaFunctionDescriptor;
import io.ray.streaming.runtime.config.StreamingWorkerConfig;
import io.ray.streaming.runtime.transfer.ChannelCreationParametersBuilder;
import io.ray.streaming.runtime.transfer.ChannelId;
import io.ray.streaming.runtime.transfer.DataMessage;
import io.ray.streaming.runtime.transfer.DataReader;
import io.ray.streaming.runtime.transfer.DataWriter;
import io.ray.streaming.runtime.transfer.TransferHandler;
import io.ray.streaming.runtime.transfer.channel.ChannelId;
import io.ray.streaming.runtime.transfer.message.DataMessage;
import io.ray.streaming.util.Config;
import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
@@ -104,7 +104,7 @@ class ReaderWorker extends Worker {
new JavaFunctionDescriptor(Worker.class.getName(), "onWriterMessage", "([B)V"),
new JavaFunctionDescriptor(Worker.class.getName(), "onWriterMessageSync", "([B)[B"));
StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf);
dataReader = new DataReader(inputQueueList, inputActors, new HashMap<>(), workerConfig);
dataReader = new DataReader(inputQueueList, inputActors, workerConfig);
// Should not GetBundle in RayCall thread
Thread readThread = new Thread(Ray.wrapRunnable(new Runnable() {
@@ -124,7 +124,7 @@ class ReaderWorker extends Worker {
int checkPointId = 1;
for (int i = 0; i < msgCount * inputQueueList.size(); ++i) {
DataMessage dataMessage = (DataMessage) dataReader.read(100);
DataMessage dataMessage = dataReader.read(100);
if (dataMessage == null) {
LOGGER.error("dataMessage is null");
@@ -232,7 +232,7 @@ class WriterWorker extends Worker {
new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessage", "([B)V"),
new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessageSync", "([B)[B"));
StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf);
dataWriter = new DataWriter(outputQueueList, outputActors, new HashMap<>(), workerConfig);
dataWriter = new DataWriter(outputQueueList, outputActors, workerConfig);
Thread writerThread = new Thread(Ray.wrapRunnable(new Runnable() {
@Override
public void run() {
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/transfer/ChannelIdTest.java
-1
View File
@@ -4,7 +4,6 @@ import static org.testng.Assert.assertEquals;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.transfer.channel.ChannelId;
import io.ray.streaming.runtime.util.EnvUtil;
import org.testng.annotations.Test;
streaming/java/streaming-runtime/src/test/resources/log4j.properties
+1 -1
View File
@@ -3,4 +3,4 @@ log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SS} %-4p %c{1}:%L [%t] - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n