mirror of
https://github.com/wassname/ray.git
synced 2026-07-02 17:23:09 +08:00
[Java] Format ray java code (#13056)
This commit is contained in:
+8
-4
@@ -24,13 +24,17 @@ public abstract class BaseUnitTest {
|
||||
|
||||
@BeforeMethod
|
||||
public void testBegin(Method method) {
|
||||
LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>",
|
||||
method.getDeclaringClass(), method.getName());
|
||||
LOG.info(
|
||||
">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>",
|
||||
method.getDeclaringClass(),
|
||||
method.getName());
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void testEnd(Method method) {
|
||||
LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>",
|
||||
method.getDeclaringClass(), method.getName());
|
||||
LOG.info(
|
||||
">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>",
|
||||
method.getDeclaringClass(),
|
||||
method.getName());
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -15,4 +15,4 @@ public class TestHelper {
|
||||
public static boolean isUT() {
|
||||
return UT_FLAG;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+31
-23
@@ -43,20 +43,25 @@ public class ExecutionGraphTest extends BaseUnitTest {
|
||||
|
||||
Assert.assertEquals(executionJobVertices.size(), jobGraph.getJobVertices().size());
|
||||
|
||||
int totalVertexNum = jobGraph.getJobVertices().stream()
|
||||
.mapToInt(JobVertex::getParallelism).sum();
|
||||
int totalVertexNum =
|
||||
jobGraph.getJobVertices().stream().mapToInt(JobVertex::getParallelism).sum();
|
||||
Assert.assertEquals(executionGraph.getAllExecutionVertices().size(), totalVertexNum);
|
||||
Assert.assertEquals(executionGraph.getAllExecutionVertices().size(),
|
||||
Assert.assertEquals(
|
||||
executionGraph.getAllExecutionVertices().size(),
|
||||
executionGraph.getExecutionVertexIdGenerator().get());
|
||||
|
||||
executionGraph.getAllExecutionVertices().forEach(vertex -> {
|
||||
Assert.assertNotNull(vertex.getStreamOperator());
|
||||
Assert.assertNotNull(vertex.getExecutionJobVertexName());
|
||||
Assert.assertNotNull(vertex.getVertexType());
|
||||
Assert.assertNotNull(vertex.getLanguage());
|
||||
Assert.assertEquals(vertex.getExecutionVertexName(),
|
||||
vertex.getExecutionJobVertexName() + "-" + vertex.getExecutionVertexIndex());
|
||||
});
|
||||
executionGraph
|
||||
.getAllExecutionVertices()
|
||||
.forEach(
|
||||
vertex -> {
|
||||
Assert.assertNotNull(vertex.getStreamOperator());
|
||||
Assert.assertNotNull(vertex.getExecutionJobVertexName());
|
||||
Assert.assertNotNull(vertex.getVertexType());
|
||||
Assert.assertNotNull(vertex.getLanguage());
|
||||
Assert.assertEquals(
|
||||
vertex.getExecutionVertexName(),
|
||||
vertex.getExecutionJobVertexName() + "-" + vertex.getExecutionVertexIndex());
|
||||
});
|
||||
|
||||
int startIndex = 0;
|
||||
ExecutionJobVertex upStream = executionJobVertices.get(startIndex);
|
||||
@@ -65,13 +70,17 @@ public class ExecutionGraphTest extends BaseUnitTest {
|
||||
|
||||
List<ExecutionVertex> upStreamVertices = upStream.getExecutionVertices();
|
||||
List<ExecutionVertex> downStreamVertices = downStream.getExecutionVertices();
|
||||
upStreamVertices.forEach(vertex -> {
|
||||
Assert.assertEquals((double) vertex.getResource().get(ResourceType.CPU.name()), 2.0);
|
||||
vertex.getOutputEdges().forEach(upStreamOutPutEdge -> {
|
||||
Assert
|
||||
.assertTrue(downStreamVertices.contains(upStreamOutPutEdge.getTargetExecutionVertex()));
|
||||
});
|
||||
});
|
||||
upStreamVertices.forEach(
|
||||
vertex -> {
|
||||
Assert.assertEquals((double) vertex.getResource().get(ResourceType.CPU.name()), 2.0);
|
||||
vertex
|
||||
.getOutputEdges()
|
||||
.forEach(
|
||||
upStreamOutPutEdge -> {
|
||||
Assert.assertTrue(
|
||||
downStreamVertices.contains(upStreamOutPutEdge.getTargetExecutionVertex()));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public static ExecutionGraph buildExecutionGraph(GraphManager graphManager) {
|
||||
@@ -84,8 +93,8 @@ public class ExecutionGraphTest extends BaseUnitTest {
|
||||
|
||||
public static JobGraph buildJobGraph() {
|
||||
StreamingContext streamingContext = StreamingContext.buildContext();
|
||||
DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
|
||||
Lists.newArrayList("a", "b", "c"));
|
||||
DataStream<String> dataStream =
|
||||
DataStreamSource.fromCollection(streamingContext, Lists.newArrayList("a", "b", "c"));
|
||||
StreamSink streamSink = dataStream.sink(x -> LOG.info(x));
|
||||
|
||||
Map<String, String> jobConfig = new HashMap<>();
|
||||
@@ -94,10 +103,9 @@ public class ExecutionGraphTest extends BaseUnitTest {
|
||||
jobConfig.put(ResourceConfig.TASK_RESOURCE_CPU, "2.0");
|
||||
jobConfig.put(ResourceConfig.TASK_RESOURCE_MEM, "2.0");
|
||||
|
||||
JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(
|
||||
Lists.newArrayList(streamSink), "test", jobConfig);
|
||||
JobGraphBuilder jobGraphBuilder =
|
||||
new JobGraphBuilder(Lists.newArrayList(streamSink), "test", jobConfig);
|
||||
|
||||
return jobGraphBuilder.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+17
-15
@@ -45,8 +45,7 @@ public class HybridStreamTest {
|
||||
@Test(timeOut = 60000)
|
||||
public void testHybridDataStream() throws Exception {
|
||||
Ray.shutdown();
|
||||
Preconditions.checkArgument(
|
||||
EnvUtil.executeCommand(ImmutableList.of("ray", "stop"), 5));
|
||||
Preconditions.checkArgument(EnvUtil.executeCommand(ImmutableList.of("ray", "stop"), 5));
|
||||
String sinkFileName = "/tmp/testHybridDataStream.txt";
|
||||
Files.deleteIfExists(Paths.get(sinkFileName));
|
||||
|
||||
@@ -59,18 +58,22 @@ public class HybridStreamTest {
|
||||
.map("ray.streaming.tests.test_hybrid_stream", "map_func1")
|
||||
.filter("ray.streaming.tests.test_hybrid_stream", "filter_func1")
|
||||
.asJavaStream()
|
||||
.sink((SinkFunction<Object>) value -> {
|
||||
LOG.info("HybridStreamTest: {}", value);
|
||||
try {
|
||||
if (!Files.exists(Paths.get(sinkFileName))) {
|
||||
Files.createFile(Paths.get(sinkFileName));
|
||||
}
|
||||
Files.write(Paths.get(sinkFileName), value.toString().getBytes(),
|
||||
StandardOpenOption.APPEND);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
.sink(
|
||||
(SinkFunction<Object>)
|
||||
value -> {
|
||||
LOG.info("HybridStreamTest: {}", value);
|
||||
try {
|
||||
if (!Files.exists(Paths.get(sinkFileName))) {
|
||||
Files.createFile(Paths.get(sinkFileName));
|
||||
}
|
||||
Files.write(
|
||||
Paths.get(sinkFileName),
|
||||
value.toString().getBytes(),
|
||||
StandardOpenOption.APPEND);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
context.execute("HybridStreamTestJob");
|
||||
int sleptTime = 0;
|
||||
TimeUnit.SECONDS.sleep(3);
|
||||
@@ -94,5 +97,4 @@ public class HybridStreamTest {
|
||||
context.stop();
|
||||
LOG.info("HybridStreamTest succeed");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+16
-13
@@ -35,18 +35,22 @@ public class UnionStreamTest {
|
||||
DataStreamSource.fromCollection(context, Arrays.asList(1, 1));
|
||||
streamSource1
|
||||
.union(streamSource2, streamSource3)
|
||||
.sink((SinkFunction<Integer>) value -> {
|
||||
LOG.info("UnionStreamTest, sink: {}", value);
|
||||
try {
|
||||
if (!Files.exists(Paths.get(sinkFileName))) {
|
||||
Files.createFile(Paths.get(sinkFileName));
|
||||
}
|
||||
Files.write(Paths.get(sinkFileName), value.toString().getBytes(),
|
||||
StandardOpenOption.APPEND);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
.sink(
|
||||
(SinkFunction<Integer>)
|
||||
value -> {
|
||||
LOG.info("UnionStreamTest, sink: {}", value);
|
||||
try {
|
||||
if (!Files.exists(Paths.get(sinkFileName))) {
|
||||
Files.createFile(Paths.get(sinkFileName));
|
||||
}
|
||||
Files.write(
|
||||
Paths.get(sinkFileName),
|
||||
value.toString().getBytes(),
|
||||
StandardOpenOption.APPEND);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
context.execute("UnionStreamTest");
|
||||
int sleptTime = 0;
|
||||
TimeUnit.SECONDS.sleep(3);
|
||||
@@ -68,5 +72,4 @@ public class UnionStreamTest {
|
||||
context.stop();
|
||||
LOG.info("HybridStreamTest succeed");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+13
-12
@@ -37,19 +37,21 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
|
||||
text.add("hello world eagle eagle eagle");
|
||||
DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
|
||||
streamSource
|
||||
.flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
|
||||
String[] records = value.split(" ");
|
||||
for (String record : records) {
|
||||
collector.collect(new WordAndCount(record, 1));
|
||||
}
|
||||
})
|
||||
.flatMap(
|
||||
(FlatMapFunction<String, WordAndCount>)
|
||||
(value, collector) -> {
|
||||
String[] records = value.split(" ");
|
||||
for (String record : records) {
|
||||
collector.collect(new WordAndCount(record, 1));
|
||||
}
|
||||
})
|
||||
.filter(pair -> !pair.word.contains("world"))
|
||||
.keyBy(pair -> pair.word)
|
||||
.reduce((ReduceFunction<WordAndCount>) (oldValue, newValue) ->
|
||||
new WordAndCount(oldValue.word,
|
||||
oldValue.count + newValue.count))
|
||||
.sink((SinkFunction<WordAndCount>)
|
||||
result -> wordCount.put(result.word, result.count));
|
||||
.reduce(
|
||||
(ReduceFunction<WordAndCount>)
|
||||
(oldValue, newValue) ->
|
||||
new WordAndCount(oldValue.word, oldValue.count + newValue.count))
|
||||
.sink((SinkFunction<WordAndCount>) result -> wordCount.put(result.word, result.count));
|
||||
|
||||
streamingContext.execute("testWordCount");
|
||||
|
||||
@@ -74,5 +76,4 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
|
||||
this.count = count;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -31,5 +31,4 @@ public class JobMasterTest extends BaseUnitTest {
|
||||
Assert.assertNull(jobMaster.getJobMasterActor());
|
||||
Assert.assertFalse(jobMaster.init(false));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -8,5 +8,4 @@ public class JobClientTest {
|
||||
public void testSchedule() {
|
||||
// TODO (tianyi): need JobWorker Part to do this.
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-2
@@ -70,6 +70,4 @@ public class PipelineFirstStrategyTest extends BaseUnitTest {
|
||||
ResourceAssignmentView assignmentView = strategy.assignResource(containers, executionGraph);
|
||||
Assert.assertNotNull(assignmentView);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+11
-14
@@ -2,7 +2,6 @@ package io.ray.streaming.runtime.python;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import io.ray.streaming.api.stream.StreamSink;
|
||||
import io.ray.streaming.jobgraph.JobGraph;
|
||||
import io.ray.streaming.jobgraph.JobGraphBuilder;
|
||||
@@ -24,27 +23,25 @@ public class PythonGatewayTest {
|
||||
config.put("k1", "v1");
|
||||
gateway.withConfig(serializer.serialize(config));
|
||||
byte[] mockPySource = new byte[0];
|
||||
Object source = serializer.deserialize(
|
||||
gateway.createPythonStreamSource(mockPySource));
|
||||
Object source = serializer.deserialize(gateway.createPythonStreamSource(mockPySource));
|
||||
byte[] mockPyFunc = new byte[0];
|
||||
Object mapPyFunc = serializer.deserialize(gateway.createPyFunc(mockPyFunc));
|
||||
Object mapStream = serializer.deserialize(
|
||||
gateway.callMethod(
|
||||
serializer.serialize(Arrays.asList(source, "map", mapPyFunc))));
|
||||
Object mapStream =
|
||||
serializer.deserialize(
|
||||
gateway.callMethod(serializer.serialize(Arrays.asList(source, "map", mapPyFunc))));
|
||||
byte[] mockPyPartition = new byte[0];
|
||||
Object partition = serializer.deserialize(
|
||||
gateway.createPyPartition(mockPyPartition));
|
||||
Object partitionedStream = serializer.deserialize(
|
||||
gateway.callMethod(
|
||||
serializer.serialize(Arrays.asList(mapStream, "partitionBy", partition))));
|
||||
Object partition = serializer.deserialize(gateway.createPyPartition(mockPyPartition));
|
||||
Object partitionedStream =
|
||||
serializer.deserialize(
|
||||
gateway.callMethod(
|
||||
serializer.serialize(Arrays.asList(mapStream, "partitionBy", partition))));
|
||||
byte[] mockSinkFunc = new byte[0];
|
||||
Object sinkPyFunc = serializer.deserialize(gateway.createPyFunc(mockSinkFunc));
|
||||
gateway.callMethod(
|
||||
serializer.serialize(Arrays.asList(partitionedStream, "sink", sinkPyFunc)));
|
||||
gateway.callMethod(serializer.serialize(Arrays.asList(partitionedStream, "sink", sinkPyFunc)));
|
||||
List<StreamSink> streamSinks = gateway.getStreamingContext().getStreamSinks();
|
||||
assertEquals(streamSinks.size(), 1);
|
||||
JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(streamSinks, "py_job");
|
||||
JobGraph jobGraph = jobGraphBuilder.build();
|
||||
jobGraph.printJobGraph();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-6
@@ -3,7 +3,6 @@ package io.ray.streaming.runtime.serialization;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
|
||||
import io.ray.streaming.message.KeyRecord;
|
||||
import io.ray.streaming.message.Record;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
@@ -17,11 +16,11 @@ public class CrossLangSerializerTest {
|
||||
CrossLangSerializer serializer = new CrossLangSerializer();
|
||||
Record record = new Record("value");
|
||||
record.setStream("stream1");
|
||||
assertTrue(EqualsBuilder.reflectionEquals(record,
|
||||
serializer.deserialize(serializer.serialize(record))));
|
||||
assertTrue(
|
||||
EqualsBuilder.reflectionEquals(
|
||||
record, serializer.deserialize(serializer.serialize(record))));
|
||||
KeyRecord keyRecord = new KeyRecord("key", "value");
|
||||
keyRecord.setStream("stream2");
|
||||
assertEquals(keyRecord,
|
||||
serializer.deserialize(serializer.serialize(keyRecord)));
|
||||
assertEquals(keyRecord, serializer.deserialize(serializer.serialize(keyRecord)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+8
-13
@@ -3,7 +3,6 @@ package io.ray.streaming.runtime.serialization;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
@@ -18,20 +17,17 @@ public class MsgPackSerializerTest {
|
||||
public void testSerializeByte() {
|
||||
MsgPackSerializer serializer = new MsgPackSerializer();
|
||||
|
||||
assertEquals(serializer.deserialize(
|
||||
serializer.serialize((byte) 1)), (byte) 1);
|
||||
assertEquals(serializer.deserialize(serializer.serialize((byte) 1)), (byte) 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerialize() {
|
||||
MsgPackSerializer serializer = new MsgPackSerializer();
|
||||
|
||||
assertEquals(serializer.deserialize
|
||||
(serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE);
|
||||
assertEquals(serializer.deserialize(
|
||||
serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE);
|
||||
assertEquals(serializer.deserialize(
|
||||
serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE);
|
||||
assertEquals(serializer.deserialize(serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE);
|
||||
assertEquals(
|
||||
serializer.deserialize(serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE);
|
||||
assertEquals(serializer.deserialize(serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE);
|
||||
|
||||
Map map = new HashMap();
|
||||
List list = new ArrayList<>();
|
||||
@@ -47,8 +43,7 @@ public class MsgPackSerializerTest {
|
||||
assertEquals(o, map);
|
||||
|
||||
byte[] binary = {1, 2, 3, 4};
|
||||
assertTrue(Arrays.equals(
|
||||
binary, (byte[]) (serializer.deserialize(serializer.serialize(binary)))));
|
||||
assertTrue(
|
||||
Arrays.equals(binary, (byte[]) (serializer.deserialize(serializer.serialize(binary)))));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+33
-32
@@ -3,7 +3,6 @@ package io.ray.streaming.runtime.streamingqueue;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.ray.api.ActorHandle;
|
||||
import io.ray.api.Ray;
|
||||
import io.ray.runtime.config.RayConfig;
|
||||
import io.ray.streaming.api.context.StreamingContext;
|
||||
import io.ray.streaming.api.function.impl.FlatMapFunction;
|
||||
import io.ray.streaming.api.function.impl.ReduceFunction;
|
||||
@@ -76,8 +75,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
|
||||
@Test(timeOut = 300000)
|
||||
public void testReaderWriter() {
|
||||
LOGGER.info("StreamingQueueTest.testReaderWriter run-mode: {}",
|
||||
System.getProperty("ray.run-mode"));
|
||||
LOGGER.info(
|
||||
"StreamingQueueTest.testReaderWriter run-mode: {}", System.getProperty("ray.run-mode"));
|
||||
Ray.shutdown();
|
||||
System.setProperty("ray.head-args.0", "--num-cpus=4");
|
||||
System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}");
|
||||
@@ -90,10 +89,10 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
ActorHandle<WriterWorker> writerActor = Ray.actor(WriterWorker::new, "writer").remote();
|
||||
ActorHandle<ReaderWorker> readerActor = Ray.actor(ReaderWorker::new, "reader").remote();
|
||||
|
||||
LOGGER.info("call getName on writerActor: {}",
|
||||
writerActor.task(WriterWorker::getName).remote().get());
|
||||
LOGGER.info("call getName on readerActor: {}",
|
||||
readerActor.task(ReaderWorker::getName).remote().get());
|
||||
LOGGER.info(
|
||||
"call getName on writerActor: {}", writerActor.task(WriterWorker::getName).remote().get());
|
||||
LOGGER.info(
|
||||
"call getName on readerActor: {}", readerActor.task(ReaderWorker::getName).remote().get());
|
||||
|
||||
// LOGGER.info(writerActor.task(WriterWorker::testCallReader, readerActor).remote().get());
|
||||
List<String> outputQueueList = new ArrayList<>();
|
||||
@@ -117,8 +116,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
writerActor.task(WriterWorker::init, outputQueueList, readerActor, msgCount).remote();
|
||||
|
||||
long time = 0;
|
||||
while (time < 20000 &&
|
||||
readerActor.task(ReaderWorker::getTotalMsg).remote().get() < msgCount * queueNum) {
|
||||
while (time < 20000
|
||||
&& readerActor.task(ReaderWorker::getTotalMsg).remote().get() < msgCount * queueNum) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
time += 1000;
|
||||
@@ -128,8 +127,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
}
|
||||
|
||||
Assert.assertEquals(
|
||||
readerActor.task(ReaderWorker::getTotalMsg).remote().get().intValue(),
|
||||
msgCount * queueNum);
|
||||
readerActor.task(ReaderWorker::getTotalMsg).remote().get().intValue(), msgCount * queueNum);
|
||||
}
|
||||
|
||||
@Test(timeOut = 60000)
|
||||
@@ -143,8 +141,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
// ray init
|
||||
Ray.init();
|
||||
LOGGER.info("testWordCount");
|
||||
LOGGER.info("StreamingQueueTest.testWordCount run-mode: {}",
|
||||
System.getProperty("ray.run-mode"));
|
||||
LOGGER.info(
|
||||
"StreamingQueueTest.testWordCount run-mode: {}", System.getProperty("ray.run-mode"));
|
||||
String resultFile = "/tmp/io.ray.streaming.runtime.streamingqueue.testWordCount.txt";
|
||||
deleteResultFile(resultFile);
|
||||
|
||||
@@ -158,22 +156,27 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
text.add("hello world eagle eagle eagle");
|
||||
DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
|
||||
streamSource
|
||||
.flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
|
||||
String[] records = value.split(" ");
|
||||
for (String record : records) {
|
||||
collector.collect(new WordAndCount(record, 1));
|
||||
}
|
||||
})
|
||||
.flatMap(
|
||||
(FlatMapFunction<String, WordAndCount>)
|
||||
(value, collector) -> {
|
||||
String[] records = value.split(" ");
|
||||
for (String record : records) {
|
||||
collector.collect(new WordAndCount(record, 1));
|
||||
}
|
||||
})
|
||||
.keyBy(pair -> pair.word)
|
||||
.reduce((ReduceFunction<WordAndCount>) (oldValue, newValue) -> {
|
||||
LOGGER.info("reduce: {} {}", oldValue, newValue);
|
||||
return new WordAndCount(oldValue.word, oldValue.count + newValue.count);
|
||||
})
|
||||
.sink(s -> {
|
||||
LOGGER.info("sink {} {}", s.word, s.count);
|
||||
wordCount.put(s.word, s.count);
|
||||
serializeResultToFile(resultFile, wordCount);
|
||||
});
|
||||
.reduce(
|
||||
(ReduceFunction<WordAndCount>)
|
||||
(oldValue, newValue) -> {
|
||||
LOGGER.info("reduce: {} {}", oldValue, newValue);
|
||||
return new WordAndCount(oldValue.word, oldValue.count + newValue.count);
|
||||
})
|
||||
.sink(
|
||||
s -> {
|
||||
LOGGER.info("sink {} {}", s.word, s.count);
|
||||
wordCount.put(s.word, s.count);
|
||||
serializeResultToFile(resultFile, wordCount);
|
||||
});
|
||||
|
||||
streamingContext.execute("testSQWordCount");
|
||||
|
||||
@@ -190,8 +193,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
checkWordCount = (Map<String, Integer>) deserializeResultFromFile(resultFile);
|
||||
}
|
||||
LOGGER.info("check");
|
||||
Assert.assertEquals(checkWordCount,
|
||||
ImmutableMap.of("eagle", 3, "hello", 1, "world", 1));
|
||||
Assert.assertEquals(checkWordCount, ImmutableMap.of("eagle", 3, "hello", 1, "world", 1));
|
||||
}
|
||||
|
||||
private void serializeResultToFile(String fileName, Object obj) {
|
||||
@@ -208,8 +210,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
|
||||
try {
|
||||
ObjectInputStream in = new ObjectInputStream(new FileInputStream(fileName));
|
||||
checkWordCount = (Map<String, Integer>) in.readObject();
|
||||
Assert.assertEquals(checkWordCount,
|
||||
ImmutableMap.of("eagle", 3, "hello", 1, "world", 1));
|
||||
Assert.assertEquals(checkWordCount, ImmutableMap.of("eagle", 3, "hello", 1, "world", 1));
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(String.valueOf(e));
|
||||
}
|
||||
|
||||
+24
-17
@@ -107,12 +107,15 @@ class ReaderWorker extends Worker {
|
||||
dataReader = new DataReader(inputQueueList, inputActors, new HashMap<>(), workerConfig);
|
||||
|
||||
// Should not GetBundle in RayCall thread
|
||||
Thread readThread = new Thread(Ray.wrapRunnable(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
consume();
|
||||
}
|
||||
}));
|
||||
Thread readThread =
|
||||
new Thread(
|
||||
Ray.wrapRunnable(
|
||||
new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
consume();
|
||||
}
|
||||
}));
|
||||
readThread.start();
|
||||
|
||||
LOGGER.info("ReaderWorker init done");
|
||||
@@ -136,8 +139,11 @@ class ReaderWorker extends Worker {
|
||||
int dataSize = dataMessage.body().getInt();
|
||||
|
||||
// check size
|
||||
LOGGER.info("capacity {} bufferSize {} dataSize {}",
|
||||
dataMessage.body().capacity(), bufferSize, dataSize);
|
||||
LOGGER.info(
|
||||
"capacity {} bufferSize {} dataSize {}",
|
||||
dataMessage.body().capacity(),
|
||||
bufferSize,
|
||||
dataSize);
|
||||
Assert.assertEquals(bufferSize, dataSize);
|
||||
if (dataMessage instanceof DataMessage) {
|
||||
if (LOGGER.isInfoEnabled()) {
|
||||
@@ -158,9 +164,7 @@ class ReaderWorker extends Worker {
|
||||
LOGGER.info("ReaderWorker consume data done.");
|
||||
}
|
||||
|
||||
void onQueueTransfer(long handler, byte[] buffer) {
|
||||
}
|
||||
|
||||
void onQueueTransfer(long handler, byte[] buffer) {}
|
||||
|
||||
public boolean done() {
|
||||
return totalMsg == msgCount;
|
||||
@@ -233,12 +237,15 @@ class WriterWorker extends Worker {
|
||||
new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessageSync", "([B)[B"));
|
||||
StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf);
|
||||
dataWriter = new DataWriter(outputQueueList, outputActors, new HashMap<>(), workerConfig);
|
||||
Thread writerThread = new Thread(Ray.wrapRunnable(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
produce();
|
||||
}
|
||||
}));
|
||||
Thread writerThread =
|
||||
new Thread(
|
||||
Ray.wrapRunnable(
|
||||
new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
produce();
|
||||
}
|
||||
}));
|
||||
writerThread.start();
|
||||
|
||||
LOGGER.info("WriterWorker init done");
|
||||
|
||||
+1
-3
@@ -2,7 +2,6 @@ package io.ray.streaming.runtime.transfer;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import io.ray.streaming.runtime.BaseUnitTest;
|
||||
import io.ray.streaming.runtime.transfer.channel.ChannelId;
|
||||
import io.ray.streaming.runtime.util.EnvUtil;
|
||||
@@ -20,5 +19,4 @@ public class ChannelIdTest extends BaseUnitTest {
|
||||
assertEquals(idStr.length(), ChannelId.ID_LENGTH * 2);
|
||||
assertEquals(ChannelId.idStrToBytes(idStr).length, ChannelId.ID_LENGTH);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+9
-22
@@ -10,23 +10,17 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.powermock.api.mockito.PowerMockito;
|
||||
|
||||
/**
|
||||
* Mockitools is a tool based on powermock and mokito to mock external service api
|
||||
*/
|
||||
/** Mockitools is a tool based on powermock and mokito to mock external service api */
|
||||
public class Mockitools {
|
||||
|
||||
/**
|
||||
* Mock GCS get node info api
|
||||
*/
|
||||
/** Mock GCS get node info api */
|
||||
public static void mockGscApi() {
|
||||
PowerMockito.mockStatic(RayUtils.class);
|
||||
PowerMockito.when(RayUtils.getAliveNodeInfoMap())
|
||||
.thenReturn(mockGetNodeInfoMap(mockGetAllNodeInfo()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mock get all node info from GCS
|
||||
*/
|
||||
/** Mock get all node info from GCS */
|
||||
public static List<NodeInfo> mockGetAllNodeInfo() {
|
||||
List<NodeInfo> nodeInfos = new LinkedList<>();
|
||||
|
||||
@@ -55,25 +49,18 @@ public class Mockitools {
|
||||
/**
|
||||
* Mock get node info map
|
||||
*
|
||||
* @param nodeInfos all node infos fetched from GCS
|
||||
* @return node info map, key is node unique id, value is node info
|
||||
* @param nodeInfos all node infos fetched from GCS Returns node info map, key is node unique id,
|
||||
* value is node info
|
||||
*/
|
||||
public static Map<UniqueId, NodeInfo> mockGetNodeInfoMap(List<NodeInfo> nodeInfos) {
|
||||
return nodeInfos.stream().filter(nodeInfo -> nodeInfo.isAlive).collect(
|
||||
Collectors.toMap(nodeInfo -> nodeInfo.nodeId, nodeInfo -> nodeInfo));
|
||||
return nodeInfos.stream()
|
||||
.filter(nodeInfo -> nodeInfo.isAlive)
|
||||
.collect(Collectors.toMap(nodeInfo -> nodeInfo.nodeId, nodeInfo -> nodeInfo));
|
||||
}
|
||||
|
||||
private static NodeInfo mockNodeInfo(int i, Map<String, Double> resources) {
|
||||
return new NodeInfo(
|
||||
createNodeId(i),
|
||||
"localhost" + i,
|
||||
"localhost" + i,
|
||||
-1,
|
||||
"",
|
||||
"",
|
||||
|
||||
true,
|
||||
resources);
|
||||
createNodeId(i), "localhost" + i, "localhost" + i, -1, "", "", true, resources);
|
||||
}
|
||||
|
||||
private static UniqueId createNodeId(int id) {
|
||||
|
||||
+7
-12
@@ -2,7 +2,6 @@ package io.ray.streaming.runtime.util;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Collections;
|
||||
import org.testng.annotations.Test;
|
||||
@@ -11,20 +10,16 @@ public class ReflectionUtilsTest {
|
||||
|
||||
static class Foo implements Serializable {
|
||||
|
||||
public void f1() {
|
||||
}
|
||||
public void f1() {}
|
||||
|
||||
public void f2() {
|
||||
}
|
||||
public void f2() {}
|
||||
|
||||
public void f2(boolean a1) {
|
||||
}
|
||||
public void f2(boolean a1) {}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFindMethod() throws NoSuchMethodException {
|
||||
assertEquals(Foo.class.getDeclaredMethod("f1"),
|
||||
ReflectionUtils.findMethod(Foo.class, "f1"));
|
||||
assertEquals(Foo.class.getDeclaredMethod("f1"), ReflectionUtils.findMethod(Foo.class, "f1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -34,7 +29,7 @@ public class ReflectionUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testGetAllInterfaces() {
|
||||
assertEquals(ReflectionUtils.getAllInterfaces(Foo.class),
|
||||
Collections.singletonList(Serializable.class));
|
||||
assertEquals(
|
||||
ReflectionUtils.getAllInterfaces(Foo.class), Collections.singletonList(Serializable.class));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user