wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-19 11:27:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Union api (#8612)

Browse Source
This commit is contained in:
chaokunyang
2020-06-08 14:28:11 +08:00
committed by GitHub
parent 3ee2e9f7e5
commit d04953ab3c
24 changed files with 579 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java
+42 -4
View File
@@ -4,16 +4,22 @@ import io.ray.api.Ray;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.function.impl.FilterFunction;
import io.ray.streaming.api.function.impl.MapFunction;
import io.ray.streaming.api.function.impl.SinkFunction;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.runtime.BaseUnitTest;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
public class HybridStreamTest extends BaseUnitTest implements Serializable {
public class HybridStreamTest {
private static final Logger LOG = LoggerFactory.getLogger(HybridStreamTest.class);
public static class Mapper1 implements MapFunction<Object, Object> {
@@ -34,9 +40,12 @@ public class HybridStreamTest extends BaseUnitTest implements Serializable {
}
}
@Test
public void testHybridDataStream() throws InterruptedException {
@Test(timeOut = 60000)
public void testHybridDataStream() throws Exception {
Ray.shutdown();
String sinkFileName = "/tmp/testHybridDataStream.txt";
Files.deleteIfExists(Paths.get(sinkFileName));
StreamingContext context = StreamingContext.buildContext();
DataStreamSource<String> streamSource =
DataStreamSource.fromCollection(context, Arrays.asList("a", "b", "c"));
@@ -46,9 +55,38 @@ public class HybridStreamTest extends BaseUnitTest implements Serializable {
.map("ray.streaming.tests.test_hybrid_stream", "map_func1")
.filter("ray.streaming.tests.test_hybrid_stream", "filter_func1")
.asJavaStream()
.sink(x -> System.out.println("HybridStreamTest: " + x));
.sink((SinkFunction<Object>) value -> {
LOG.info("HybridStreamTest: {}", value);
try {
if (!Files.exists(Paths.get(sinkFileName))) {
Files.createFile(Paths.get(sinkFileName));
}
Files.write(Paths.get(sinkFileName), value.toString().getBytes(),
StandardOpenOption.APPEND);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
context.execute("HybridStreamTestJob");
int sleptTime = 0;
TimeUnit.SECONDS.sleep(3);
while (true) {
if (Files.exists(Paths.get(sinkFileName))) {
TimeUnit.SECONDS.sleep(3);
String text = String.join(", ", Files.readAllLines(Paths.get(sinkFileName)));
Assert.assertTrue(text.contains("a"));
Assert.assertFalse(text.contains("b"));
Assert.assertTrue(text.contains("c"));
LOG.info("Execution succeed");
break;
}
sleptTime += 1;
if (sleptTime >= 60) {
throw new RuntimeException("Execution not finished");
}
LOG.info("Wait finish...");
TimeUnit.SECONDS.sleep(1);
}
context.stop();
LOG.info("HybridStreamTest succeed");
}
streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java
+71
View File
@@ -0,0 +1,71 @@
package io.ray.streaming.runtime.demo;
import io.ray.api.Ray;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.function.impl.SinkFunction;
import io.ray.streaming.api.stream.DataStreamSource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
public class UnionStreamTest {
private static final Logger LOG = LoggerFactory.getLogger( UnionStreamTest.class );
@Test(timeOut = 60000)
public void testUnionStream() throws Exception {
Ray.shutdown();
String sinkFileName = "/tmp/testUnionStream.txt";
Files.deleteIfExists(Paths.get(sinkFileName));
StreamingContext context = StreamingContext.buildContext();
DataStreamSource<Integer> streamSource1 =
DataStreamSource.fromCollection(context, Arrays.asList(1, 1));
DataStreamSource<Integer> streamSource2 =
DataStreamSource.fromCollection(context, Arrays.asList(1, 1));
DataStreamSource<Integer> streamSource3 =
DataStreamSource.fromCollection(context, Arrays.asList(1, 1));
streamSource1
.union(streamSource2, streamSource3)
.sink((SinkFunction<Integer>) value -> {
LOG.info("UnionStreamTest: {}", value);
try {
if (!Files.exists(Paths.get(sinkFileName))) {
Files.createFile(Paths.get(sinkFileName));
}
Files.write(Paths.get(sinkFileName), value.toString().getBytes(),
StandardOpenOption.APPEND);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
context.execute("UnionStreamTest");
int sleptTime = 0;
TimeUnit.SECONDS.sleep(3);
while (true) {
if (Files.exists(Paths.get(sinkFileName))) {
TimeUnit.SECONDS.sleep(3);
String text = String.join(", ", Files.readAllLines(Paths.get(sinkFileName)));
Assert.assertEquals(text, StringUtils.repeat("1", 6));
LOG.info("Execution succeed");
break;
}
sleptTime += 1;
if (sleptTime >= 60) {
throw new RuntimeException("Execution not finished");
}
LOG.info("Wait finish...");
TimeUnit.SECONDS.sleep(1);
}
context.stop();
LOG.info("HybridStreamTest succeed");
}
}