[Streaming] Streaming Cross-Lang API (#7464)

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/BaseUnitTest.java

@@ -24,11 +24,13 @@ public abstract class BaseUnitTest {
 
   @BeforeMethod
   public void testBegin(Method method) {
-    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: " + method.getName() + " began >>>>>>>>>>>>>>>>>>>>");
+    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>",
+        method.getDeclaringClass(),  method.getName());
   }
 
   @AfterMethod
   public void testEnd(Method method) {
-    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: " + method.getName() + " end >>>>>>>>>>>>>>>>>>");
+    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>",
+        method.getDeclaringClass(),  method.getName());
   }
 }

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java

@@ -80,7 +80,7 @@ public class ExecutionGraphTest extends BaseUnitTest {
 
   public static JobGraph buildJobGraph() {
     StreamingContext streamingContext = StreamingContext.buildContext();
-    DataStream<String> dataStream = DataStreamSource.buildSource(streamingContext,
+    DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
         Lists.newArrayList("a", "b", "c"));
     StreamSink streamSink = dataStream.sink(x -> LOG.info(x));
 

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java

@@ -0,0 +1,56 @@
+package io.ray.streaming.runtime.demo;
+
+import io.ray.api.Ray;
+import io.ray.streaming.api.context.StreamingContext;
+import io.ray.streaming.api.function.impl.FilterFunction;
+import io.ray.streaming.api.function.impl.MapFunction;
+import io.ray.streaming.api.stream.DataStreamSource;
+import io.ray.streaming.runtime.BaseUnitTest;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.Test;
+
+public class HybridStreamTest extends BaseUnitTest implements Serializable {
+  private static final Logger LOG = LoggerFactory.getLogger(HybridStreamTest.class);
+
+  public static class Mapper1 implements MapFunction<Object, Object> {
+
+    @Override
+    public Object map(Object value) {
+      LOG.info("HybridStreamTest Mapper1 {}", value);
+      return value.toString();
+    }
+  }
+
+  public static class Filter1 implements FilterFunction<Object> {
+
+    @Override
+    public boolean filter(Object value) throws Exception {
+      LOG.info("HybridStreamTest Filter1 {}", value);
+      return !value.toString().contains("b");
+    }
+  }
+
+  @Test
+  public void testHybridDataStream() throws InterruptedException {
+    Ray.shutdown();
+    StreamingContext context = StreamingContext.buildContext();
+    DataStreamSource<String> streamSource =
+        DataStreamSource.fromCollection(context, Arrays.asList("a", "b", "c"));
+    streamSource
+        .map(x -> x + x)
+        .asPythonStream()
+        .map("ray.streaming.tests.test_hybrid_stream", "map_func1")
+        .filter("ray.streaming.tests.test_hybrid_stream", "filter_func1")
+        .asJavaStream()
+        .sink(x -> System.out.println("HybridStreamTest: " + x));
+    context.execute("HybridStreamTestJob");
+    TimeUnit.SECONDS.sleep(3);
+    context.stop();
+    LOG.info("HybridStreamTest succeed");
+  }
+
+}

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java

@@ -1,6 +1,7 @@
 package io.ray.streaming.runtime.demo;
 
 import com.google.common.collect.ImmutableMap;
+import io.ray.api.Ray;
 import io.ray.streaming.api.context.StreamingContext;
 import io.ray.streaming.api.function.impl.FlatMapFunction;
 import io.ray.streaming.api.function.impl.ReduceFunction;
@@ -29,6 +30,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
 
   @Test
   public void testWordCount() {
+    Ray.shutdown();
     StreamingContext streamingContext = StreamingContext.buildContext();
     Map<String, String> config = new HashMap<>();
     config.put(Config.STREAMING_BATCH_MAX_COUNT, "1");
@@ -36,7 +38,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
     streamingContext.withConfig(config);
     List<String> text = new ArrayList<>();
     text.add("hello world eagle eagle eagle");
-    DataStreamSource<String> streamSource = DataStreamSource.buildSource(streamingContext, text);
+    DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
     streamSource
         .flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
           String[] records = value.split(" ");
@@ -62,6 +64,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
       }
     }
     Assert.assertEquals(wordCount, ImmutableMap.of("eagle", 3, "hello", 1));
+    streamingContext.stop();
   }
 
   private static class WordAndCount implements Serializable {

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/python/PythonGatewayTest.java

@@ -3,6 +3,7 @@ package io.ray.streaming.runtime.python;
 import io.ray.streaming.api.stream.StreamSink;
 import io.ray.streaming.jobgraph.JobGraph;
 import io.ray.streaming.jobgraph.JobGraphBuilder;
+import io.ray.streaming.runtime.serialization.MsgPackSerializer;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/schedule/TaskAssignerImplTest.java

@@ -57,7 +57,7 @@ public class TaskAssignerImplTest extends BaseUnitTest {
 
   public JobGraph buildDataSyncPlan() {
     StreamingContext streamingContext = StreamingContext.buildContext();
-    DataStream<String> dataStream = DataStreamSource.buildSource(streamingContext,
+    DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
         Lists.newArrayList("a", "b", "c"));
     DataStreamSink streamSink = dataStream.sink(LOGGER::info);
     JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink));

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/CrossLangSerializerTest.java

@@ -0,0 +1,26 @@
+package io.ray.streaming.runtime.serialization;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import io.ray.streaming.message.KeyRecord;
+import io.ray.streaming.message.Record;
+import org.testng.annotations.Test;
+
+public class CrossLangSerializerTest {
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testSerialize() {
+    CrossLangSerializer serializer = new CrossLangSerializer();
+    Record record = new Record("value");
+    record.setStream("stream1");
+    assertTrue(EqualsBuilder.reflectionEquals(record,
+      serializer.deserialize(serializer.serialize(record))));
+    KeyRecord keyRecord = new KeyRecord("key", "value");
+    keyRecord.setStream("stream2");
+    assertEquals(keyRecord,
+      serializer.deserialize(serializer.serialize(keyRecord)));
+  }
+}

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/MsgPackSerializerTest.java

@@ -1,4 +1,7 @@
-package io.ray.streaming.runtime.python;
+package io.ray.streaming.runtime.serialization;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -6,25 +9,37 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.testng.annotations.Test;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
 
 @SuppressWarnings("unchecked")
 public class MsgPackSerializerTest {
 
+  @Test
+  public void testSerializeByte() {
+    MsgPackSerializer serializer = new MsgPackSerializer();
+
+    assertEquals(serializer.deserialize(
+        serializer.serialize((byte)1)), (byte)1);
+  }
+
   @Test
   public void testSerialize() {
     MsgPackSerializer serializer = new MsgPackSerializer();
 
+    assertEquals(serializer.deserialize
+        (serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE);
+    assertEquals(serializer.deserialize(
+        serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE);
+    assertEquals(serializer.deserialize(
+        serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE);
+
     Map map = new HashMap();
     List list = new ArrayList<>();
     list.add(null);
     list.add(true);
-    list.add(1);
     list.add(1.0d);
     list.add("str");
     map.put("k1", "value1");
-    map.put("k2", 2);
+    map.put("k2", new HashMap<>());
     map.put("k3", list);
     byte[] bytes = serializer.serialize(map);
     Object o = serializer.deserialize(bytes);

streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java

@@ -5,6 +5,7 @@ import io.ray.api.Ray;
 import io.ray.api.RayActor;
 import io.ray.api.options.ActorCreationOptions;
 import io.ray.api.options.ActorCreationOptions.Builder;
+import io.ray.runtime.config.RayConfig;
 import io.ray.streaming.api.context.StreamingContext;
 import io.ray.streaming.api.function.impl.FlatMapFunction;
 import io.ray.streaming.api.function.impl.ReduceFunction;
@@ -67,7 +68,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
     System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
     System.setProperty("ray.run-mode", "CLUSTER");
     System.setProperty("ray.redirect-output", "true");
-    // ray init
+    RayConfig.reset();
     Ray.init();
   }
 
@@ -142,6 +143,14 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
 
   @Test(timeOut = 60000)
   public void testWordCount() {
+    Ray.shutdown();
+    System.setProperty("ray.resources", "CPU:4,RES-A:4");
+    System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
+
+    System.setProperty("ray.run-mode", "CLUSTER");
+    System.setProperty("ray.redirect-output", "true");
+    // ray init
+    Ray.init();
     LOGGER.info("testWordCount");
     LOGGER.info("StreamingQueueTest.testWordCount run-mode: {}",
         System.getProperty("ray.run-mode"));
@@ -157,7 +166,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
     streamingContext.withConfig(config);
     List<String> text = new ArrayList<>();
     text.add("hello world eagle eagle eagle");
-    DataStreamSource<String> streamSource = DataStreamSource.buildSource(streamingContext, text);
+    DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
     streamSource
         .flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
           String[] records = value.split(" ");
@@ -176,7 +185,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
           serializeResultToFile(resultFile, wordCount);
         });
 
-    streamingContext.execute("testWordCount");
+    streamingContext.execute("testSQWordCount");
 
     Map<String, Integer> checkWordCount =
         (Map<String, Integer>) deserializeResultFromFile(resultFile);