[Streaming] Streaming Python API (#6755)

streaming/java/streaming-runtime/src/test/java/org/ray/streaming/runtime/python/MsgPackSerializerTest.java

@@ -0,0 +1,39 @@
+package org.ray.streaming.runtime.python;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("unchecked")
+public class MsgPackSerializerTest {
+
+  @Test
+  public void testSerialize() {
+    MsgPackSerializer serializer = new MsgPackSerializer();
+
+    Map map = new HashMap();
+    List list = new ArrayList<>();
+    list.add(null);
+    list.add(true);
+    list.add(1);
+    list.add(1.0d);
+    list.add("str");
+    map.put("k1", "value1");
+    map.put("k2", 2);
+    map.put("k3", list);
+    byte[] bytes = serializer.serialize(map);
+    Object o = serializer.deserialize(bytes);
+    assertEquals(o, map);
+
+    byte[] binary = {1, 2, 3, 4};
+    assertTrue(Arrays.equals(
+        binary, (byte[]) (serializer.deserialize(serializer.serialize(binary)))));
+  }
+
+}

streaming/java/streaming-runtime/src/test/java/org/ray/streaming/runtime/python/PythonGatewayTest.java

@@ -0,0 +1,48 @@
+package org.ray.streaming.runtime.python;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.ray.streaming.api.stream.StreamSink;
+import org.ray.streaming.jobgraph.JobGraph;
+import org.ray.streaming.jobgraph.JobGraphBuilder;
+import org.testng.annotations.Test;
+
+public class PythonGatewayTest {
+
+  @Test
+  public void testPythonGateway() {
+    MsgPackSerializer serializer = new MsgPackSerializer();
+    PythonGateway gateway = new PythonGateway();
+    gateway.createStreamingContext();
+    Map<String, String> config = new HashMap<>();
+    config.put("k1", "v1");
+    gateway.withConfig(serializer.serialize(config));
+    byte[] mockPySource = new byte[0];
+    Object source = serializer.deserialize(
+        gateway.createPythonStreamSource(mockPySource));
+    byte[] mockPyFunc = new byte[0];
+    Object mapPyFunc = serializer.deserialize(gateway.createPyFunc(mockPyFunc));
+    Object mapStream = serializer.deserialize(
+        gateway.callMethod(
+            serializer.serialize(Arrays.asList(source, "map", mapPyFunc))));
+    byte[] mockPyPartition = new byte[0];
+    Object partition = serializer.deserialize(
+        gateway.createPyPartition(mockPyPartition));
+    Object partitionedStream = serializer.deserialize(
+        gateway.callMethod(
+            serializer.serialize(Arrays.asList(mapStream, "partitionBy", partition))));
+    byte[] mockSinkFunc = new byte[0];
+    Object sinkPyFunc = serializer.deserialize(gateway.createPyFunc(mockSinkFunc));
+    gateway.callMethod(
+        serializer.serialize(Arrays.asList(partitionedStream, "sink", sinkPyFunc)));
+    List<StreamSink> streamSinks = gateway.getStreamingContext().getStreamSinks();
+    assertEquals(streamSinks.size(), 1);
+    JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(streamSinks, "py_job");
+    JobGraph jobGraph = jobGraphBuilder.build();
+    jobGraph.printJobGraph();
+  }
+}

streaming/java/streaming-runtime/src/test/java/org/ray/streaming/runtime/schedule/TaskAssignerImplTest.java

@@ -1,26 +1,20 @@
 package org.ray.streaming.runtime.schedule;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import com.google.common.collect.Lists;
-import org.ray.api.RayActor;
-import org.ray.api.id.ActorId;
-import org.ray.api.id.ObjectId;
-import org.ray.runtime.actor.LocalModeRayActor;
+import java.util.List;
+import org.ray.api.Ray;
 import org.ray.streaming.api.context.StreamingContext;
 import org.ray.streaming.api.partition.impl.RoundRobinPartition;
 import org.ray.streaming.api.stream.DataStream;
 import org.ray.streaming.api.stream.DataStreamSink;
 import org.ray.streaming.api.stream.DataStreamSource;
+import org.ray.streaming.jobgraph.JobGraph;
+import org.ray.streaming.jobgraph.JobGraphBuilder;
 import org.ray.streaming.runtime.BaseUnitTest;
 import org.ray.streaming.runtime.core.graph.ExecutionEdge;
 import org.ray.streaming.runtime.core.graph.ExecutionGraph;
 import org.ray.streaming.runtime.core.graph.ExecutionNode;
 import org.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
-import org.ray.streaming.runtime.worker.JobWorker;
-import org.ray.streaming.jobgraph.JobGraph;
-import org.ray.streaming.jobgraph.JobGraphBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
@@ -32,15 +26,11 @@ public class TaskAssignerImplTest extends BaseUnitTest {
 
   @Test
   public void testTaskAssignImpl() {
+    Ray.init();
     JobGraph jobGraph = buildDataSyncPlan();
 
-    List<RayActor<JobWorker>> workers = new ArrayList<>();
-    for(int i = 0; i < jobGraph.getJobVertexList().size(); i++) {
-      workers.add(new LocalModeRayActor(ActorId.fromRandom(), ObjectId.fromRandom()));
-    }
-
     TaskAssigner taskAssigner = new TaskAssignerImpl();
-    ExecutionGraph executionGraph = taskAssigner.assign(jobGraph, workers);
+    ExecutionGraph executionGraph = taskAssigner.assign(jobGraph);
 
     List<ExecutionNode> executionNodeList = executionGraph.getExecutionNodeList();
 
@@ -61,16 +51,17 @@ public class TaskAssignerImplTest extends BaseUnitTest {
     Assert.assertEquals(sinkNode.getNodeType(), NodeType.SINK);
     Assert.assertEquals(sinkNode.getExecutionTasks().size(), 1);
     Assert.assertEquals(sinkNode.getOutputEdges().size(), 0);
+
+    Ray.shutdown();
   }
 
   public JobGraph buildDataSyncPlan() {
     StreamingContext streamingContext = StreamingContext.buildContext();
     DataStream<String> dataStream = DataStreamSource.buildSource(streamingContext,
         Lists.newArrayList("a", "b", "c"));
-    DataStreamSink streamSink = dataStream.sink(x -> LOGGER.info(x));
+    DataStreamSink streamSink = dataStream.sink(LOGGER::info);
     JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink));
 
-    JobGraph jobGraph = jobGraphBuilder.build();
-    return jobGraph;
+    return jobGraphBuilder.build();
   }
 }

streaming/java/streaming-runtime/src/test/java/org/ray/streaming/runtime/util/ReflectionUtilsTest.java

@@ -0,0 +1,38 @@
+package org.ray.streaming.runtime.util;
+
+import static org.testng.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.Collections;
+import org.testng.annotations.Test;
+
+public class ReflectionUtilsTest {
+
+  static class Foo implements Serializable {
+    public void f1() {
+    }
+
+    public void f2() {
+    }
+
+    public void f2(boolean a1) {
+    }
+  }
+
+  @Test
+  public void testFindMethod() throws NoSuchMethodException {
+    assertEquals(Foo.class.getDeclaredMethod("f1"),
+        ReflectionUtils.findMethod(Foo.class, "f1"));
+  }
+
+  @Test
+  public void testFindMethods() {
+    assertEquals(ReflectionUtils.findMethods(Foo.class, "f2").size(), 2);
+  }
+
+  @Test
+  public void testGetAllInterfaces() {
+    assertEquals(ReflectionUtils.getAllInterfaces(Foo.class),
+        Collections.singletonList(Serializable.class));
+  }
+}