[Streaming] Streaming Cross-Lang API (#7464)

streaming/python/tests/test_function.py

@@ -14,9 +14,9 @@ class MapFunc(function.MapFunction):
 
 
 def test_load_function():
-    # function_bytes, module_name, class_name, function_name,
+    # function_bytes, module_name, function_name/class_name,
     # function_interface
     descriptor_func_bytes = gateway_client.serialize(
-        [None, __name__, MapFunc.__name__, None, "MapFunction"])
+        [None, __name__, MapFunc.__name__, "MapFunction"])
     func = function.load_function(descriptor_func_bytes)
     assert type(func) is MapFunc

streaming/python/tests/test_hybrid_stream.py

@@ -0,0 +1,70 @@
+import json
+import ray
+from ray.streaming import StreamingContext
+import subprocess
+import os
+
+
+def map_func1(x):
+    print("HybridStreamTest map_func1", x)
+    return str(x)
+
+
+def filter_func1(x):
+    print("HybridStreamTest filter_func1", x)
+    return "b" not in x
+
+
+def sink_func1(x):
+    print("HybridStreamTest sink_func1 value:", x)
+
+
+def test_hybrid_stream():
+    subprocess.check_call(
+        ["bazel", "build", "//streaming/java:all_streaming_tests_deploy.jar"])
+    current_dir = os.path.abspath(os.path.dirname(__file__))
+    jar_path = os.path.join(
+        current_dir,
+        "../../../bazel-bin/streaming/java/all_streaming_tests_deploy.jar")
+    jar_path = os.path.abspath(jar_path)
+    print("jar_path", jar_path)
+    java_worker_options = json.dumps(["-classpath", jar_path])
+    print("java_worker_options", java_worker_options)
+    assert not ray.is_initialized()
+    ray.init(
+        load_code_from_local=True,
+        include_java=True,
+        java_worker_options=java_worker_options,
+        _internal_config=json.dumps({
+            "num_workers_per_process_java": 1
+        }))
+
+    sink_file = "/tmp/ray_streaming_test_hybrid_stream.txt"
+    if os.path.exists(sink_file):
+        os.remove(sink_file)
+
+    def sink_func(x):
+        print("HybridStreamTest", x)
+        with open(sink_file, "a") as f:
+            f.write(str(x))
+
+    ctx = StreamingContext.Builder().build()
+    ctx.from_values("a", "b", "c") \
+        .as_java_stream() \
+        .map("io.ray.streaming.runtime.demo.HybridStreamTest$Mapper1") \
+        .filter("io.ray.streaming.runtime.demo.HybridStreamTest$Filter1") \
+        .as_python_stream() \
+        .sink(sink_func)
+    ctx.submit("HybridStreamTest")
+    import time
+    time.sleep(3)
+    ray.shutdown()
+    with open(sink_file, "r") as f:
+        result = f.read()
+        assert "a" in result
+        assert "b" not in result
+        assert "c" in result
+
+
+if __name__ == "__main__":
+    test_hybrid_stream()

streaming/python/tests/test_serialization.py

@@ -0,0 +1,13 @@
+from ray.streaming.runtime.serialization import CrossLangSerializer
+from ray.streaming.message import Record, KeyRecord
+
+
+def test_serialize():
+    serializer = CrossLangSerializer()
+    record = Record("value")
+    record.stream = "stream1"
+    key_record = KeyRecord("key", "value")
+    key_record.stream = "stream2"
+    assert record == serializer.deserialize(serializer.serialize(record))
+    assert key_record == serializer.\
+        deserialize(serializer.serialize(key_record))

streaming/python/tests/test_stream.py

@@ -0,0 +1,31 @@
+import ray
+from ray.streaming import StreamingContext
+
+
+def test_data_stream():
+    ray.init(load_code_from_local=True, include_java=True)
+    ctx = StreamingContext.Builder().build()
+    stream = ctx.from_values(1, 2, 3)
+    java_stream = stream.as_java_stream()
+    python_stream = java_stream.as_python_stream()
+    assert stream.get_id() == java_stream.get_id()
+    assert stream.get_id() == python_stream.get_id()
+    python_stream.set_parallelism(10)
+    assert stream.get_parallelism() == java_stream.get_parallelism()
+    assert stream.get_parallelism() == python_stream.get_parallelism()
+    ray.shutdown()
+
+
+def test_key_data_stream():
+    ray.init(load_code_from_local=True, include_java=True)
+    ctx = StreamingContext.Builder().build()
+    key_stream = ctx.from_values(
+        "a", "b", "c").map(lambda x: (x, 1)).key_by(lambda x: x[0])
+    java_stream = key_stream.as_java_stream()
+    python_stream = java_stream.as_python_stream()
+    assert key_stream.get_id() == java_stream.get_id()
+    assert key_stream.get_id() == python_stream.get_id()
+    python_stream.set_parallelism(10)
+    assert key_stream.get_parallelism() == java_stream.get_parallelism()
+    assert key_stream.get_parallelism() == python_stream.get_parallelism()
+    ray.shutdown()

streaming/python/tests/test_word_count.py

@@ -32,7 +32,9 @@ def test_simple_word_count():
 
     def sink_func(x):
         with open(sink_file, "a") as f:
-            f.write("{}:{},".format(x[0], x[1]))
+            line = "{}:{},".format(x[0], x[1])
+            print("sink_func", line)
+            f.write(line)
 
     ctx.from_values("a", "b", "c") \
         .set_parallelism(1) \