[Streaming] Streaming Cross-Lang API (#7464)

streaming/python/function.py

@@ -1,13 +1,19 @@
+import enum
 import importlib
 import inspect
 import sys
-from abc import ABC, abstractmethod
 import typing
+from abc import ABC, abstractmethod
 
 from ray import cloudpickle
 from ray.streaming.runtime import gateway_client
 
 
+class Language(enum.Enum):
+    JAVA = 0
+    PYTHON = 1
+
+
 class Function(ABC):
     """The base interface for all user-defined functions."""
 
@@ -60,6 +66,7 @@ class MapFunction(Function):
     for each input element.
     """
 
+    @abstractmethod
     def map(self, value):
         pass
 
@@ -70,6 +77,7 @@ class FlatMapFunction(Function):
     transform them into zero, one, or more elements.
     """
 
+    @abstractmethod
     def flat_map(self, value, collector):
         """Takes an element from the input data set and transforms it into zero,
         one, or more elements.
@@ -87,6 +95,7 @@ class FilterFunction(Function):
     The predicate decides whether to keep the element, or to discard it.
     """
 
+    @abstractmethod
     def filter(self, value):
         """The filter function that evaluates the predicate.
 
@@ -106,6 +115,7 @@ class KeyFunction(Function):
     deterministic key for that object.
     """
 
+    @abstractmethod
     def key_by(self, value):
         """User-defined function that deterministically extracts the key from
          an object.
@@ -126,6 +136,7 @@ class ReduceFunction(Function):
     them into one.
     """
 
+    @abstractmethod
     def reduce(self, old_value, new_value):
         """
         The core method of ReduceFunction, combining two values into one value
@@ -145,6 +156,7 @@ class ReduceFunction(Function):
 class SinkFunction(Function):
     """Interface for implementing user defined sink functionality."""
 
+    @abstractmethod
     def sink(self, value):
         """Writes the given value to the sink. This function is called for
          every record."""
@@ -283,7 +295,8 @@ def load_function(descriptor_func_bytes: bytes):
     Returns:
         a streaming function
     """
-    function_bytes, module_name, class_name, function_name, function_interface\
+    assert len(descriptor_func_bytes) > 0
+    function_bytes, module_name, function_name, function_interface\
         = gateway_client.deserialize(descriptor_func_bytes)
     if function_bytes:
         return deserialize(function_bytes)
@@ -292,16 +305,18 @@ def load_function(descriptor_func_bytes: bytes):
         assert function_interface
         function_interface = getattr(sys.modules[__name__], function_interface)
         mod = importlib.import_module(module_name)
-        if class_name:
-            assert function_name is None
-            cls = getattr(mod, class_name)
-            assert issubclass(cls, function_interface)
-            return cls()
-        else:
-            assert function_name
-            func = getattr(mod, function_name)
+        assert function_name
+        func = getattr(mod, function_name)
+        # If func is a python function, user function is a simple python
+        # function, which will be wrapped as a SimpleXXXFunction.
+        # If func is a python class, user function is a sub class
+        # of XXXFunction.
+        if inspect.isfunction(func):
             simple_func_class = _get_simple_function_class(function_interface)
             return simple_func_class(func)
+        else:
+            assert issubclass(func, function_interface)
+            return func()
 
 
 def _get_simple_function_class(function_interface):