Add basic functionality for Cython functions and actors (#1193)

* Add basic functionality for Cython functions and actors

* Fix up per @pcmoritz comments

* Fixes per @richardliaw comments

* Fixes per @robertnishihara comments

* Forgot double quotes when updating masked_log

* Remove import typing for Python 2 compatibility

python/ray/actor.py

@@ -15,7 +15,7 @@ import ray.local_scheduler
 import ray.signature as signature
 import ray.worker
 from ray.utils import (binary_to_hex, FunctionProperties, random_string,
-                       release_gpus_in_use, select_local_scheduler)
+                       release_gpus_in_use, select_local_scheduler, is_cython)
 
 
 def random_actor_id():
@@ -261,7 +261,8 @@ def fetch_and_register_actor(actor_class_key, worker):
         worker.actors[actor_id_str] = unpickled_class.__new__(unpickled_class)
         actor_methods = inspect.getmembers(
             unpickled_class, predicate=(lambda x: (inspect.isfunction(x) or
-                                                   inspect.ismethod(x))))
+                                                   inspect.ismethod(x) or
+                                                   is_cython(x))))
         for actor_method_name, actor_method in actor_methods:
             function_id = compute_actor_method_function_id(
                 class_name, actor_method_name).id()
@@ -682,7 +683,8 @@ def actor_handle_from_class(Class, class_id, num_cpus, num_gpus,
             # Get the actor methods of the given class.
             actor_methods = inspect.getmembers(
                 Class, predicate=(lambda x: (inspect.isfunction(x) or
-                                             inspect.ismethod(x))))
+                                             inspect.ismethod(x) or
+                                             is_cython(x))))
             # Extract the signatures of each of the methods. This will be used
             # to catch some errors if the methods are called with inappropriate
             # arguments.

python/ray/signature.py

@@ -5,6 +5,8 @@ from __future__ import print_function
 from collections import namedtuple
 import funcsigs
 
+from ray.utils import is_cython
+
 FunctionSignature = namedtuple("FunctionSignature", ["arg_names",
                                                      "arg_defaults",
                                                      "arg_is_positionals",
@@ -27,6 +29,42 @@ Attributes:
 """
 
 
+def get_signature_params(func):
+    """Get signature parameters
+
+    Support Cython functions by grabbing relevant attributes from the Cython
+    function and attaching to a no-op function. This is somewhat brittle, since
+    funcsigs may change, but given that funcsigs is written to a PEP, we hope
+    it is relatively stable. Future versions of Python may allow overloading
+    the inspect 'isfunction' and 'ismethod' functions / create ABC for Python
+    functions. Until then, it appears that Cython won't do anything about
+    compatability with the inspect module.
+
+    Args:
+        func: The function whose signature should be checked.
+
+    Raises:
+        TypeError: A type error if the signature is not supported
+    """
+    # The first condition for Cython functions, the latter for Cython instance
+    # methods
+    if is_cython(func):
+        attrs = ["__code__", "__annotations__",
+                 "__defaults__", "__kwdefaults__"]
+
+        if all([hasattr(func, attr) for attr in attrs]):
+            original_func = func
+
+            def func(): return
+            for attr in attrs:
+                setattr(func, attr, getattr(original_func, attr))
+        else:
+            raise TypeError("{0!r} is not a Python function we can process"
+                            .format(func))
+
+    return list(funcsigs.signature(func).parameters.items())
+
+
 def check_signature_supported(func, warn=False):
     """Check if we support the signature of this function.
 
@@ -43,8 +81,7 @@ def check_signature_supported(func, warn=False):
         Exception: An exception is raised if the signature is not supported.
     """
     function_name = func.__name__
-    sig_params = [(k, v) for k, v
-                  in funcsigs.signature(func).parameters.items()]
+    sig_params = get_signature_params(func)
 
     has_vararg_param = False
     has_kwargs_param = False
@@ -88,8 +125,7 @@ def extract_signature(func, ignore_first=False):
         A function signature object, which includes the names of the keyword
             arguments as well as their default values.
     """
-    sig_params = [(k, v) for k, v
-                  in funcsigs.signature(func).parameters.items()]
+    sig_params = get_signature_params(func)
 
     if ignore_first:
         if len(sig_params) == 0:

python/ray/utils.py

@@ -12,6 +12,21 @@ import sys
 import ray.local_scheduler
 
 
+def is_cython(obj):
+    """Check if an object is a Cython function or method"""
+
+    # TODO(suo): We could split these into two functions, one for Cython
+    # functions and another for Cython methods.
+    # TODO(suo): There doesn't appear to be a Cython function 'type' we can
+    # check against via isinstance. Please correct me if I'm wrong.
+    def check_cython(x):
+        return type(x).__name__ == "cython_function_or_method"
+
+    # Check if function or method, respectively
+    return check_cython(obj) or \
+        (hasattr(obj, "__func__") and check_cython(obj.__func__))
+
+
 def random_string():
     """Generate a random string to use as an ID.
 

python/ray/worker.py

@@ -28,7 +28,8 @@ import ray.services as services
 import ray.signature as signature
 import ray.local_scheduler
 import ray.plasma
-from ray.utils import FunctionProperties, random_string, binary_to_hex
+from ray.utils import (FunctionProperties, random_string, binary_to_hex,
+                       is_cython)
 
 SCRIPT_MODE = 0
 WORKER_MODE = 1
@@ -2281,7 +2282,8 @@ def export_remote_function(function_id, func_name, func, func_invoker,
     func_name_global_valid = func.__name__ in func.__globals__
     func_name_global_value = func.__globals__.get(func.__name__)
     # Allow the function to reference itself as a global variable
-    func.__globals__[func.__name__] = func_invoker
+    if not is_cython(func):
+        func.__globals__[func.__name__] = func_invoker
     try:
         pickled_func = pickle.dumps(func)
     finally:
@@ -2330,9 +2332,11 @@ def compute_function_id(func_name, func):
     function_id_hash.update(func_name.encode("ascii"))
     # If we are running a script or are in IPython, include the source code in
     # the hash. If we are in a regular Python interpreter we skip this part
-    # because the source code is not accessible.
+    # because the source code is not accessible. If the function is a built-in
+    # (e.g., Cython), the source code is not accessible.
     import __main__ as main
-    if hasattr(main, "__file__") or in_ipython():
+    if (hasattr(main, "__file__") or in_ipython()) \
+            and inspect.isfunction(func):
         function_id_hash.update(inspect.getsource(func).encode("ascii"))
     # Compute the function ID.
     function_id = function_id_hash.digest()
@@ -2364,7 +2368,7 @@ def remote(*args, **kwargs):
                               num_custom_resource, max_calls,
                               checkpoint_interval, func_id=None):
         def remote_decorator(func_or_class):
-            if inspect.isfunction(func_or_class):
+            if inspect.isfunction(func_or_class) or is_cython(func_or_class):
                 function_properties = FunctionProperties(
                     num_return_vals=num_return_vals,
                     num_cpus=num_cpus,
@@ -2420,7 +2424,7 @@ def remote(*args, **kwargs):
             func_invoker.is_remote = True
             func_name = "{}.{}".format(func.__module__, func.__name__)
             func_invoker.func_name = func_name
-            if sys.version_info >= (3, 0):
+            if sys.version_info >= (3, 0) or is_cython(func):
                 func_invoker.__doc__ = func.__doc__
             else:
                 func_invoker.func_doc = func.func_doc