[xlang] Cross language Python support (#6709)

src/ray/common/function_descriptor.cc

@@ -0,0 +1,78 @@
+#include "ray/common/function_descriptor.h"
+
+namespace ray {
+FunctionDescriptor FunctionDescriptorBuilder::Empty() {
+  static ray::FunctionDescriptor empty =
+      ray::FunctionDescriptor(new EmptyFunctionDescriptor());
+  return empty;
+}
+
+FunctionDescriptor FunctionDescriptorBuilder::BuildJava(const std::string &class_name,
+                                                        const std::string &function_name,
+                                                        const std::string &signature) {
+  rpc::FunctionDescriptor descriptor;
+  auto typed_descriptor = descriptor.mutable_java_function_descriptor();
+  typed_descriptor->set_class_name(class_name);
+  typed_descriptor->set_function_name(function_name);
+  typed_descriptor->set_signature(signature);
+  return ray::FunctionDescriptor(new JavaFunctionDescriptor(std::move(descriptor)));
+}
+
+FunctionDescriptor FunctionDescriptorBuilder::BuildPython(
+    const std::string &module_name, const std::string &class_name,
+    const std::string &function_name, const std::string &function_hash) {
+  rpc::FunctionDescriptor descriptor;
+  auto typed_descriptor = descriptor.mutable_python_function_descriptor();
+  typed_descriptor->set_module_name(module_name);
+  typed_descriptor->set_class_name(class_name);
+  typed_descriptor->set_function_name(function_name);
+  typed_descriptor->set_function_hash(function_hash);
+  return ray::FunctionDescriptor(new PythonFunctionDescriptor(std::move(descriptor)));
+}
+
+FunctionDescriptor FunctionDescriptorBuilder::FromProto(rpc::FunctionDescriptor message) {
+  switch (message.function_descriptor_case()) {
+  case ray::FunctionDescriptorType::kJavaFunctionDescriptor:
+    return ray::FunctionDescriptor(new ray::JavaFunctionDescriptor(std::move(message)));
+  case ray::FunctionDescriptorType::kPythonFunctionDescriptor:
+    return ray::FunctionDescriptor(new ray::PythonFunctionDescriptor(std::move(message)));
+  default:
+    break;
+  }
+  RAY_LOG(DEBUG) << "Unknown function descriptor case: "
+                 << message.function_descriptor_case();
+  // When TaskSpecification() constructed without function_descriptor set,
+  // we should return a valid ray::FunctionDescriptor instance.
+  return FunctionDescriptorBuilder::Empty();
+}
+
+FunctionDescriptor FunctionDescriptorBuilder::FromVector(
+    rpc::Language language, const std::vector<std::string> &function_descriptor_list) {
+  if (language == rpc::Language::JAVA) {
+    RAY_CHECK(function_descriptor_list.size() == 3);
+    return FunctionDescriptorBuilder::BuildJava(
+        function_descriptor_list[0],  // class name
+        function_descriptor_list[1],  // function name
+        function_descriptor_list[2]   // signature
+    );
+  } else if (language == rpc::Language::PYTHON) {
+    RAY_CHECK(function_descriptor_list.size() == 4);
+    return FunctionDescriptorBuilder::BuildPython(
+        function_descriptor_list[0],  // module name
+        function_descriptor_list[1],  // class name
+        function_descriptor_list[2],  // function name
+        function_descriptor_list[3]   // function hash
+    );
+  } else {
+    RAY_LOG(FATAL) << "Unspported language " << language;
+    return FunctionDescriptorBuilder::Empty();
+  }
+}
+
+FunctionDescriptor FunctionDescriptorBuilder::Deserialize(
+    const std::string &serialized_binary) {
+  rpc::FunctionDescriptor descriptor;
+  descriptor.ParseFromString(serialized_binary);
+  return FunctionDescriptorBuilder::FromProto(std::move(descriptor));
+}
+}  // namespace ray

src/ray/common/function_descriptor.h

@@ -0,0 +1,189 @@
+#ifndef RAY_CORE_WORKER_FUNCTION_DESCRIPTOR_H
+#define RAY_CORE_WORKER_FUNCTION_DESCRIPTOR_H
+
+#include <string>
+
+#include "ray/common/grpc_util.h"
+#include "ray/protobuf/common.pb.h"
+
+namespace ray {
+/// See `common.proto` for definition of `FunctionDescriptor` oneof type.
+using FunctionDescriptorType = rpc::FunctionDescriptor::FunctionDescriptorCase;
+/// Wrap a protobuf message.
+class FunctionDescriptorInterface : public MessageWrapper<rpc::FunctionDescriptor> {
+ public:
+  /// Construct an empty FunctionDescriptor.
+  FunctionDescriptorInterface() : MessageWrapper() {}
+
+  /// Construct from a protobuf message object.
+  /// The input message will be **copied** into this object.
+  ///
+  /// \param message The protobuf message.
+  FunctionDescriptorInterface(rpc::FunctionDescriptor message)
+      : MessageWrapper(std::move(message)) {}
+
+  ray::FunctionDescriptorType Type() const {
+    return message_->function_descriptor_case();
+  }
+
+  virtual size_t Hash() const = 0;
+
+  virtual std::string ToString() const = 0;
+
+  template <typename Subtype>
+  Subtype *As() {
+    return reinterpret_cast<Subtype *>(this);
+  }
+};
+
+class EmptyFunctionDescriptor : public FunctionDescriptorInterface {
+ public:
+  /// Construct from a protobuf message object.
+  /// The input message will be **copied** into this object.
+  ///
+  /// \param message The protobuf message.
+  explicit EmptyFunctionDescriptor() : FunctionDescriptorInterface() {
+    RAY_CHECK(message_->function_descriptor_case() ==
+              ray::FunctionDescriptorType::FUNCTION_DESCRIPTOR_NOT_SET);
+  }
+
+  virtual size_t Hash() const {
+    return std::hash<int>()(ray::FunctionDescriptorType::FUNCTION_DESCRIPTOR_NOT_SET);
+  }
+
+  virtual std::string ToString() const { return "{type=EmptyFunctionDescriptor}"; }
+};
+
+class JavaFunctionDescriptor : public FunctionDescriptorInterface {
+ public:
+  /// Construct from a protobuf message object.
+  /// The input message will be **copied** into this object.
+  ///
+  /// \param message The protobuf message.
+  explicit JavaFunctionDescriptor(rpc::FunctionDescriptor message)
+      : FunctionDescriptorInterface(std::move(message)) {
+    RAY_CHECK(message_->function_descriptor_case() ==
+              ray::FunctionDescriptorType::kJavaFunctionDescriptor);
+    typed_message_ = &(message_->java_function_descriptor());
+  }
+
+  virtual size_t Hash() const {
+    return std::hash<int>()(ray::FunctionDescriptorType::kJavaFunctionDescriptor) ^
+           std::hash<std::string>()(typed_message_->class_name()) ^
+           std::hash<std::string>()(typed_message_->function_name()) ^
+           std::hash<std::string>()(typed_message_->signature());
+  }
+
+  virtual std::string ToString() const {
+    return "{type=JavaFunctionDescriptor, class_name=" + typed_message_->class_name() +
+           ", function_name=" + typed_message_->function_name() +
+           ", signature=" + typed_message_->signature() + "}";
+  }
+
+  std::string ClassName() const { return typed_message_->class_name(); }
+
+  std::string FunctionName() const { return typed_message_->function_name(); }
+
+  std::string Signature() const { return typed_message_->signature(); }
+
+ private:
+  const rpc::JavaFunctionDescriptor *typed_message_;
+};
+
+class PythonFunctionDescriptor : public FunctionDescriptorInterface {
+ public:
+  /// Construct from a protobuf message object.
+  /// The input message will be **copied** into this object.
+  ///
+  /// \param message The protobuf message.
+  explicit PythonFunctionDescriptor(rpc::FunctionDescriptor message)
+      : FunctionDescriptorInterface(std::move(message)) {
+    RAY_CHECK(message_->function_descriptor_case() ==
+              ray::FunctionDescriptorType::kPythonFunctionDescriptor);
+    typed_message_ = &(message_->python_function_descriptor());
+  }
+
+  virtual size_t Hash() const {
+    return std::hash<int>()(ray::FunctionDescriptorType::kPythonFunctionDescriptor) ^
+           std::hash<std::string>()(typed_message_->module_name()) ^
+           std::hash<std::string>()(typed_message_->class_name()) ^
+           std::hash<std::string>()(typed_message_->function_name()) ^
+           std::hash<std::string>()(typed_message_->function_hash());
+  }
+
+  virtual std::string ToString() const {
+    return "{type=PythonFunctionDescriptor, module_name=" +
+           typed_message_->module_name() +
+           ", class_name=" + typed_message_->class_name() +
+           ", function_name=" + typed_message_->function_name() +
+           ", function_hash=" + typed_message_->function_hash() + "}";
+  }
+
+  std::string ModuleName() const { return typed_message_->module_name(); }
+
+  std::string ClassName() const { return typed_message_->class_name(); }
+
+  std::string FunctionName() const { return typed_message_->function_name(); }
+
+  std::string FunctionHash() const { return typed_message_->function_hash(); }
+
+ private:
+  const rpc::PythonFunctionDescriptor *typed_message_;
+};
+
+typedef std::shared_ptr<FunctionDescriptorInterface> FunctionDescriptor;
+
+inline bool operator==(const FunctionDescriptor &left, const FunctionDescriptor &right) {
+  if (left.get() != nullptr && right.get() != nullptr && left->Type() == right->Type() &&
+      left->ToString() == right->ToString()) {
+    return true;
+  }
+  return left.get() == right.get();
+}
+
+inline bool operator!=(const FunctionDescriptor &left, const FunctionDescriptor &right) {
+  return !(left == right);
+}
+
+/// Helper class for building a `FunctionDescriptor` object.
+class FunctionDescriptorBuilder {
+ public:
+  /// Build an EmptyFunctionDescriptor.
+  ///
+  /// \return a ray::EmptyFunctionDescriptor
+  static FunctionDescriptor Empty();
+
+  /// Build a JavaFunctionDescriptor.
+  ///
+  /// \return a ray::JavaFunctionDescriptor
+  static FunctionDescriptor BuildJava(const std::string &class_name,
+                                      const std::string &function_name,
+                                      const std::string &signature);
+
+  /// Build a PythonFunctionDescriptor.
+  ///
+  /// \return a ray::PythonFunctionDescriptor
+  static FunctionDescriptor BuildPython(const std::string &module_name,
+                                        const std::string &class_name,
+                                        const std::string &function_name,
+                                        const std::string &function_hash);
+
+  /// Build a ray::FunctionDescriptor according to input message.
+  ///
+  /// \return new ray::FunctionDescriptor
+  static FunctionDescriptor FromProto(rpc::FunctionDescriptor message);
+
+  /// Build a ray::FunctionDescriptor from language and vector.
+  ///
+  /// \return new ray::FunctionDescriptor
+  static FunctionDescriptor FromVector(
+      rpc::Language language, const std::vector<std::string> &function_descriptor_list);
+
+  /// Build a ray::FunctionDescriptor from serialized binary.
+  ///
+  /// \return new ray::FunctionDescriptor
+  static FunctionDescriptor Deserialize(const std::string &serialized_binary);
+};
+}  // namespace ray
+
+#endif

src/ray/common/task/task_spec.cc

@@ -75,8 +75,8 @@ TaskID TaskSpecification::ParentTaskId() const {
 
 size_t TaskSpecification::ParentCounter() const { return message_->parent_counter(); }
 
-std::vector<std::string> TaskSpecification::FunctionDescriptor() const {
-  return VectorFromProtobuf(message_->function_descriptor());
+ray::FunctionDescriptor TaskSpecification::FunctionDescriptor() const {
+  return ray::FunctionDescriptorBuilder::FromProto(message_->function_descriptor());
 }
 
 const SchedulingClass TaskSpecification::GetSchedulingClass() const {
@@ -145,8 +145,7 @@ const ResourceSet &TaskSpecification::GetRequiredPlacementResources() const {
 }
 
 bool TaskSpecification::IsDriverTask() const {
-  // Driver tasks are empty tasks that have no function ID set.
-  return FunctionDescriptor().empty();
+  return message_->type() == TaskType::DRIVER_TASK;
 }
 
 Language TaskSpecification::GetLanguage() const { return message_->language(); }
@@ -249,15 +248,7 @@ std::string TaskSpecification::DebugString() const {
          << ", function_descriptor=";
 
   // Print function descriptor.
-  const auto list = VectorFromProtobuf(message_->function_descriptor());
-  // The 4th is the code hash which is binary bits. No need to output it.
-  const size_t size = std::min(static_cast<size_t>(3), list.size());
-  for (size_t i = 0; i < size; ++i) {
-    if (i != 0) {
-      stream << ",";
-    }
-    stream << list[i];
-  }
+  stream << FunctionDescriptor()->ToString();
 
   stream << ", task_id=" << TaskId() << ", job_id=" << JobId()
          << ", num_args=" << NumArgs() << ", num_returns=" << NumReturns();

src/ray/common/task/task_spec.h

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "absl/synchronization/mutex.h"
+#include "ray/common/function_descriptor.h"
 #include "ray/common/grpc_util.h"
 #include "ray/common/id.h"
 #include "ray/common/task/scheduling_resources.h"
@@ -17,9 +18,7 @@ extern "C" {
 }
 
 namespace ray {
-
-typedef std::vector<std::string> FunctionDescriptor;
-typedef std::pair<ResourceSet, FunctionDescriptor> SchedulingClassDescriptor;
+typedef std::pair<ResourceSet, ray::FunctionDescriptor> SchedulingClassDescriptor;
 typedef int SchedulingClass;
 
 /// Wrapper class of protobuf `TaskSpec`, see `common.proto` for details.
@@ -63,7 +62,7 @@ class TaskSpecification : public MessageWrapper<rpc::TaskSpec> {
 
   size_t ParentCounter() const;
 
-  std::vector<std::string> FunctionDescriptor() const;
+  ray::FunctionDescriptor FunctionDescriptor() const;
 
   size_t NumArgs() const;
 
@@ -202,9 +201,7 @@ template <>
 struct hash<ray::SchedulingClassDescriptor> {
   size_t operator()(ray::SchedulingClassDescriptor const &k) const {
     size_t seed = std::hash<ray::ResourceSet>()(k.first);
-    for (const auto &str : k.second) {
-      seed ^= std::hash<std::string>()(str);
-    }
+    seed ^= k.second->Hash();
     return seed;
   }
 };

src/ray/common/task/task_util.h

@@ -25,16 +25,14 @@ class TaskSpecBuilder {
   /// \return Reference to the builder object itself.
   TaskSpecBuilder &SetCommonTaskSpec(
       const TaskID &task_id, const Language &language,
-      const std::vector<std::string> &function_descriptor, const JobID &job_id,
+      const ray::FunctionDescriptor &function_descriptor, const JobID &job_id,
       const TaskID &parent_task_id, uint64_t parent_counter, const TaskID &caller_id,
       const rpc::Address &caller_address, uint64_t num_returns, bool is_direct_call,
       const std::unordered_map<std::string, double> &required_resources,
       const std::unordered_map<std::string, double> &required_placement_resources) {
     message_->set_type(TaskType::NORMAL_TASK);
     message_->set_language(language);
-    for (const auto &fd : function_descriptor) {
-      message_->add_function_descriptor(fd);
-    }
+    *message_->mutable_function_descriptor() = function_descriptor->GetMessage();
     message_->set_job_id(job_id.Binary());
     message_->set_task_id(task_id.Binary());
     message_->set_parent_task_id(parent_task_id.Binary());
@@ -50,6 +48,27 @@ class TaskSpecBuilder {
     return *this;
   }
 
+  /// Set the driver attributes of the task spec.
+  /// See `common.proto` for meaning of the arguments.
+  ///
+  /// \return Reference to the builder object itself.
+  TaskSpecBuilder &SetDriverTaskSpec(const TaskID &task_id, const Language &language,
+                                     const JobID &job_id, const TaskID &parent_task_id,
+                                     const TaskID &caller_id,
+                                     const rpc::Address &caller_address) {
+    message_->set_type(TaskType::DRIVER_TASK);
+    message_->set_language(language);
+    message_->set_job_id(job_id.Binary());
+    message_->set_task_id(task_id.Binary());
+    message_->set_parent_task_id(parent_task_id.Binary());
+    message_->set_parent_counter(0);
+    message_->set_caller_id(caller_id.Binary());
+    message_->mutable_caller_address()->CopyFrom(caller_address);
+    message_->set_num_returns(0);
+    message_->set_is_direct_call(false);
+    return *this;
+  }
+
   /// Add a by-reference argument to the task.
   ///
   /// \param arg_id Id of the argument.

src/ray/core_worker/actor_handle.cc

@@ -7,14 +7,13 @@ namespace {
 ray::rpc::ActorHandle CreateInnerActorHandle(
     const class ActorID &actor_id, const class JobID &job_id,
     const ObjectID &initial_cursor, const Language actor_language, bool is_direct_call,
-    const std::vector<std::string> &actor_creation_task_function_descriptor) {
+    const ray::FunctionDescriptor &actor_creation_task_function_descriptor) {
   ray::rpc::ActorHandle inner;
   inner.set_actor_id(actor_id.Data(), actor_id.Size());
   inner.set_creation_job_id(job_id.Data(), job_id.Size());
   inner.set_actor_language(actor_language);
-  *inner.mutable_actor_creation_task_function_descriptor() = {
-      actor_creation_task_function_descriptor.begin(),
-      actor_creation_task_function_descriptor.end()};
+  *inner.mutable_actor_creation_task_function_descriptor() =
+      actor_creation_task_function_descriptor->GetMessage();
   inner.set_actor_cursor(initial_cursor.Binary());
   inner.set_is_direct_call(is_direct_call);
   return inner;
@@ -33,7 +32,7 @@ namespace ray {
 ActorHandle::ActorHandle(
     const class ActorID &actor_id, const class JobID &job_id,
     const ObjectID &initial_cursor, const Language actor_language, bool is_direct_call,
-    const std::vector<std::string> &actor_creation_task_function_descriptor)
+    const ray::FunctionDescriptor &actor_creation_task_function_descriptor)
     : ActorHandle(CreateInnerActorHandle(actor_id, job_id, initial_cursor, actor_language,
                                          is_direct_call,
                                          actor_creation_task_function_descriptor)) {}

src/ray/core_worker/actor_handle.h

@@ -21,7 +21,7 @@ class ActorHandle {
   ActorHandle(const ActorID &actor_id, const JobID &job_id,
               const ObjectID &initial_cursor, const Language actor_language,
               bool is_direct_call,
-              const std::vector<std::string> &actor_creation_task_function_descriptor);
+              const ray::FunctionDescriptor &actor_creation_task_function_descriptor);
 
   /// Constructs an ActorHandle from a serialized string.
   ActorHandle(const std::string &serialized);
@@ -34,8 +34,9 @@ class ActorHandle {
 
   Language ActorLanguage() const { return inner_.actor_language(); };
 
-  std::vector<std::string> ActorCreationTaskFunctionDescriptor() const {
-    return VectorFromProtobuf(inner_.actor_creation_task_function_descriptor());
+  ray::FunctionDescriptor ActorCreationTaskFunctionDescriptor() const {
+    return ray::FunctionDescriptorBuilder::FromProto(
+        inner_.actor_creation_task_function_descriptor());
   };
 
   bool IsDirectCallActor() const { return inner_.is_direct_call(); }

src/ray/core_worker/common.h

@@ -23,18 +23,18 @@ std::string LanguageString(Language language);
 class RayFunction {
  public:
   RayFunction() {}
-  RayFunction(Language language, const std::vector<std::string> &function_descriptor)
+  RayFunction(Language language, const ray::FunctionDescriptor &function_descriptor)
       : language_(language), function_descriptor_(function_descriptor) {}
 
   Language GetLanguage() const { return language_; }
 
-  const std::vector<std::string> &GetFunctionDescriptor() const {
+  const ray::FunctionDescriptor &GetFunctionDescriptor() const {
     return function_descriptor_;
   }
 
  private:
   Language language_;
-  std::vector<std::string> function_descriptor_;
+  ray::FunctionDescriptor function_descriptor_;
 };
 
 /// Argument of a task.

src/ray/core_worker/core_worker.cc

@@ -200,13 +200,10 @@ CoreWorker::CoreWorker(const WorkerType worker_type, const Language language,
   // rerun the driver.
   if (worker_type_ == WorkerType::DRIVER) {
     TaskSpecBuilder builder;
-    std::vector<std::string> empty_descriptor;
-    std::unordered_map<std::string, double> empty_resources;
     const TaskID task_id = TaskID::ForDriverTask(worker_context_.GetCurrentJobID());
-    builder.SetCommonTaskSpec(
-        task_id, language_, empty_descriptor, worker_context_.GetCurrentJobID(),
-        TaskID::ComputeDriverTaskId(worker_context_.GetWorkerID()), 0, GetCallerId(),
-        rpc_address_, 0, false, empty_resources, empty_resources);
+    builder.SetDriverTaskSpec(task_id, language_, worker_context_.GetCurrentJobID(),
+                              TaskID::ComputeDriverTaskId(worker_context_.GetWorkerID()),
+                              GetCallerId(), rpc_address_);
 
     std::shared_ptr<gcs::TaskTableData> data = std::make_shared<gcs::TaskTableData>();
     data->mutable_task()->mutable_task_spec()->CopyFrom(builder.Build().GetMessage());
@@ -1194,12 +1191,7 @@ void CoreWorker::HandleGetCoreWorkerStats(const rpc::GetCoreWorkerStatsRequest &
   stats->set_task_queue_length(task_queue_length_);
   stats->set_num_executed_tasks(num_executed_tasks_);
   stats->set_num_object_ids_in_scope(reference_counter_->NumObjectIDsInScope());
-  if (!current_task_.TaskId().IsNil()) {
-    stats->set_current_task_desc(current_task_.DebugString());
-    for (auto const it : current_task_.FunctionDescriptor()) {
-      stats->add_current_task_func_desc(it);
-    }
-  }
+  stats->set_current_task_func_desc(current_task_.FunctionDescriptor()->ToString());
   stats->set_ip_address(rpc_address_.ip_address());
   stats->set_port(rpc_address_.port());
   stats->set_actor_id(actor_id_.Binary());

src/ray/core_worker/lib/java/jni_utils.h

@@ -3,6 +3,7 @@
 
 #include <jni.h>
 #include "ray/common/buffer.h"
+#include "ray/common/function_descriptor.h"
 #include "ray/common/id.h"
 #include "ray/common/ray_object.h"
 #include "ray/common/status.h"
@@ -344,4 +345,26 @@ inline jobject NativeRayObjectToJavaNativeRayObject(
   return java_obj;
 }
 
+// TODO(po): Convert C++ ray::FunctionDescriptor to Java FunctionDescriptor
+inline jobject NativeRayFunctionDescriptorToJavaStringList(
+    JNIEnv *env, const ray::FunctionDescriptor &function_descriptor) {
+  if (function_descriptor->Type() ==
+      ray::FunctionDescriptorType::kJavaFunctionDescriptor) {
+    auto typed_descriptor = function_descriptor->As<ray::JavaFunctionDescriptor>();
+    std::vector<std::string> function_descriptor_list = {typed_descriptor->ClassName(),
+                                                         typed_descriptor->FunctionName(),
+                                                         typed_descriptor->Signature()};
+    return NativeStringVectorToJavaStringList(env, function_descriptor_list);
+  } else if (function_descriptor->Type() ==
+             ray::FunctionDescriptorType::kPythonFunctionDescriptor) {
+    auto typed_descriptor = function_descriptor->As<ray::PythonFunctionDescriptor>();
+    std::vector<std::string> function_descriptor_list = {
+        typed_descriptor->ModuleName(), typed_descriptor->ClassName(),
+        typed_descriptor->FunctionName(), typed_descriptor->FunctionHash()};
+    return NativeStringVectorToJavaStringList(env, function_descriptor_list);
+  }
+  RAY_LOG(FATAL) << "Unknown function descriptor type: " << function_descriptor->Type();
+  return NativeStringVectorToJavaStringList(env, std::vector<std::string>());
+}
+
 #endif  // RAY_COMMON_JAVA_JNI_UTILS_H

src/ray/core_worker/lib/java/org_ray_runtime_RayNativeRuntime.cc

@@ -44,8 +44,8 @@ JNIEXPORT jlong JNICALL Java_org_ray_runtime_RayNativeRuntime_nativeInitCoreWork
         RAY_CHECK(env);
         RAY_CHECK(local_java_task_executor);
         // convert RayFunction
-        jobject ray_function_array_list =
-            NativeStringVectorToJavaStringList(env, ray_function.GetFunctionDescriptor());
+        jobject ray_function_array_list = NativeRayFunctionDescriptorToJavaStringList(
+            env, ray_function.GetFunctionDescriptor());
         // convert args
         // TODO (kfstorm): Avoid copying binary data from Java to C++
         jobject args_array_list = NativeVectorToJavaList<std::shared_ptr<ray::RayObject>>(

src/ray/core_worker/lib/java/org_ray_runtime_actor_NativeRayActor.cc

@@ -33,7 +33,7 @@ Java_org_ray_runtime_actor_NativeRayActor_nativeGetActorCreationTaskFunctionDesc
                     .GetActorHandle(actor_id, &native_actor_handle);
   THROW_EXCEPTION_AND_RETURN_IF_NOT_OK(env, status, nullptr);
   auto function_descriptor = native_actor_handle->ActorCreationTaskFunctionDescriptor();
-  return NativeStringVectorToJavaStringList(env, function_descriptor);
+  return NativeRayFunctionDescriptorToJavaStringList(env, function_descriptor);
 }
 
 JNIEXPORT jbyteArray JNICALL Java_org_ray_runtime_actor_NativeRayActor_nativeSerialize(

src/ray/core_worker/lib/java/org_ray_runtime_task_NativeTaskSubmitter.cc

@@ -10,17 +10,20 @@ inline ray::CoreWorker &GetCoreWorker(jlong nativeCoreWorkerPointer) {
 }
 
 inline ray::RayFunction ToRayFunction(JNIEnv *env, jobject functionDescriptor) {
-  std::vector<std::string> function_descriptor;
+  std::vector<std::string> function_descriptor_list;
   jobject list =
       env->CallObjectMethod(functionDescriptor, java_function_descriptor_to_list);
   RAY_CHECK_JAVA_EXCEPTION(env);
-  JavaStringListToNativeStringVector(env, list, &function_descriptor);
+  JavaStringListToNativeStringVector(env, list, &function_descriptor_list);
   jobject java_language =
       env->CallObjectMethod(functionDescriptor, java_function_descriptor_get_language);
   RAY_CHECK_JAVA_EXCEPTION(env);
-  int language = env->CallIntMethod(java_language, java_language_get_number);
+  auto language = static_cast<::Language>(
+      env->CallIntMethod(java_language, java_language_get_number));
   RAY_CHECK_JAVA_EXCEPTION(env);
-  ray::RayFunction ray_function{static_cast<::Language>(language), function_descriptor};
+  ray::FunctionDescriptor function_descriptor =
+      ray::FunctionDescriptorBuilder::FromVector(language, function_descriptor_list);
+  ray::RayFunction ray_function{language, function_descriptor};
   return ray_function;
 }
 
@@ -134,7 +137,8 @@ JNIEXPORT jobject JNICALL Java_org_ray_runtime_task_NativeTaskSubmitter_nativeSu
 
   std::vector<ObjectID> return_ids;
   auto status = GetCoreWorker(nativeCoreWorkerPointer)
-                    .SubmitTask(ray_function, task_args, task_options, &return_ids, /*max_retries=*/1);
+                    .SubmitTask(ray_function, task_args, task_options, &return_ids,
+                                /*max_retries=*/1);
 
   THROW_EXCEPTION_AND_RETURN_IF_NOT_OK(env, status, nullptr);
 

src/ray/core_worker/test/core_worker_test.cc

@@ -51,14 +51,15 @@ ActorID CreateActorHelper(CoreWorker &worker,
   uint8_t array[] = {1, 2, 3};
   auto buffer = std::make_shared<LocalMemoryBuffer>(array, sizeof(array));
 
-  RayFunction func(ray::Language::PYTHON, {"actor creation task"});
+  RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                              "actor creation task", "", "", ""));
   std::vector<TaskArg> args;
   args.emplace_back(TaskArg::PassByValue(std::make_shared<RayObject>(buffer, nullptr)));
 
-  ActorCreationOptions actor_options{
-      max_reconstructions,   is_direct_call,
-      /*max_concurrency*/ 1, resources,           resources, {},
-      /*is_detached*/ false, /*is_asyncio*/ false};
+  ActorCreationOptions actor_options{max_reconstructions,   is_direct_call,
+                                     /*max_concurrency*/ 1, resources,      resources, {},
+                                     /*is_detached*/ false,
+                                     /*is_asyncio*/ false};
 
   // Create an actor.
   ActorID actor_id;
@@ -284,7 +285,8 @@ int CoreWorkerTest::GetActorPid(CoreWorker &worker, const ActorID &actor_id,
   std::vector<TaskArg> args;
   TaskOptions options{1, is_direct_call, resources};
   std::vector<ObjectID> return_ids;
-  RayFunction func{Language::PYTHON, {"GetWorkerPid"}};
+  RayFunction func{Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                         "GetWorkerPid", "", "", "")};
 
   RAY_CHECK_OK(worker.SubmitActorTask(actor_id, func, args, options, &return_ids));
 
@@ -321,7 +323,8 @@ void CoreWorkerTest::TestNormalTask(std::unordered_map<std::string, double> &res
           TaskArg::PassByValue(std::make_shared<RayObject>(buffer1, nullptr)));
       args.emplace_back(TaskArg::PassByReference(object_id));
 
-      RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+      RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                  "MergeInputArgsAsOutput", "", "", ""));
       TaskOptions options;
       options.is_direct_call = true;
 
@@ -369,7 +372,8 @@ void CoreWorkerTest::TestActorTask(std::unordered_map<std::string, double> &reso
 
       TaskOptions options{1, false, resources};
       std::vector<ObjectID> return_ids;
-      RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+      RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                  "MergeInputArgsAsOutput", "", "", ""));
 
       RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
       ASSERT_EQ(return_ids.size(), 1);
@@ -412,7 +416,8 @@ void CoreWorkerTest::TestActorTask(std::unordered_map<std::string, double> &reso
 
     TaskOptions options{1, false, resources};
     std::vector<ObjectID> return_ids;
-    RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+    RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                "MergeInputArgsAsOutput", "", "", ""));
     auto status = driver.SubmitActorTask(actor_id, func, args, options, &return_ids);
     ASSERT_TRUE(status.ok());
 
@@ -477,7 +482,8 @@ void CoreWorkerTest::TestActorReconstruction(
 
       TaskOptions options{1, false, resources};
       std::vector<ObjectID> return_ids;
-      RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+      RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                  "MergeInputArgsAsOutput", "", "", ""));
 
       RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
       ASSERT_EQ(return_ids.size(), 1);
@@ -522,7 +528,8 @@ void CoreWorkerTest::TestActorFailure(std::unordered_map<std::string, double> &r
 
       TaskOptions options{1, false, resources};
       std::vector<ObjectID> return_ids;
-      RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+      RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                  "MergeInputArgsAsOutput", "", "", ""));
 
       RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
 
@@ -587,7 +594,8 @@ TEST_F(ZeroNodeTest, TestTaskSpecPerf) {
   // to benchmark performance.
   uint8_t array[] = {1, 2, 3};
   auto buffer = std::make_shared<LocalMemoryBuffer>(array, sizeof(array));
-  RayFunction function(ray::Language::PYTHON, {});
+  RayFunction function(ray::Language::PYTHON,
+                       ray::FunctionDescriptorBuilder::BuildPython("", "", "", ""));
   std::vector<TaskArg> args;
   args.emplace_back(TaskArg::PassByValue(std::make_shared<RayObject>(buffer, nullptr)));
 
@@ -670,7 +678,8 @@ TEST_F(SingleNodeTest, TestDirectActorTaskSubmissionPerf) {
 
     TaskOptions options{1, false, resources};
     std::vector<ObjectID> return_ids;
-    RayFunction func(ray::Language::PYTHON, {"MergeInputArgsAsOutput"});
+    RayFunction func(ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
+                                                "MergeInputArgsAsOutput", "", "", ""));
 
     RAY_CHECK_OK(driver.SubmitActorTask(actor_id, func, args, options, &return_ids));
     ASSERT_EQ(return_ids.size(), 1);
@@ -717,7 +726,7 @@ TEST_F(ZeroNodeTest, TestActorHandle) {
   JobID job_id = NextJobId();
   ActorHandle original(ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 0), job_id,
                        ObjectID::FromRandom(), Language::PYTHON, /*is_direct_call=*/false,
-                       {});
+                       ray::FunctionDescriptorBuilder::BuildPython("", "", "", ""));
   std::string output;
   original.Serialize(&output);
   ActorHandle deserialized(output);

src/ray/core_worker/test/direct_task_transport_test.cc

@@ -241,7 +241,7 @@ TEST(LocalDependencyResolverTest, TestInlinePendingDependencies) {
 }
 
 TaskSpecification BuildTaskSpec(const std::unordered_map<std::string, double> &resources,
-                                const std::vector<std::string> &function_descriptor) {
+                                const ray::FunctionDescriptor &function_descriptor) {
   TaskSpecBuilder builder;
   rpc::Address empty_address;
   builder.SetCommonTaskSpec(TaskID::Nil(), Language::PYTHON, function_descriptor,
@@ -261,7 +261,8 @@ TEST(DirectTaskTransportTest, TestSubmitOneTask) {
                                           task_finisher, ClientID::Nil(), kLongTimeout);
 
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task = BuildTaskSpec(empty_resources, empty_descriptor);
 
   ASSERT_TRUE(submitter.SubmitTask(task).ok());
@@ -291,7 +292,8 @@ TEST(DirectTaskTransportTest, TestHandleTaskFailure) {
   CoreWorkerDirectTaskSubmitter submitter(address, raylet_client, factory, nullptr, store,
                                           task_finisher, ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task = BuildTaskSpec(empty_resources, empty_descriptor);
 
   ASSERT_TRUE(submitter.SubmitTask(task).ok());
@@ -315,7 +317,8 @@ TEST(DirectTaskTransportTest, TestConcurrentWorkerLeases) {
   CoreWorkerDirectTaskSubmitter submitter(address, raylet_client, factory, nullptr, store,
                                           task_finisher, ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task1 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task2 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task3 = BuildTaskSpec(empty_resources, empty_descriptor);
@@ -360,7 +363,8 @@ TEST(DirectTaskTransportTest, TestReuseWorkerLease) {
   CoreWorkerDirectTaskSubmitter submitter(address, raylet_client, factory, nullptr, store,
                                           task_finisher, ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task1 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task2 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task3 = BuildTaskSpec(empty_resources, empty_descriptor);
@@ -408,7 +412,8 @@ TEST(DirectTaskTransportTest, TestWorkerNotReusedOnError) {
   CoreWorkerDirectTaskSubmitter submitter(address, raylet_client, factory, nullptr, store,
                                           task_finisher, ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task1 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task2 = BuildTaskSpec(empty_resources, empty_descriptor);
 
@@ -446,7 +451,8 @@ TEST(DirectTaskTransportTest, TestWorkerNotReturnedOnExit) {
   CoreWorkerDirectTaskSubmitter submitter(address, raylet_client, factory, nullptr, store,
                                           task_finisher, ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task1 = BuildTaskSpec(empty_resources, empty_descriptor);
 
   ASSERT_TRUE(submitter.SubmitTask(task1).ok());
@@ -484,7 +490,8 @@ TEST(DirectTaskTransportTest, TestSpillback) {
                                           lease_client_factory, store, task_finisher,
                                           ClientID::Nil(), kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task = BuildTaskSpec(empty_resources, empty_descriptor);
 
   ASSERT_TRUE(submitter.SubmitTask(task).ok());
@@ -534,7 +541,8 @@ TEST(DirectTaskTransportTest, TestSpillbackRoundTrip) {
                                           lease_client_factory, store, task_finisher,
                                           local_raylet_id, kLongTimeout);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task = BuildTaskSpec(empty_resources, empty_descriptor);
 
   ASSERT_TRUE(submitter.SubmitTask(task).ok());
@@ -620,8 +628,10 @@ TEST(DirectTaskTransportTest, TestSchedulingKeys) {
 
   std::unordered_map<std::string, double> resources1({{"a", 1.0}});
   std::unordered_map<std::string, double> resources2({{"b", 2.0}});
-  std::vector<std::string> descriptor1({"a"});
-  std::vector<std::string> descriptor2({"b"});
+  ray::FunctionDescriptor descriptor1 =
+      ray::FunctionDescriptorBuilder::BuildPython("a", "", "", "");
+  ray::FunctionDescriptor descriptor2 =
+      ray::FunctionDescriptorBuilder::BuildPython("b", "", "", "");
 
   // Tasks with different resources should request different worker leases.
   RAY_LOG(INFO) << "Test different resources";
@@ -682,7 +692,8 @@ TEST(DirectTaskTransportTest, TestWorkerLeaseTimeout) {
                                           task_finisher, ClientID::Nil(),
                                           /*lease_timeout_ms=*/5);
   std::unordered_map<std::string, double> empty_resources;
-  std::vector<std::string> empty_descriptor;
+  ray::FunctionDescriptor empty_descriptor =
+      ray::FunctionDescriptorBuilder::BuildPython("", "", "", "");
   TaskSpecification task1 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task2 = BuildTaskSpec(empty_resources, empty_descriptor);
   TaskSpecification task3 = BuildTaskSpec(empty_resources, empty_descriptor);

src/ray/core_worker/test/mock_worker.cc

@@ -35,20 +35,23 @@ class MockWorker {
                      const std::vector<ObjectID> &return_ids,
                      std::vector<std::shared_ptr<RayObject>> *results) {
     // Note that this doesn't include dummy object id.
-    const std::vector<std::string> &function_descriptor =
+    const ray::FunctionDescriptor function_descriptor =
         ray_function.GetFunctionDescriptor();
-    RAY_CHECK(return_ids.size() >= 0 && 1 == function_descriptor.size());
+    RAY_CHECK(function_descriptor->Type() ==
+              ray::FunctionDescriptorType::kPythonFunctionDescriptor);
+    auto typed_descriptor = function_descriptor->As<ray::PythonFunctionDescriptor>();
 
-    if ("actor creation task" == function_descriptor[0]) {
+    if ("actor creation task" == typed_descriptor->ModuleName()) {
       return Status::OK();
-    } else if ("GetWorkerPid" == function_descriptor[0]) {
+    } else if ("GetWorkerPid" == typed_descriptor->ModuleName()) {
       // Get mock worker pid
       return GetWorkerPid(results);
-    } else if ("MergeInputArgsAsOutput" == function_descriptor[0]) {
+    } else if ("MergeInputArgsAsOutput" == typed_descriptor->ModuleName()) {
       // Merge input args and write the merged content to each of return ids
       return MergeInputArgsAsOutput(args, return_ids, results);
     } else {
-      return Status::TypeError("Unknown function descriptor: " + function_descriptor[0]);
+      return Status::TypeError("Unknown function descriptor: " +
+                               typed_descriptor->ModuleName());
     }
   }
 

src/ray/protobuf/common.proto

@@ -25,6 +25,8 @@ enum TaskType {
   ACTOR_CREATION_TASK = 1;
   // Actor task.
   ACTOR_TASK = 2;
+  // Driver task.
+  DRIVER_TASK = 3;
 }
 
 // Address of a worker or node manager.
@@ -36,6 +38,29 @@ message Address {
   bytes worker_id = 4;
 }
 
+/// Function descriptor for Java.
+message JavaFunctionDescriptor {
+  string class_name = 1;
+  string function_name = 2;
+  string signature = 3;
+}
+
+/// Function descriptor for Python.
+message PythonFunctionDescriptor {
+  string module_name = 1;
+  string class_name = 2;
+  string function_name = 3;
+  string function_hash = 4;
+}
+
+// A union wrapper for various function descriptor types.
+message FunctionDescriptor {
+  oneof function_descriptor {
+    JavaFunctionDescriptor java_function_descriptor = 1;
+    PythonFunctionDescriptor python_function_descriptor = 2;
+  }
+}
+
 /// The task specification encapsulates all immutable information about the
 /// task. These fields are determined at submission time, converse to the
 /// `TaskExecutionSpec` may change at execution time.
@@ -44,11 +69,8 @@ message TaskSpec {
   TaskType type = 1;
   // Language of this task.
   Language language = 2;
-  // Function descriptor of this task, which is a list of strings that can
-  // uniquely describe the function to execute.
-  // For a Python function, it should be: [module_name, class_name, function_name]
-  // For a Java function, it should be: [class_name, method_name, type_descriptor]
-  repeated bytes function_descriptor = 3;
+  // Function descriptor of this task uniquely describe the function to execute.
+  FunctionDescriptor function_descriptor = 3;
   // ID of the job that this task belongs to.
   bytes job_id = 4;
   // Task ID of the task.
@@ -194,8 +216,8 @@ message CoreWorkerStats {
   int32 num_pending_tasks = 2;
   // Number of object ids in local scope.
   int32 num_object_ids_in_scope = 3;
-  // Function descriptor of the currently executing task.
-  repeated bytes current_task_func_desc = 4;
+  // String representation of the function descriptor of the currently executing task.
+  string current_task_func_desc = 4;
   // IP address of the core worker.
   string ip_address = 6;
   // Port of the core worker.

src/ray/protobuf/core_worker.proto

@@ -21,7 +21,7 @@ message ActorHandle {
   Language actor_language = 4;
 
   // Function descriptor of actor creation task.
-  repeated string actor_creation_task_function_descriptor = 5;
+  FunctionDescriptor actor_creation_task_function_descriptor = 5;
 
   // The unique id of the dummy object returned by the actor creation task.
   // It's used as a dependency for the first task.

src/ray/ray_exported_symbols.lds

@@ -7,6 +7,7 @@
 *ray*RayObject*
 *ray*Status*
 *ray*RayFunction*
+*ray*FunctionDescriptorBuilder*
 *ray*TaskArg*
 *ray*TaskOptions*
 *ray*Buffer*

src/ray/ray_version_script.lds

@@ -9,6 +9,7 @@ VERSION_1.0 {
         *ray*RayObject*;
         *ray*Status*;
         *ray*RayFunction*;
+        *ray*FunctionDescriptorBuilder*;
         *ray*TaskArg*;
         *ray*TaskOptions*;
         *ray*Buffer*;

src/ray/raylet/lineage_cache_test.cc

@@ -183,9 +183,10 @@ static inline Task ExampleTask(const std::vector<ObjectID> &arguments,
                                uint64_t num_returns) {
   TaskSpecBuilder builder;
   rpc::Address address;
-  builder.SetCommonTaskSpec(RandomTaskId(), Language::PYTHON, {"", "", ""}, JobID::Nil(),
-                            RandomTaskId(), 0, RandomTaskId(), address, num_returns,
-                            false, {}, {});
+  builder.SetCommonTaskSpec(RandomTaskId(), Language::PYTHON,
+                            ray::FunctionDescriptorBuilder::BuildPython("", "", "", ""),
+                            JobID::Nil(), RandomTaskId(), 0, RandomTaskId(), address,
+                            num_returns, false, {}, {});
   for (const auto &arg : arguments) {
     builder.AddByRefArg(arg);
   }

src/ray/raylet/scheduling_queue.cc

@@ -447,19 +447,7 @@ std::string SchedulingQueue::DebugString() const {
   for (const auto &pair : num_running_tasks_) {
     result << "\n- ";
     auto desc = TaskSpecification::GetSchedulingClassDescriptor(pair.first);
-    for (const auto &str : desc.second) {
-      // Only print the ASCII parts of the function descriptor.
-      bool ok = str.size() > 0;
-      for (char c : str) {
-        if (!isprint(c)) {
-          ok = false;
-        }
-      }
-      if (ok) {
-        result << str;
-        result << ".";
-      }
-    }
+    result << desc.second->ToString();
     result << desc.first.ToString();
     result << ": " << pair.second;
     total += pair.second;

src/ray/raylet/task_dependency_manager_test.cc

@@ -94,9 +94,10 @@ static inline Task ExampleTask(const std::vector<ObjectID> &arguments,
                                uint64_t num_returns) {
   TaskSpecBuilder builder;
   rpc::Address address;
-  builder.SetCommonTaskSpec(RandomTaskId(), Language::PYTHON, {"", "", ""}, JobID::Nil(),
-                            RandomTaskId(), 0, RandomTaskId(), address, num_returns,
-                            false, {}, {});
+  builder.SetCommonTaskSpec(RandomTaskId(), Language::PYTHON,
+                            FunctionDescriptorBuilder::BuildPython("", "", "", ""),
+                            JobID::Nil(), RandomTaskId(), 0, RandomTaskId(), address,
+                            num_returns, false, {}, {});
   for (const auto &arg : arguments) {
     builder.AddByRefArg(arg);
   }