Support multiple core workers in one process (#7623)

python/ray/includes/libcoreworker.pxd

@@ -17,7 +17,6 @@ from ray.includes.unique_ids cimport (
     CJobID,
     CTaskID,
     CObjectID,
-    CWorkerID,
 )
 from ray.includes.common cimport (
     CAddress,
@@ -80,31 +79,9 @@ cdef extern from "ray/core_worker/core_worker.h" nogil:
         c_string ExtensionData() const
 
     cdef cppclass CCoreWorker "ray::CoreWorker":
-        CCoreWorker(const CWorkerType worker_type, const CLanguage language,
-                    const c_string &store_socket,
-                    const c_string &raylet_socket, const CJobID &job_id,
-                    const CGcsClientOptions &gcs_options,
-                    const c_string &log_dir, const c_string &node_ip_address,
-                    int node_manager_port,
-                    CRayStatus (
-                        CTaskType task_type,
-                        const CRayFunction &ray_function,
-                        const unordered_map[c_string, double] &resources,
-                        const c_vector[shared_ptr[CRayObject]] &args,
-                        const c_vector[CObjectID] &arg_reference_ids,
-                        const c_vector[CObjectID] &return_ids,
-                        c_vector[shared_ptr[CRayObject]] *returns,
-                        const CWorkerID &worker_id) nogil,
-                    CRayStatus() nogil,
-                    void() nogil,
-                    void(c_string *stack_out) nogil,
-                    c_bool ref_counting_enabled,
-                    c_bool local_worker)
         CWorkerType &GetWorkerType()
         CLanguage &GetLanguage()
 
-        void StartExecutingTasks()
-
         CRayStatus SubmitTask(
             const CRayFunction &function, const c_vector[CTaskArg] &args,
             const CTaskOptions &options, c_vector[CObjectID] *return_ids,
@@ -206,3 +183,46 @@ cdef extern from "ray/core_worker/core_worker.h" nogil:
         void SetPlasmaAddedCallback(plasma_callback_function callback)
 
         void SubscribeToPlasmaAdd(const CObjectID &object_id)
+
+    cdef cppclass CCoreWorkerOptions "ray::CoreWorkerOptions":
+        CWorkerType worker_type
+        CLanguage language
+        c_string store_socket
+        c_string raylet_socket
+        CJobID job_id
+        CGcsClientOptions gcs_options
+        c_string log_dir
+        c_bool install_failure_signal_handler
+        c_string node_ip_address
+        int node_manager_port
+        c_string driver_name
+        c_string stdout_file
+        c_string stderr_file
+        (CRayStatus(
+            CTaskType task_type,
+            const CRayFunction &ray_function,
+            const unordered_map[c_string, double] &resources,
+            const c_vector[shared_ptr[CRayObject]] &args,
+            const c_vector[CObjectID] &arg_reference_ids,
+            const c_vector[CObjectID] &return_ids,
+            c_vector[shared_ptr[CRayObject]] *returns) nogil
+         ) task_execution_callback
+        (CRayStatus() nogil) check_signals
+        (void() nogil) gc_collect
+        (void(c_string *stack_out) nogil) get_lang_stack
+        c_bool ref_counting_enabled
+        c_bool is_local_mode
+        int num_workers
+        CCoreWorkerOptions()
+
+    cdef cppclass CCoreWorkerProcess "ray::CoreWorkerProcess":
+        @staticmethod
+        void Initialize(const CCoreWorkerOptions &options)
+        # Only call this in CoreWorker.__cinit__,
+        # use CoreWorker.core_worker to access C++ CoreWorker.
+        @staticmethod
+        CCoreWorker &GetCoreWorker()
+        @staticmethod
+        void Shutdown()
+        @staticmethod
+        void RunTaskExecutionLoop()