[Core] Allow users to specify the classpath and import path (#10560)

* move job resource path to job config

* job resource path support list

* job resource path support for python

* fix job_resource_path support

* fix worker command

* fix job config

* use jar file instead of parent path

* fix job resource path

* add test to test.sh

* lint

* Update java/runtime/src/main/resources/ray.default.conf

Co-authored-by: Kai Yang <kfstorm@outlook.com>

* fix testGetFunctionFromLocalResource

* lint

* fix rebase

* add jars in resource path to classloader

* add job_resource_path to worker

* add ray stop

* rename job_resource_path to resource_path

* fix resource_path

* refine resource_path comments

* rename job resource path to code search path

* Add instruction about starting a cross-language cluster

* fix ClassLoaderTest.java

* add code-search-path to RunManager

* refine comments for code-search-path

* rename resourcePath to codeSearchPath

* Update doc

* fix

* rename resourcePath to codeSearchPath

* update doc

* filter out empty path

* fix comments

* fix comments

* fix tests

* revert pom

* lint

* fix doc

* update doc

* Apply suggestions from code review

* lint

Co-authored-by: Kai Yang <kfstorm@outlook.com>
Co-authored-by: Hao Chen <chenh1024@gmail.com>

python/ray/job_config.py

@@ -10,6 +10,9 @@ class JobConfig:
         num_java_workers_per_process (int): The number of java workers per
             worker process.
         jvm_options (str[]): The jvm options for java workers of the job.
+        code_search_path (list): A list of directories or jar files that
+            specify the search path for user code. This will be used as
+            `CLASSPATH` in Java and `PYTHONPATH` in Python.
     """
 
     def __init__(
@@ -17,6 +20,7 @@ class JobConfig:
             worker_env=None,
             num_java_workers_per_process=1,
             jvm_options=None,
+            code_search_path=None,
     ):
         if worker_env is None:
             self.worker_env = dict()
@@ -27,6 +31,8 @@ class JobConfig:
             self.jvm_options = []
         else:
             self.jvm_options = jvm_options
+        if code_search_path is None:
+            self.code_search_path = []
 
     def serialize(self):
         job_config = ray.gcs_utils.JobConfig()
@@ -35,4 +41,5 @@ class JobConfig:
         job_config.num_java_workers_per_process = (
             self.num_java_workers_per_process)
         job_config.jvm_options.extend(self.jvm_options)
+        job_config.code_search_path.extend(self.code_search_path)
         return job_config.SerializeToString()