wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-19 11:27:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Allow users to specify the classpath and import path (#10560)

Browse Source 
* move job resource path to job config

* job resource path support list

* job resource path support for python

* fix job_resource_path support

* fix worker command

* fix job config

* use jar file instead of parent path

* fix job resource path

* add test to test.sh

* lint

* Update java/runtime/src/main/resources/ray.default.conf

Co-authored-by: Kai Yang <kfstorm@outlook.com>

* fix testGetFunctionFromLocalResource

* lint

* fix rebase

* add jars in resource path to classloader

* add job_resource_path to worker

* add ray stop

* rename job_resource_path to resource_path

* fix resource_path

* refine resource_path comments

* rename job resource path to code search path

* Add instruction about starting a cross-language cluster

* fix ClassLoaderTest.java

* add code-search-path to RunManager

* refine comments for code-search-path

* rename resourcePath to codeSearchPath

* Update doc

* fix

* rename resourcePath to codeSearchPath

* update doc

* filter out empty path

* fix comments

* fix comments

* fix tests

* revert pom

* lint

* fix doc

* update doc

* Apply suggestions from code review

* lint

Co-authored-by: Kai Yang <kfstorm@outlook.com>
Co-authored-by: Hao Chen <chenh1024@gmail.com>
This commit is contained in:
chaokunyang
2020-09-09 00:46:32 +08:00
committed by GitHub
co-authored by Kai Yang Hao Chen
parent 3645a05644
commit bbfbc98a41
22 changed files with 226 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/protobuf/gcs.proto
+4
View File
@@ -321,6 +321,10 @@ message JobConfig {
uint32 num_java_workers_per_process = 2;
// The jvm options for java workers of the job.
repeated string jvm_options = 3;
// A list of directories or jar files that specify the search path for user
// code. This will be used as `CLASSPATH` in Java, and `PYTHONPATH` in
// Python.
repeated string code_search_path = 4;
}
message JobTableData {
src/ray/raylet/worker_pool.cc
+27
View File
@@ -224,6 +224,33 @@ Process WorkerPool::StartWorkerProcess(const Language &language,
dynamic_options.insert(dynamic_options.begin(), job_config->jvm_options().begin(),
job_config->jvm_options().end());
}
// For non-multi-tenancy mode, job code search path is embedded in worker_command.
if (RayConfig::instance().enable_multi_tenancy() && job_config) {
std::string code_search_path_str;
for (int i = 0; i < job_config->code_search_path_size(); i++) {
auto path = job_config->code_search_path(i);
if (i != 0) {
code_search_path_str += ":";
}
code_search_path_str += path;
}
if (!code_search_path_str.empty()) {
switch (language) {
case Language::PYTHON: {
code_search_path_str = "--code-search-path=" + code_search_path_str;
break;
}
case Language::JAVA: {
code_search_path_str = "-Dray.job.code-search-path" + code_search_path_str;
break;
}
default:
RAY_LOG(FATAL) << "code_search_path is not supported for worker language "
<< language;
}
dynamic_options.push_back(code_search_path_str);
}
}
// Extract pointers from the worker command to pass into execvp.
std::vector<std::string> worker_command_args;