wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 21:08:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core] Allow users to specify the classpath and import path (#10560)

Browse Source 
* move job resource path to job config

* job resource path support list

* job resource path support for python

* fix job_resource_path support

* fix worker command

* fix job config

* use jar file instead of parent path

* fix job resource path

* add test to test.sh

* lint

* Update java/runtime/src/main/resources/ray.default.conf

Co-authored-by: Kai Yang <kfstorm@outlook.com>

* fix testGetFunctionFromLocalResource

* lint

* fix rebase

* add jars in resource path to classloader

* add job_resource_path to worker

* add ray stop

* rename job_resource_path to resource_path

* fix resource_path

* refine resource_path comments

* rename job resource path to code search path

* Add instruction about starting a cross-language cluster

* fix ClassLoaderTest.java

* add code-search-path to RunManager

* refine comments for code-search-path

* rename resourcePath to codeSearchPath

* Update doc

* fix

* rename resourcePath to codeSearchPath

* update doc

* filter out empty path

* fix comments

* fix comments

* fix tests

* revert pom

* lint

* fix doc

* update doc

* Apply suggestions from code review

* lint

Co-authored-by: Kai Yang <kfstorm@outlook.com>
Co-authored-by: Hao Chen <chenh1024@gmail.com>
This commit is contained in:
chaokunyang
2020-09-09 00:46:32 +08:00
committed by GitHub
parent 3645a05644
commit bbfbc98a41
22 changed files with 226 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java
+7 -6
View File
@@ -8,6 +8,7 @@ import io.ray.runtime.config.RayConfig;
import io.ray.runtime.util.NetworkUtil;
import java.io.File;
import java.lang.ProcessBuilder.Redirect;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -66,12 +67,11 @@ public abstract class BaseMultiLanguageTest {
// jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly.
// Since mvn test classes contains `test` in path and bazel test classes is located at a jar
// with `test` included in the name, we can check classpath `test` to filter out test classes.
String classpath = Stream.of(System.getProperty("java.class.path").split(":"))
List<String> classpath = Stream.of(System.getProperty("java.class.path").split(":"))
.filter(s -> !s.contains(" ") && s.contains("test"))
.collect(Collectors.joining(":"));
String workerOptions = new Gson().toJson(ImmutableList.of("-classpath", classpath));
.collect(Collectors.toList());
// Start ray cluster.
List<String> startCommand = ImmutableList.of(
List<String> startCommand = Arrays.asList(
"ray",
"start",
"--head",
@@ -83,9 +83,10 @@ public abstract class BaseMultiLanguageTest {
String.format("--node-manager-port=%s", nodeManagerPort),
"--load-code-from-local",
"--include-java",
"--java-worker-options=" + workerOptions,
"--system-config=" + new Gson().toJson(RayConfig.create().rayletConfigParameters)
"--system-config=" + new Gson().toJson(RayConfig.create().rayletConfigParameters),
"--code-search-path=" + String.join(":", classpath)
);
if (!executeCommand(startCommand, 10, getRayStartEnv())) {
throw new RuntimeException("Couldn't start ray cluster.");
}
java/test/src/main/java/io/ray/test/ClassLoaderTest.java
+11 -12
View File
@@ -3,7 +3,6 @@ package io.ray.test;
import io.ray.api.ActorHandle;
import io.ray.api.BaseActorHandle;
import io.ray.api.ObjectRef;
import io.ray.api.Ray;
import io.ray.api.options.ActorCreationOptions;
import io.ray.runtime.AbstractRayRuntime;
import io.ray.runtime.functionmanager.FunctionDescriptor;
@@ -23,31 +22,31 @@ import org.testng.annotations.Test;
public class ClassLoaderTest extends BaseTest {
private final String resourcePath = FileUtils.getTempDirectoryPath()
private final String codeSearchPath = FileUtils.getTempDirectoryPath()
+ "/ray_test/ClassLoaderTest";
@BeforeClass
public void setUp() {
// The potential issue of multiple `ClassLoader` instances for the same job on multi-threading
// scenario only occurs if the classes are loaded from the job resource path.
System.setProperty("ray.job.resource-path", resourcePath);
// scenario only occurs if the classes are loaded from the job code search path.
System.setProperty("ray.job.code-search-path", codeSearchPath);
}
@AfterClass
public void tearDown() {
System.clearProperty("ray.job.resource-path");
System.clearProperty("ray.job.code-search-path");
}
@Test(groups = {"cluster"})
public void testClassLoaderInMultiThreading() throws Exception {
final String jobResourcePath = resourcePath + "/" + Ray.getRuntimeContext().getCurrentJobId();
File jobResourceDir = new File(jobResourcePath);
File jobResourceDir = new File(codeSearchPath);
FileUtils.deleteQuietly(jobResourceDir);
jobResourceDir.mkdirs();
jobResourceDir.deleteOnExit();
// In this test case the class is expected to be loaded from the job resource path, so we need
// to put the compiled class file into the job resource path and load it later.
// In this test case the class is expected to be loaded from the job code search path,
// so we need to put the compiled class file into the job code search path and load it
// later.
String testJavaFile = ""
+ "import java.lang.management.ManagementFactory;\n"
+ "import java.lang.management.RuntimeMXBean;\n"
@@ -83,14 +82,14 @@ public class ClassLoaderTest extends BaseTest {
+ " }\n"
+ "}";
// Write the demo java file to the job resource path.
String javaFilePath = jobResourcePath + "/ClassLoaderTester.java";
// Write the demo java file to the job code search path.
String javaFilePath = codeSearchPath + "/ClassLoaderTester.java";
Files.write(Paths.get(javaFilePath), testJavaFile.getBytes());
// Compile the java file.
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
int result = compiler.run(null, null, null, "-d",
jobResourcePath, javaFilePath);
codeSearchPath, javaFilePath);
if (result != 0) {
throw new RuntimeException("Couldn't compile ClassLoaderTester.java.");
}