wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 05:43:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Java] Allow passing internal config from raylet to Java worker (#7532)

Browse Source
This commit is contained in:
Kai Yang
2020-03-15 12:03:38 +08:00
committed by GitHub
parent a87199d240
commit 630e48967d
17 changed files with 224 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java
+2 -2
View File
@@ -61,7 +61,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
} catch (IOException e) {
throw new RuntimeException("Failed to create the log directory.", e);
}
nativeSetup(rayConfig.logDir);
nativeSetup(rayConfig.logDir, rayConfig.rayletConfigParameters);
Runtime.getRuntime().addShutdownHook(new Thread(RayNativeRuntime::nativeShutdownHook));
}
@@ -193,7 +193,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
private static native void nativeDestroyCoreWorker(long nativeCoreWorkerPointer);
private static native void nativeSetup(String logDir);
private static native void nativeSetup(String logDir, Map<String, String> rayletConfigParameters);
private static native void nativeShutdownHook();
java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java
+5 -5
View File
@@ -10,7 +10,7 @@ import com.typesafe.config.ConfigValue;
import java.io.File;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -67,7 +67,7 @@ public class RayConfig {
public String rayletSocketName;
private int nodeManagerPort;
public final List<String> rayletConfigParameters;
public final Map<String, String> rayletConfigParameters;
public final String jobResourcePath;
public final String pythonWorkerCommand;
@@ -204,11 +204,11 @@ public class RayConfig {
}
// Raylet parameters.
rayletConfigParameters = new ArrayList<>();
rayletConfigParameters = new HashMap<>();
Config rayletConfig = config.getConfig("ray.raylet.config");
for (Map.Entry<String, ConfigValue> entry : rayletConfig.entrySet()) {
String parameter = entry.getKey() + "," + entry.getValue().unwrapped();
rayletConfigParameters.add(parameter);
Object value = entry.getValue().unwrapped();
rayletConfigParameters.put(entry.getKey(), value == null ? "" : value.toString());
}
// Job resource path.
java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java
+9 -4
View File
@@ -240,7 +240,10 @@ public class RunManager {
gcsServerFile.getAbsolutePath(),
String.format("--redis_address=%s", rayConfig.getRedisIp()),
String.format("--redis_port=%d", rayConfig.getRedisPort()),
String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)),
String.format("--config_list=%s",
rayConfig.rayletConfigParameters.entrySet().stream()
.map(entry -> entry.getKey() + "," + entry.getValue()).collect(Collectors
.joining(","))),
String.format("--redis_password=%s", redisPasswordOption)
);
startProcess(command, null, "gcs_server");
@@ -316,7 +319,9 @@ public class RunManager {
String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency),
String.format("--static_resource_list=%s",
ResourceUtil.getResourcesStringFromMap(rayConfig.resources)),
String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)),
String.format("--config_list=%s", rayConfig.rayletConfigParameters.entrySet().stream()
.map(entry -> entry.getKey() + "," + entry.getValue())
.collect(Collectors.joining(","))),
String.format("--python_worker_command=%s", buildPythonWorkerCommand()),
String.format("--java_worker_command=%s", buildWorkerCommand()),
String.format("--redis_password=%s", redisPasswordOption)
@@ -378,8 +383,8 @@ public class RunManager {
cmd.add("-Dray.redis.password=" + rayConfig.headRedisPassword);
}
// Number of workers per Java worker process
cmd.add("-Dray.raylet.config.num_workers_per_process_java=RAY_WORKER_NUM_WORKERS_PLACEHOLDER");
cmd.add("RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER");
cmd.addAll(rayConfig.jvmParameters);